From 47233c8beeb243b24e8a460af1dbf7befbc8281e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 12 Jun 2018 12:24:51 -0500 Subject: [PATCH 01/33] replace PQL parser with one created by PEG parser generator --- Makefile | 15 +- http/handler_test.go | 4 +- pql/ast.go | 138 ++++ pql/parser.go | 299 +-------- pql/pql.peg | 44 ++ pql/pql.peg.go | 1518 ++++++++++++++++++++++++++++++++++++++++++ pql/pqlpeg_test.go | 16 + pql/scanner.go | 303 --------- pql/scanner_test.go | 74 -- pql/token.go | 58 -- 10 files changed, 1747 insertions(+), 722 deletions(-) create mode 100644 pql/pql.peg create mode 100644 pql/pql.peg.go create mode 100644 pql/pqlpeg_test.go delete mode 100644 pql/scanner.go delete mode 100644 pql/scanner_test.go diff --git a/Makefile b/Makefile index 821ba33b6..c7aefa3e0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -92,8 +92,11 @@ generate-protoc: require-protoc require-protoc-gen-gofast generate-stringer: go generate github.com/pilosa/pilosa +generate-pql: require-peg + cd pql && peg -inline -switch pql.peg && cd .. + # `go generate` all needed packages -generate: generate-protoc generate-stringer +generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile docker: @@ -128,7 +131,10 @@ require-protoc-gen-gofast: require-protoc: $(call require,protoc) -install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer +require-peg: + $(call require,peg) + +install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg install-dep: go get -u github.com/golang/dep/cmd/dep @@ -141,3 +147,6 @@ install-protoc-gen-gofast: install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html + +install-peg: + go get github.com/pointlander/peg diff --git a/http/handler_test.go b/http/handler_test.go index 93f9906b2..a6bd2b98e 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -653,8 +653,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { - t.Fatalf("unexpected body: %s", body) + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near PegText (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { + t.Fatalf("unexpected body: \n%s", body) } } diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..7a91ef987 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -26,6 +26,144 @@ import ( // Query represents a PQL query. type Query struct { Calls []*Call + + lastField string + lastCond Token + inList bool + callStack []*Call +} + +func (q *Query) startCall(name string) { + newCall := &Call{Name: name} + q.callStack = append(q.callStack, newCall) + + if len(q.callStack) == 1 { + q.Calls = append(q.Calls, newCall) + } else { + calls := q.callStack[len(q.callStack)-2].Children + q.callStack[len(q.callStack)-2].Children = append(calls, newCall) + } + +} + +func (q *Query) endCall() { + q.callStack = q.callStack[:len(q.callStack)-1] +} + +func (q *Query) addField(field string) { + if q.lastField != "" { + panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) + } + q.lastField = field + call := q.callStack[len(q.callStack)-1] + if call.Args == nil { + call.Args = make(map[string]interface{}) + } +} + +func (q *Query) addVal(val interface{}) { + if q.lastField == "" { + panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, val) + return + } + if q.lastCond != ILLEGAL { + if val != nil || q.lastCond != NEQ { + panic(fmt.Sprintf("can't add val %s with condition %s", val, q.lastCond)) + } + call.Args[q.lastField] = &Condition{ + Op: NEQ, + Value: val, + } + } else { + call.Args[q.lastField] = val + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addNumVal(val string) { + if q.lastField == "" { + panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val)) + } + var ival interface{} + var err error + if strings.Contains(val, ".") { + ival, err = strconv.ParseFloat(val, 64) + } else { + ival, err = strconv.ParseInt(val, 10, 64) + } + if err != nil { + panic(err) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + if q.lastCond != ILLEGAL { + list := call.Args[q.lastField].(*Condition).Value.([]interface{}) + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: append(list, ival), + } + } else { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, ival) + } + return + } else if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: ival, + } + } else { + call.Args[q.lastField] = ival + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) startList() { + call := q.callStack[len(q.callStack)-1] + if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: make([]interface{}, 0), + } + } else { + call.Args[q.lastField] = make([]interface{}, 0) + } + q.inList = true +} + +func (q *Query) endList() { + q.inList = false + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addGT() { + q.lastCond = GT +} +func (q *Query) addLT() { + q.lastCond = LT +} +func (q *Query) addGTE() { + q.lastCond = GTE +} +func (q *Query) addLTE() { + q.lastCond = LTE +} +func (q *Query) addEQ() { + q.lastCond = EQ +} +func (q *Query) addNEQ() { + q.lastCond = NEQ +} +func (q *Query) addBTWN() { + q.lastCond = BETWEEN } // WriteCallN returns the number of mutating calls. diff --git a/pql/parser.go b/pql/parser.go index 3af0cbc9c..83498f207 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,10 +15,11 @@ package pql import ( - "fmt" "io" - "strconv" + "io/ioutil" "strings" + + "github.com/pkg/errors" ) // TimeFormat is the go-style time format used to parse string dates. @@ -26,13 +27,16 @@ const TimeFormat = "2006-01-02T15:04" // Parser represents a parser for the PQL language. type Parser struct { - scanner *bufScanner + r io.Reader + //scanner *bufScanner + PQL } // NewParser returns a new instance of Parser. func NewParser(r io.Reader) *Parser { return &Parser{ - scanner: newBufScanner(r), + r: r, + // scanner: newBufScanner(r), } } @@ -43,287 +47,18 @@ func ParseString(s string) (*Query, error) { // Parse parses the next node in the query. func (p *Parser) Parse() (*Query, error) { - q := &Query{} - for { - call, err := p.parseCall() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - q.Calls = append(q.Calls, call) - } - - // Require at least one call. - if len(q.Calls) == 0 { - return nil, io.ErrUnexpectedEOF - } - - return q, nil -} - -// parseCall parses the next function call. -func (p *Parser) parseCall() (*Call, error) { - var c Call - - // Read call name. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == EOF { - return nil, io.EOF - } else if tok != IDENT { - return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} - } - c.Name = lit - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse children first. - children, err := p.parseChildren() + buf, err := ioutil.ReadAll(p.r) if err != nil { - return nil, err + return nil, errors.Wrap(err, "reading buffer to parse") } - c.Children = children - - // If next token is a closing paren then exit. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - return &c, nil - } else if tok == IDENT { - p.unscan(1) - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + p.PQL = PQL{ + Buffer: string(buf), } - - // Parse key/value arguments. - args, err := p.parseArgs() + p.Init() + err = p.PQL.Parse() if err != nil { - return nil, err - } - c.Args = args - - // Scan closing parenthesis. - if err := p.expect(RPAREN); err != nil { - return nil, err - } - - return &c, nil -} - -// parseChildren parses call children. -func (p *Parser) parseChildren() ([]*Call, error) { - var offset int - var children []*Call - for { - // Ensure next two tokens are IDENT+LPAREN. - if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { - p.unscanIgnoreWhitespace(1 + offset) - return children, nil - } - if tok, _, _ := p.scan(); tok != LPAREN { - p.unscanIgnoreWhitespace(2 + offset) - return children, nil - } - - // Push tokens back on scanner and parse as a call. - p.unscan(2) - child, err := p.parseCall() - if err != nil { - return nil, err - } - children = append(children, child) - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return children, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - - // Make sure comma is unscanned. - offset = 1 - } -} - -// parseArgs parses key/value arguments. -func (p *Parser) parseArgs() (map[string]interface{}, error) { - args := make(map[string]interface{}) - for { - // Parse key. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != IDENT { - return nil, parseErrorf(pos, "expected argument key, found %q", lit) - } - key := lit - - // Expect '=' or a comparison next. - var op Token - switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { - case ASSIGN: - case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: - op = tok - default: - return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) - } - - // Parse value. - var value interface{} - tok, pos, lit = p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - value = true - } else if lit == "false" { - value = false - } else if lit == "null" { - value = nil - } else { - value = lit - } - case STRING: - value = lit - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - value = v - case FLOAT: - v, err := strconv.ParseFloat(lit, 64) - if err != nil { - return nil, err - } - value = v - case LBRACK: - v, err := p.parseList() - if err != nil { - return nil, err - } - value = v - default: - return nil, parseErrorf(pos, "invalid argument value: %q", lit) - } - - // Ensure key doesn't already exist. - if _, ok := args[key]; ok { - return nil, parseErrorf(pos, "argument key already used: %s", key) - } - - // If op is specified then create a condition. - if op != 0 { - value = &Condition{Op: op, Value: value} - } - - // Add key/value pair to arguments. - args[key] = value - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - } -} - -// parseList parses a list of primitives. This is used by the TopN() filters. -func (p *Parser) parseList() ([]interface{}, error) { - var values []interface{} - for { - // Read next value. - tok, pos, lit := p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - values = append(values, true) - } else if lit == "false" { - values = append(values, false) - } else { - values = append(values, lit) - } - case STRING: - values = append(values, lit) - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - values = append(values, v) - default: - return nil, parseErrorf(pos, "invalid list value: %q", lit) - } - - // Expect a comma or closing bracket next. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { - break - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, found %q", lit) - } - } - return values, nil -} - -// scan returns the next token from the scanner. -func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } - -// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. -func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { - tok, pos, lit = p.scan() - if tok == WS { - tok, pos, lit = p.scan() - } - return -} - -// unscan returns the last n tokens back to the scanner. -func (p *Parser) unscan(n int) { - for i := 0; i < n; i++ { - p.scanner.unscan() - } -} - -// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. -func (p *Parser) unscanIgnoreWhitespace(n int) { - for i := 0; i < n; { - p.scanner.unscan() - if tok, _, _ := p.scanner.curr(); tok != WS { - i++ - } - } -} - -// expect returns an error if the next token is not exp. -func (p *Parser) expect(exp Token) error { - if tok, pos, lit := p.scan(); tok != exp { - return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) - } - return nil -} - -// pos returns the current position. -func (p *Parser) pos() Pos { return p.scanner.pos() } - -// ParseError represents an error that occurred while parsing a PQL query. -type ParseError struct { - Message string - Pos Pos -} - -// Error returns a string representation of e. -func (e *ParseError) Error() string { - return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) -} - -// parseErrorf returns a formatted parse error. -func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { - return &ParseError{ - Message: fmt.Sprintf(format, args...), - Pos: pos, + return nil, errors.Wrap(err, "parsing") } + p.Execute() + return &p.Query, nil } diff --git a/pql/pql.peg b/pql/pql.peg new file mode 100644 index 000000000..0d9aeeb66 --- /dev/null +++ b/pql/pql.peg @@ -0,0 +1,44 @@ +package pql + +type PQL Peg { + Query +} + + +Calls <- Call* !. +Call <- newline* < [[A-Z]]+ > { p.startCall(buffer[begin:end] ) } open args close newline* { p.endCall() } +args <- arg (comma args)? sp / sp +arg <- ( Call + / field sp '=' sp value + / field sp COND sp value + ) +COND <- ( '><' { p.addBTWN() } + / '<=' { p.addLTE() } + / '>=' { p.addGTE() } + / '==' { p.addEQ() } + / '!=' { p.addNEQ() } + / '<' { p.addLT() } + / '>' { p.addGT() } + ) +open <- '(' sp +value <- ( item + / lbrack { p.startList() } list rbrack { p.endList() } + ) +list <- item (comma list)? +item <- ( 'null' { p.addVal(nil) } + / 'true' { p.addVal(true) } + / 'false' { p.addVal(false) } + / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } + / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } + / '"' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '"' { p.addVal(buffer[begin:end]) } + / '\'' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '\'' { p.addVal(buffer[begin:end]) } + ) + +field <- < [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* > { p.addField(buffer[begin:end]) } +close <- ')' sp +sp <- ( ' ' / '\t' )* +comma <- sp ',' sp +lbrack <- '[' sp +rbrack <- sp ']' sp +newline <- sp '\n' sp \ No newline at end of file diff --git a/pql/pql.peg.go b/pql/pql.peg.go new file mode 100644 index 000000000..3a2b25315 --- /dev/null +++ b/pql/pql.peg.go @@ -0,0 +1,1518 @@ +package pql + +//go:generate peg -inline -switch pql.peg + +import ( + "fmt" + "math" + "sort" + "strconv" +) + +const endSymbol rune = 1114112 + +/* The rule types inferred from the grammar are below. */ +type pegRule uint8 + +const ( + ruleUnknown pegRule = iota + ruleCalls + ruleCall + ruleargs + rulearg + ruleCOND + ruleopen + rulevalue + rulelist + ruleitem + rulefield + ruleclose + rulesp + rulecomma + rulelbrack + rulerbrack + rulenewline + rulePegText + ruleAction0 + ruleAction1 + ruleAction2 + ruleAction3 + ruleAction4 + ruleAction5 + ruleAction6 + ruleAction7 + ruleAction8 + ruleAction9 + ruleAction10 + ruleAction11 + ruleAction12 + ruleAction13 + ruleAction14 + ruleAction15 + ruleAction16 + ruleAction17 + ruleAction18 + ruleAction19 +) + +var rul3s = [...]string{ + "Unknown", + "Calls", + "Call", + "args", + "arg", + "COND", + "open", + "value", + "list", + "item", + "field", + "close", + "sp", + "comma", + "lbrack", + "rbrack", + "newline", + "PegText", + "Action0", + "Action1", + "Action2", + "Action3", + "Action4", + "Action5", + "Action6", + "Action7", + "Action8", + "Action9", + "Action10", + "Action11", + "Action12", + "Action13", + "Action14", + "Action15", + "Action16", + "Action17", + "Action18", + "Action19", +} + +type token32 struct { + pegRule + begin, end uint32 +} + +func (t *token32) String() string { + return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) +} + +type node32 struct { + token32 + up, next *node32 +} + +func (node *node32) print(pretty bool, buffer string) { + var print func(node *node32, depth int) + print = func(node *node32, depth int) { + for node != nil { + for c := 0; c < depth; c++ { + fmt.Printf(" ") + } + rule := rul3s[node.pegRule] + quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) + if !pretty { + fmt.Printf("%v %v\n", rule, quote) + } else { + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) + } + if node.up != nil { + print(node.up, depth+1) + } + node = node.next + } + } + print(node, 0) +} + +func (node *node32) Print(buffer string) { + node.print(false, buffer) +} + +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) +} + +type tokens32 struct { + tree []token32 +} + +func (t *tokens32) Trim(length uint32) { + t.tree = t.tree[:length] +} + +func (t *tokens32) Print() { + for _, token := range t.tree { + fmt.Println(token.String()) + } +} + +func (t *tokens32) AST() *node32 { + type element struct { + node *node32 + down *element + } + tokens := t.Tokens() + var stack *element + for _, token := range tokens { + if token.begin == token.end { + continue + } + node := &node32{token32: token} + for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { + stack.node.next = node.up + node.up = stack.node + stack = stack.down + } + stack = &element{node: node, down: stack} + } + if stack != nil { + return stack.node + } + return nil +} + +func (t *tokens32) PrintSyntaxTree(buffer string) { + t.AST().Print(buffer) +} + +func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { + t.AST().PrettyPrint(buffer) +} + +func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, + } +} + +func (t *tokens32) Tokens() []token32 { + return t.tree +} + +type PQL struct { + Query + + Buffer string + buffer []rune + rules [38]func() bool + parse func(rule ...int) error + reset func() + Pretty bool + tokens32 +} + +func (p *PQL) Parse(rule ...int) error { + return p.parse(rule...) +} + +func (p *PQL) Reset() { + p.reset() +} + +type textPosition struct { + line, symbol int +} + +type textPositionMap map[int]textPosition + +func translatePositions(buffer []rune, positions []int) textPositionMap { + length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 + sort.Ints(positions) + +search: + for i, c := range buffer { + if c == '\n' { + line, symbol = line+1, 0 + } else { + symbol++ + } + if i == positions[j] { + translations[positions[j]] = textPosition{line, symbol} + for j++; j < length; j++ { + if i != positions[j] { + continue search + } + } + break search + } + } + + return translations +} + +type parseError struct { + p *PQL + max token32 +} + +func (e *parseError) Error() string { + tokens, error := []token32{e.max}, "\n" + positions, p := make([]int, 2*len(tokens)), 0 + for _, token := range tokens { + positions[p], p = int(token.begin), p+1 + positions[p], p = int(token.end), p+1 + } + translations := translatePositions(e.p.buffer, positions) + format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" + if e.p.Pretty { + format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" + } + for _, token := range tokens { + begin, end := int(token.begin), int(token.end) + error += fmt.Sprintf(format, + rul3s[token.pegRule], + translations[begin].line, translations[begin].symbol, + translations[end].line, translations[end].symbol, + strconv.Quote(string(e.p.buffer[begin:end]))) + } + + return error +} + +func (p *PQL) PrintSyntaxTree() { + if p.Pretty { + p.tokens32.PrettyPrintSyntaxTree(p.Buffer) + } else { + p.tokens32.PrintSyntaxTree(p.Buffer) + } +} + +func (p *PQL) Execute() { + buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 + for _, token := range p.Tokens() { + switch token.pegRule { + + case rulePegText: + begin, end = int(token.begin), int(token.end) + text = string(_buffer[begin:end]) + + case ruleAction0: + p.startCall(buffer[begin:end]) + case ruleAction1: + p.endCall() + case ruleAction2: + p.addBTWN() + case ruleAction3: + p.addLTE() + case ruleAction4: + p.addGTE() + case ruleAction5: + p.addEQ() + case ruleAction6: + p.addNEQ() + case ruleAction7: + p.addLT() + case ruleAction8: + p.addGT() + case ruleAction9: + p.startList() + case ruleAction10: + p.endList() + case ruleAction11: + p.addVal(nil) + case ruleAction12: + p.addVal(true) + case ruleAction13: + p.addVal(false) + case ruleAction14: + p.addNumVal(buffer[begin:end]) + case ruleAction15: + p.addNumVal(buffer[begin:end]) + case ruleAction16: + p.addVal(buffer[begin:end]) + case ruleAction17: + p.addVal(buffer[begin:end]) + case ruleAction18: + p.addVal(buffer[begin:end]) + case ruleAction19: + p.addField(buffer[begin:end]) + + } + } + _, _, _, _, _ = buffer, _buffer, text, begin, end +} + +func (p *PQL) Init() { + var ( + max token32 + position, tokenIndex uint32 + buffer []rune + ) + p.reset = func() { + max = token32{} + position, tokenIndex = 0, 0 + + p.buffer = []rune(p.Buffer) + if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { + p.buffer = append(p.buffer, endSymbol) + } + buffer = p.buffer + } + p.reset() + + _rules := p.rules + tree := tokens32{tree: make([]token32, math.MaxInt16)} + p.parse = func(rule ...int) error { + r := 1 + if len(rule) > 0 { + r = rule[0] + } + matches := p.rules[r]() + p.tokens32 = tree + if matches { + p.Trim(tokenIndex) + return nil + } + return &parseError{p, max} + } + + add := func(rule pegRule, begin uint32) { + tree.Add(rule, begin, position, tokenIndex) + tokenIndex++ + if begin != position && position > max.end { + max = token32{rule, begin, position} + } + } + + matchDot := func() bool { + if buffer[position] != endSymbol { + position++ + return true + } + return false + } + + /*matchChar := func(c byte) bool { + if buffer[position] == c { + position++ + return true + } + return false + }*/ + + /*matchRange := func(lower byte, upper byte) bool { + if c := buffer[position]; c >= lower && c <= upper { + position++ + return true + } + return false + }*/ + + _rules = [...]func() bool{ + nil, + /* 0 Calls <- <(Call* !.)> */ + func() bool { + position0, tokenIndex0 := position, tokenIndex + { + position1 := position + l2: + { + position3, tokenIndex3 := position, tokenIndex + if !_rules[ruleCall]() { + goto l3 + } + goto l2 + l3: + position, tokenIndex = position3, tokenIndex3 + } + { + position4, tokenIndex4 := position, tokenIndex + if !matchDot() { + goto l4 + } + goto l0 + l4: + position, tokenIndex = position4, tokenIndex4 + } + add(ruleCalls, position1) + } + return true + l0: + position, tokenIndex = position0, tokenIndex0 + return false + }, + /* 1 Call <- <(newline* <([a-z] / [A-Z])+> Action0 open args close newline* Action1)> */ + func() bool { + position5, tokenIndex5 := position, tokenIndex + { + position6 := position + l7: + { + position8, tokenIndex8 := position, tokenIndex + if !_rules[rulenewline]() { + goto l8 + } + goto l7 + l8: + position, tokenIndex = position8, tokenIndex8 + } + { + position9 := position + { + position12, tokenIndex12 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l13 + } + position++ + goto l12 + l13: + position, tokenIndex = position12, tokenIndex12 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l5 + } + position++ + } + l12: + l10: + { + position11, tokenIndex11 := position, tokenIndex + { + position14, tokenIndex14 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l15 + } + position++ + goto l14 + l15: + position, tokenIndex = position14, tokenIndex14 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l11 + } + position++ + } + l14: + goto l10 + l11: + position, tokenIndex = position11, tokenIndex11 + } + add(rulePegText, position9) + } + { + add(ruleAction0, position) + } + { + position17 := position + if buffer[position] != rune('(') { + goto l5 + } + position++ + if !_rules[rulesp]() { + goto l5 + } + add(ruleopen, position17) + } + if !_rules[ruleargs]() { + goto l5 + } + { + position18 := position + if buffer[position] != rune(')') { + goto l5 + } + position++ + if !_rules[rulesp]() { + goto l5 + } + add(ruleclose, position18) + } + l19: + { + position20, tokenIndex20 := position, tokenIndex + if !_rules[rulenewline]() { + goto l20 + } + goto l19 + l20: + position, tokenIndex = position20, tokenIndex20 + } + { + add(ruleAction1, position) + } + add(ruleCall, position6) + } + return true + l5: + position, tokenIndex = position5, tokenIndex5 + return false + }, + /* 2 args <- <((arg (comma args)? sp) / sp)> */ + func() bool { + position22, tokenIndex22 := position, tokenIndex + { + position23 := position + { + position24, tokenIndex24 := position, tokenIndex + { + position26 := position + { + position27, tokenIndex27 := position, tokenIndex + if !_rules[ruleCall]() { + goto l28 + } + goto l27 + l28: + position, tokenIndex = position27, tokenIndex27 + if !_rules[rulefield]() { + goto l29 + } + if !_rules[rulesp]() { + goto l29 + } + if buffer[position] != rune('=') { + goto l29 + } + position++ + if !_rules[rulesp]() { + goto l29 + } + if !_rules[rulevalue]() { + goto l29 + } + goto l27 + l29: + position, tokenIndex = position27, tokenIndex27 + if !_rules[rulefield]() { + goto l25 + } + if !_rules[rulesp]() { + goto l25 + } + { + position30 := position + { + position31, tokenIndex31 := position, tokenIndex + if buffer[position] != rune('>') { + goto l32 + } + position++ + if buffer[position] != rune('<') { + goto l32 + } + position++ + { + add(ruleAction2, position) + } + goto l31 + l32: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('<') { + goto l34 + } + position++ + if buffer[position] != rune('=') { + goto l34 + } + position++ + { + add(ruleAction3, position) + } + goto l31 + l34: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('>') { + goto l36 + } + position++ + if buffer[position] != rune('=') { + goto l36 + } + position++ + { + add(ruleAction4, position) + } + goto l31 + l36: + position, tokenIndex = position31, tokenIndex31 + { + switch buffer[position] { + case '>': + if buffer[position] != rune('>') { + goto l25 + } + position++ + { + add(ruleAction8, position) + } + break + case '<': + if buffer[position] != rune('<') { + goto l25 + } + position++ + { + add(ruleAction7, position) + } + break + case '!': + if buffer[position] != rune('!') { + goto l25 + } + position++ + if buffer[position] != rune('=') { + goto l25 + } + position++ + { + add(ruleAction6, position) + } + break + default: + if buffer[position] != rune('=') { + goto l25 + } + position++ + if buffer[position] != rune('=') { + goto l25 + } + position++ + { + add(ruleAction5, position) + } + break + } + } + + } + l31: + add(ruleCOND, position30) + } + if !_rules[rulesp]() { + goto l25 + } + if !_rules[rulevalue]() { + goto l25 + } + } + l27: + add(rulearg, position26) + } + { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulecomma]() { + goto l43 + } + if !_rules[ruleargs]() { + goto l43 + } + goto l44 + l43: + position, tokenIndex = position43, tokenIndex43 + } + l44: + if !_rules[rulesp]() { + goto l25 + } + goto l24 + l25: + position, tokenIndex = position24, tokenIndex24 + if !_rules[rulesp]() { + goto l22 + } + } + l24: + add(ruleargs, position23) + } + return true + l22: + position, tokenIndex = position22, tokenIndex22 + return false + }, + /* 3 arg <- <(Call / (field sp '=' sp value) / (field sp COND sp value))> */ + nil, + /* 4 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ((&('>') ('>' Action8)) | (&('<') ('<' Action7)) | (&('!') ('!' '=' Action6)) | (&('=') ('=' '=' Action5))))> */ + nil, + /* 5 open <- <('(' sp)> */ + nil, + /* 6 value <- <(item / (lbrack Action9 list rbrack Action10))> */ + func() bool { + position48, tokenIndex48 := position, tokenIndex + { + position49 := position + { + position50, tokenIndex50 := position, tokenIndex + if !_rules[ruleitem]() { + goto l51 + } + goto l50 + l51: + position, tokenIndex = position50, tokenIndex50 + { + position52 := position + if buffer[position] != rune('[') { + goto l48 + } + position++ + if !_rules[rulesp]() { + goto l48 + } + add(rulelbrack, position52) + } + { + add(ruleAction9, position) + } + if !_rules[rulelist]() { + goto l48 + } + { + position54 := position + if !_rules[rulesp]() { + goto l48 + } + if buffer[position] != rune(']') { + goto l48 + } + position++ + if !_rules[rulesp]() { + goto l48 + } + add(rulerbrack, position54) + } + { + add(ruleAction10, position) + } + } + l50: + add(rulevalue, position49) + } + return true + l48: + position, tokenIndex = position48, tokenIndex48 + return false + }, + /* 7 list <- <(item (comma list)?)> */ + func() bool { + position56, tokenIndex56 := position, tokenIndex + { + position57 := position + if !_rules[ruleitem]() { + goto l56 + } + { + position58, tokenIndex58 := position, tokenIndex + if !_rules[rulecomma]() { + goto l58 + } + if !_rules[rulelist]() { + goto l58 + } + goto l59 + l58: + position, tokenIndex = position58, tokenIndex58 + } + l59: + add(rulelist, position57) + } + return true + l56: + position, tokenIndex = position56, tokenIndex56 + return false + }, + /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / ((&('\'') ('\'' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '\'' Action18)) | (&('"') ('"' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '"' Action17)) | (&('-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '_' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') (<((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> Action16))))> */ + func() bool { + position60, tokenIndex60 := position, tokenIndex + { + position61 := position + { + position62, tokenIndex62 := position, tokenIndex + if buffer[position] != rune('n') { + goto l63 + } + position++ + if buffer[position] != rune('u') { + goto l63 + } + position++ + if buffer[position] != rune('l') { + goto l63 + } + position++ + if buffer[position] != rune('l') { + goto l63 + } + position++ + { + add(ruleAction11, position) + } + goto l62 + l63: + position, tokenIndex = position62, tokenIndex62 + if buffer[position] != rune('t') { + goto l65 + } + position++ + if buffer[position] != rune('r') { + goto l65 + } + position++ + if buffer[position] != rune('u') { + goto l65 + } + position++ + if buffer[position] != rune('e') { + goto l65 + } + position++ + { + add(ruleAction12, position) + } + goto l62 + l65: + position, tokenIndex = position62, tokenIndex62 + if buffer[position] != rune('f') { + goto l67 + } + position++ + if buffer[position] != rune('a') { + goto l67 + } + position++ + if buffer[position] != rune('l') { + goto l67 + } + position++ + if buffer[position] != rune('s') { + goto l67 + } + position++ + if buffer[position] != rune('e') { + goto l67 + } + position++ + { + add(ruleAction13, position) + } + goto l62 + l67: + position, tokenIndex = position62, tokenIndex62 + { + position70 := position + { + position71, tokenIndex71 := position, tokenIndex + if buffer[position] != rune('-') { + goto l71 + } + position++ + goto l72 + l71: + position, tokenIndex = position71, tokenIndex71 + } + l72: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l69 + } + position++ + l73: + { + position74, tokenIndex74 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l74 + } + position++ + goto l73 + l74: + position, tokenIndex = position74, tokenIndex74 + } + { + position75, tokenIndex75 := position, tokenIndex + if buffer[position] != rune('.') { + goto l75 + } + position++ + l77: + { + position78, tokenIndex78 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l78 + } + position++ + goto l77 + l78: + position, tokenIndex = position78, tokenIndex78 + } + goto l76 + l75: + position, tokenIndex = position75, tokenIndex75 + } + l76: + add(rulePegText, position70) + } + { + add(ruleAction14, position) + } + goto l62 + l69: + position, tokenIndex = position62, tokenIndex62 + { + position81 := position + { + position82, tokenIndex82 := position, tokenIndex + if buffer[position] != rune('-') { + goto l82 + } + position++ + goto l83 + l82: + position, tokenIndex = position82, tokenIndex82 + } + l83: + if buffer[position] != rune('.') { + goto l80 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l80 + } + position++ + l84: + { + position85, tokenIndex85 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l85 + } + position++ + goto l84 + l85: + position, tokenIndex = position85, tokenIndex85 + } + add(rulePegText, position81) + } + { + add(ruleAction15, position) + } + goto l62 + l80: + position, tokenIndex = position62, tokenIndex62 + { + switch buffer[position] { + case '\'': + if buffer[position] != rune('\'') { + goto l60 + } + position++ + { + position88 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l89: + { + position90, tokenIndex90 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l90 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l90 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l90 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l90 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l90 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l90 + } + position++ + break + } + } + + goto l89 + l90: + position, tokenIndex = position90, tokenIndex90 + } + add(rulePegText, position88) + } + if buffer[position] != rune('\'') { + goto l60 + } + position++ + { + add(ruleAction18, position) + } + break + case '"': + if buffer[position] != rune('"') { + goto l60 + } + position++ + { + position94 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l95: + { + position96, tokenIndex96 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l96 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l96 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l96 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l96 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l96 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l96 + } + position++ + break + } + } + + goto l95 + l96: + position, tokenIndex = position96, tokenIndex96 + } + add(rulePegText, position94) + } + if buffer[position] != rune('"') { + goto l60 + } + position++ + { + add(ruleAction17, position) + } + break + default: + { + position100 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l101: + { + position102, tokenIndex102 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l102 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l102 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l102 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l102 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l102 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l102 + } + position++ + break + } + } + + goto l101 + l102: + position, tokenIndex = position102, tokenIndex102 + } + add(rulePegText, position100) + } + { + add(ruleAction16, position) + } + break + } + } + + } + l62: + add(ruleitem, position61) + } + return true + l60: + position, tokenIndex = position60, tokenIndex60 + return false + }, + /* 9 field <- <(<(([a-z] / [A-Z]) ((&('_') '_') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))*)> Action19)> */ + func() bool { + position106, tokenIndex106 := position, tokenIndex + { + position107 := position + { + position108 := position + { + position109, tokenIndex109 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l110 + } + position++ + goto l109 + l110: + position, tokenIndex = position109, tokenIndex109 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l106 + } + position++ + } + l109: + l111: + { + position112, tokenIndex112 := position, tokenIndex + { + switch buffer[position] { + case '_': + if buffer[position] != rune('_') { + goto l112 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l112 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l112 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l112 + } + position++ + break + } + } + + goto l111 + l112: + position, tokenIndex = position112, tokenIndex112 + } + add(rulePegText, position108) + } + { + add(ruleAction19, position) + } + add(rulefield, position107) + } + return true + l106: + position, tokenIndex = position106, tokenIndex106 + return false + }, + /* 10 close <- <(')' sp)> */ + nil, + /* 11 sp <- <(' ' / '\t')*> */ + func() bool { + { + position117 := position + l118: + { + position119, tokenIndex119 := position, tokenIndex + { + position120, tokenIndex120 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l121 + } + position++ + goto l120 + l121: + position, tokenIndex = position120, tokenIndex120 + if buffer[position] != rune('\t') { + goto l119 + } + position++ + } + l120: + goto l118 + l119: + position, tokenIndex = position119, tokenIndex119 + } + add(rulesp, position117) + } + return true + }, + /* 12 comma <- <(sp ',' sp)> */ + func() bool { + position122, tokenIndex122 := position, tokenIndex + { + position123 := position + if !_rules[rulesp]() { + goto l122 + } + if buffer[position] != rune(',') { + goto l122 + } + position++ + if !_rules[rulesp]() { + goto l122 + } + add(rulecomma, position123) + } + return true + l122: + position, tokenIndex = position122, tokenIndex122 + return false + }, + /* 13 lbrack <- <('[' sp)> */ + nil, + /* 14 rbrack <- <(sp ']' sp)> */ + nil, + /* 15 newline <- <(sp '\n' sp)> */ + func() bool { + position126, tokenIndex126 := position, tokenIndex + { + position127 := position + if !_rules[rulesp]() { + goto l126 + } + if buffer[position] != rune('\n') { + goto l126 + } + position++ + if !_rules[rulesp]() { + goto l126 + } + add(rulenewline, position127) + } + return true + l126: + position, tokenIndex = position126, tokenIndex126 + return false + }, + nil, + /* 18 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ + nil, + /* 19 Action1 <- <{ p.endCall() }> */ + nil, + /* 20 Action2 <- <{ p.addBTWN() }> */ + nil, + /* 21 Action3 <- <{ p.addLTE() }> */ + nil, + /* 22 Action4 <- <{ p.addGTE() }> */ + nil, + /* 23 Action5 <- <{ p.addEQ() }> */ + nil, + /* 24 Action6 <- <{ p.addNEQ() }> */ + nil, + /* 25 Action7 <- <{ p.addLT() }> */ + nil, + /* 26 Action8 <- <{ p.addGT() }> */ + nil, + /* 27 Action9 <- <{ p.startList() }> */ + nil, + /* 28 Action10 <- <{ p.endList() }> */ + nil, + /* 29 Action11 <- <{ p.addVal(nil) }> */ + nil, + /* 30 Action12 <- <{ p.addVal(true) }> */ + nil, + /* 31 Action13 <- <{ p.addVal(false) }> */ + nil, + /* 32 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 33 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 34 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 35 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 36 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 37 Action19 <- <{ p.addField(buffer[begin:end]) }> */ + nil, + } + p.rules = _rules +} diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go new file mode 100644 index 000000000..8b4884fb1 --- /dev/null +++ b/pql/pqlpeg_test.go @@ -0,0 +1,16 @@ +package pql + +import ( + "testing" +) + +func TestPEG(t *testing.T) { + p := PQL{Buffer: ` +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="zoo9")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]} + p.Init() + err := p.Parse() + if err != nil { + t.Fatalf("parse error: %v", err) + } + p.Execute() +} diff --git a/pql/scanner.go b/pql/scanner.go deleted file mode 100644 index 5a24b6af2..000000000 --- a/pql/scanner.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql - -import ( - "bufio" - "bytes" - "io" - "unicode" -) - -// Scanner represents a PQL lexical scanner. -type Scanner struct { - r io.RuneScanner - pos Pos -} - -// NewScanner returns a new instance of Scanner. -func NewScanner(r io.Reader) *Scanner { - return &Scanner{r: bufio.NewReader(r)} -} - -// Scan returns the next token and position from the underlying reader. -func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { - pos = s.pos - - // Read next code point. - ch := s.read() - - // If we see whitespace then consume all contiguous whitespace. - // If we see a letter, or certain acceptable special characters, then consume - // as an ident or reserved word. If we see quotes, then scan as string. - if isWhitespace(ch) { - s.unread() - return s.scanWhitespace() - } else if isIdentFirstChar(ch) { - s.unread() - return s.scanIdent() - } else if isDigit(ch) || ch == '-' { - s.unread() - return s.scanNumber() - } else if ch == '"' || ch == '\'' { - s.unread() - return s.scanString() - } - - // Otherwise parse individual characters. - switch ch { - case eof: - return EOF, pos, "" - case '=': - if next := s.read(); next == '=' { - return EQ, pos, "==" - } - s.unread() - return ASSIGN, pos, string(ch) - case '!': - if next := s.read(); next == '=' { - return NEQ, pos, "!=" - } - s.unread() - return ASSIGN, pos, string(ch) - case '<': - if next := s.read(); next == '=' { - return LTE, pos, "<=" - } - s.unread() - return LT, pos, string(ch) - case '>': - next := s.read() - if next == '=' { - return GTE, pos, ">=" - } else if next == '<' { - return BETWEEN, pos, "><" - } - s.unread() - return GT, pos, string(ch) - case ',': - return COMMA, pos, string(ch) - case '(': - return LPAREN, pos, string(ch) - case ')': - return RPAREN, pos, string(ch) - case '[': - return LBRACK, pos, string(ch) - case ']': - return RBRACK, pos, string(ch) - default: - return ILLEGAL, pos, string(ch) - } -} - -// read returns the next code point from the underlying reader and updates the pos. -func (s *Scanner) read() rune { - // Read next rune from underlying reader. - ch, _, err := s.r.ReadRune() - if err != nil { - return eof - } - - // Update position information. - if ch == '\n' { - s.pos.Line++ - s.pos.Char = 0 - } else { - s.pos.Char++ - } - - return ch -} - -// unread pushes the previously read rune back onto the reader. -func (s *Scanner) unread() { - if s.pos.Char == 0 { - s.pos.Line-- - } else { - s.pos.Char-- - } - - s.r.UnreadRune() -} - -// scanWhitespace consumes the current rune and all contiguous whitespace. -func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isWhitespace(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - - return WS, pos, buf.String() -} - -func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isIdentChar(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - lit = buf.String() - - // If the literal matches a keyword then return that keyword. - if tok = Lookup(lit); tok != IDENT { - return tok, pos, lit - } - - return IDENT, pos, lit -} - -// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. -func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { - pos = s.pos - tok = INTEGER - - var buf bytes.Buffer - var seenDot bool - first := true - for { - ch := s.read() - if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { - s.unread() - break - } - if ch == '.' { - seenDot = true - tok = FLOAT - } - buf.WriteRune(ch) - first = false - } - return tok, pos, buf.String() -} - -// scanString consumes a single-quoted or double-quoted string. -func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { - pos = s.pos - - // This must be either a single- or double-quote. - ending := s.read() - - var buf bytes.Buffer - for { - ch := s.read() - if ch == ending { - break - } else if ch == '\n' || ch == eof { - return BADSTRING, pos, buf.String() - } else if ch == '\\' { - next := s.read() - if next == 'n' { - buf.WriteRune('\n') - } else if next == '\\' { - buf.WriteRune('\\') - } else if next == '"' { - buf.WriteRune('"') - } else if next == '\'' { - buf.WriteRune('\'') - } else { - return BADSTRING, pos, buf.String() - } - } else { - buf.WriteRune(ch) - } - } - - return STRING, pos, buf.String() -} - -// bufScanner represents a wrapper for scanner to add a buffer. -// It provides a fixed-length circular buffer that can be unread. -type bufScanner struct { - s *Scanner - i int // buffer index - n int // buffer size - buf [8]struct { - tok Token - pos Pos - lit string - } -} - -// newBufScanner returns a new buffered scanner for a reader. -func newBufScanner(r io.Reader) *bufScanner { - return &bufScanner{s: NewScanner(r)} -} - -// Scan reads the next token from the scanner. -func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { - // If we have unread tokens then read them off the buffer first. - if s.n > 0 { - s.n-- - return s.curr() - } - - // Move buffer position forward and save the token. - s.i = (s.i + 1) % len(s.buf) - buf := &s.buf[s.i] - buf.tok, buf.pos, buf.lit = s.s.Scan() - - return s.curr() -} - -// unscan pushes the previously token back onto the buffer. -func (s *bufScanner) unscan() { s.n++ } - -// curr returns the last read token. -func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { - buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] - return buf.tok, buf.pos, buf.lit -} - -// pos returns the current position. -func (s *bufScanner) pos() Pos { - _, pos, _ := s.curr() - return pos -} - -// isWhitespace returns true if the rune a Unicode space character. -func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } - -// isLetter returns true if the rune is a letter. -func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } - -// isDigit returns true if the rune is a digit. -func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } - -// isIdentChar returns true if the rune can be used in an unquoted identifier. -func isIdentChar(ch rune) bool { - return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' -} - -// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. -func isIdentFirstChar(ch rune) bool { return isLetter(ch) } - -const eof = rune(0) diff --git a/pql/scanner_test.go b/pql/scanner_test.go deleted file mode 100644 index e48896748..000000000 --- a/pql/scanner_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/pql" -) - -func TestScanner_Scan(t *testing.T) { - var tests = []struct { - name string - s string - tok pql.Token - lit string - pos pql.Pos - }{ - // Special tokens (EOF, ILLEGAL, WS) - {name: "EOF", s: ``, tok: pql.EOF}, - {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, - {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, - {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, - {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, - - {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, - {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, - {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, - {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, - {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, - {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, - {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, - {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, - {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, - {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, - {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, - {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, - {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, - - {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, - {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, - {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, - - {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, - {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := pql.NewScanner(strings.NewReader(tt.s)) - tok, pos, lit := s.Scan() - if tt.tok != tok { - t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) - } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { - t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) - } else if tt.lit != lit { - t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) - } - }) - } -} diff --git a/pql/token.go b/pql/token.go index 6997f17af..51eea410d 100644 --- a/pql/token.go +++ b/pql/token.go @@ -14,28 +14,12 @@ package pql -import "strings" - // Token is a lexical token of the PQL language. type Token int const ( // Special tokens ILLEGAL Token = iota - EOF - WS - - literal_beg - IDENT // main - STRING // "foo" - BADSTRING // bad escape or unclosed string - INTEGER // 12345 - FLOAT // 100.2 - literal_end - - keyword_beg - ALL - keyword_end ASSIGN // = EQ // == @@ -45,23 +29,10 @@ const ( GT // > GTE // >= BETWEEN // >< - COMMA // , - LPAREN // ( - RPAREN // ) - LBRACK // ( - RBRACK // ) ) var tokens = [...]string{ ILLEGAL: "ILLEGAL", - EOF: "EOF", - WS: "WS", - - IDENT: "IDENT", - INTEGER: "INTEGER", - FLOAT: "FLOAT", - - ALL: "ALL", ASSIGN: "=", EQ: "==", @@ -71,20 +42,6 @@ var tokens = [...]string{ GT: ">", GTE: ">=", BETWEEN: "><", - COMMA: ",", - LPAREN: "(", - RPAREN: ")", - LBRACK: "(", - RBRACK: ")", -} - -var keywords map[string]Token - -func init() { - keywords = make(map[string]Token) - for tok := keyword_beg + 1; tok < keyword_end; tok++ { - keywords[strings.ToLower(tokens[tok])] = tok - } } // String returns the string representation of the token. @@ -94,18 +51,3 @@ func (tok Token) String() string { } return "" } - -// Lookup returns the token associated with a given string. -func Lookup(ident string) Token { - if tok, ok := keywords[strings.ToLower(ident)]; ok { - return tok - } - return IDENT -} - -// Pos specifies the line and character position of a token. -// The Char and Line are both zero-based indexes. -type Pos struct { - Line int - Char int -} From 3656bc83a0b7093b26ee04320977050ededdf543 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 12 Jun 2018 13:40:48 -0500 Subject: [PATCH 02/33] support quoted strings properly --- pql/pql.peg | 7 +- pql/pql.peg.go | 548 +++++++++++++++++++++++---------------------- pql/pqlpeg_test.go | 9 +- 3 files changed, 293 insertions(+), 271 deletions(-) diff --git a/pql/pql.peg b/pql/pql.peg index 0d9aeeb66..b909c4450 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -31,10 +31,13 @@ item <- ( 'null' { p.addVal(nil) } / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / '"' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '"' { p.addVal(buffer[begin:end]) } - / '\'' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '\'' { p.addVal(buffer[begin:end]) } + / '"' < doublequotedstring > '"' { p.addVal(buffer[begin:end]) } + / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } ) +doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* +singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* + field <- < [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* > { p.addField(buffer[begin:end]) } close <- ')' sp sp <- ( ' ' / '\t' )* diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 3a2b25315..466f0ed4a 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -25,6 +25,8 @@ const ( rulevalue rulelist ruleitem + ruledquotedstring + rulesquotedstring rulefield ruleclose rulesp @@ -66,6 +68,8 @@ var rul3s = [...]string{ "value", "list", "item", + "dquotedstring", + "squotedstring", "field", "close", "sp", @@ -210,7 +214,7 @@ type PQL struct { Buffer string buffer []rune - rules [38]func() bool + rules [40]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -823,7 +827,7 @@ func (p *PQL) Init() { position, tokenIndex = position56, tokenIndex56 return false }, - /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / ((&('\'') ('\'' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '\'' Action18)) | (&('"') ('"' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '"' Action17)) | (&('-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '_' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') (<((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> Action16))))> */ + /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / ((&('\'') ('\'' '\'' Action18)) | (&('"') ('"' '"' Action17)) | (&('-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '_' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') (<((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> Action16))))> */ func() bool { position60, tokenIndex60 := position, tokenIndex { @@ -1008,93 +1012,95 @@ func (p *PQL) Init() { { position88 := position { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l60 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l60 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l60 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l60 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l60 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 - } - position++ - break - } - } - - l89: - { - position90, tokenIndex90 := position, tokenIndex - { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l90 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l90 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l90 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l90 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l90 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l90 - } - position++ - break - } - } - - goto l89 + position89 := position l90: - position, tokenIndex = position90, tokenIndex90 + { + position91, tokenIndex91 := position, tokenIndex + { + position92, tokenIndex92 := position, tokenIndex + { + position94, tokenIndex94 := position, tokenIndex + { + switch buffer[position] { + case '\n': + if buffer[position] != rune('\n') { + goto l94 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l94 + } + position++ + break + default: + if buffer[position] != rune('\'') { + goto l94 + } + position++ + break + } + } + + goto l93 + l94: + position, tokenIndex = position94, tokenIndex94 + } + if !matchDot() { + goto l93 + } + goto l92 + l93: + position, tokenIndex = position92, tokenIndex92 + if buffer[position] != rune('\\') { + goto l96 + } + position++ + if buffer[position] != rune('n') { + goto l96 + } + position++ + goto l92 + l96: + position, tokenIndex = position92, tokenIndex92 + if buffer[position] != rune('\\') { + goto l97 + } + position++ + if buffer[position] != rune('"') { + goto l97 + } + position++ + goto l92 + l97: + position, tokenIndex = position92, tokenIndex92 + if buffer[position] != rune('\\') { + goto l98 + } + position++ + if buffer[position] != rune('\'') { + goto l98 + } + position++ + goto l92 + l98: + position, tokenIndex = position92, tokenIndex92 + if buffer[position] != rune('\\') { + goto l91 + } + position++ + if buffer[position] != rune('\\') { + goto l91 + } + position++ + } + l92: + goto l90 + l91: + position, tokenIndex = position91, tokenIndex91 + } + add(rulesquotedstring, position89) } add(rulePegText, position88) } @@ -1112,97 +1118,99 @@ func (p *PQL) Init() { } position++ { - position94 := position + position100 := position { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l60 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l60 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l60 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l60 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l60 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 - } - position++ - break - } - } - - l95: - { - position96, tokenIndex96 := position, tokenIndex + position101 := position + l102: { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l96 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l96 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l96 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l96 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l96 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l96 - } - position++ - break - } - } + position103, tokenIndex103 := position, tokenIndex + { + position104, tokenIndex104 := position, tokenIndex + { + position106, tokenIndex106 := position, tokenIndex + { + switch buffer[position] { + case '\n': + if buffer[position] != rune('\n') { + goto l106 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l106 + } + position++ + break + default: + if buffer[position] != rune('"') { + goto l106 + } + position++ + break + } + } - goto l95 - l96: - position, tokenIndex = position96, tokenIndex96 + goto l105 + l106: + position, tokenIndex = position106, tokenIndex106 + } + if !matchDot() { + goto l105 + } + goto l104 + l105: + position, tokenIndex = position104, tokenIndex104 + if buffer[position] != rune('\\') { + goto l108 + } + position++ + if buffer[position] != rune('n') { + goto l108 + } + position++ + goto l104 + l108: + position, tokenIndex = position104, tokenIndex104 + if buffer[position] != rune('\\') { + goto l109 + } + position++ + if buffer[position] != rune('"') { + goto l109 + } + position++ + goto l104 + l109: + position, tokenIndex = position104, tokenIndex104 + if buffer[position] != rune('\\') { + goto l110 + } + position++ + if buffer[position] != rune('\'') { + goto l110 + } + position++ + goto l104 + l110: + position, tokenIndex = position104, tokenIndex104 + if buffer[position] != rune('\\') { + goto l103 + } + position++ + if buffer[position] != rune('\\') { + goto l103 + } + position++ + } + l104: + goto l102 + l103: + position, tokenIndex = position103, tokenIndex103 + } + add(ruledquotedstring, position101) } - add(rulePegText, position94) + add(rulePegText, position100) } if buffer[position] != rune('"') { goto l60 @@ -1214,7 +1222,7 @@ func (p *PQL) Init() { break default: { - position100 := position + position112 := position { switch buffer[position] { case ':': @@ -1256,55 +1264,55 @@ func (p *PQL) Init() { } } - l101: + l113: { - position102, tokenIndex102 := position, tokenIndex + position114, tokenIndex114 := position, tokenIndex { switch buffer[position] { case ':': if buffer[position] != rune(':') { - goto l102 + goto l114 } position++ break case '_': if buffer[position] != rune('_') { - goto l102 + goto l114 } position++ break case '-': if buffer[position] != rune('-') { - goto l102 + goto l114 } position++ break case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l102 + goto l114 } position++ break case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l102 + goto l114 } position++ break default: if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l102 + goto l114 } position++ break } } - goto l101 - l102: - position, tokenIndex = position102, tokenIndex102 + goto l113 + l114: + position, tokenIndex = position114, tokenIndex114 } - add(rulePegText, position100) + add(rulePegText, position112) } { add(ruleAction16, position) @@ -1322,196 +1330,200 @@ func (p *PQL) Init() { position, tokenIndex = position60, tokenIndex60 return false }, - /* 9 field <- <(<(([a-z] / [A-Z]) ((&('_') '_') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))*)> Action19)> */ + /* 9 dquotedstring <- <((!((&('\n') '\n') | (&('\\') '\\') | (&('"') '"')) .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + nil, + /* 10 squotedstring <- <((!((&('\n') '\n') | (&('\\') '\\') | (&('\'') '\'')) .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + nil, + /* 11 field <- <(<(([a-z] / [A-Z]) ((&('_') '_') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))*)> Action19)> */ func() bool { - position106, tokenIndex106 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex { - position107 := position + position121 := position { - position108 := position + position122 := position { - position109, tokenIndex109 := position, tokenIndex + position123, tokenIndex123 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l110 + goto l124 } position++ - goto l109 - l110: - position, tokenIndex = position109, tokenIndex109 + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l106 + goto l120 } position++ } - l109: - l111: + l123: + l125: { - position112, tokenIndex112 := position, tokenIndex + position126, tokenIndex126 := position, tokenIndex { switch buffer[position] { case '_': if buffer[position] != rune('_') { - goto l112 + goto l126 } position++ break case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l112 + goto l126 } position++ break case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l112 + goto l126 } position++ break default: if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l112 + goto l126 } position++ break } } - goto l111 - l112: - position, tokenIndex = position112, tokenIndex112 + goto l125 + l126: + position, tokenIndex = position126, tokenIndex126 } - add(rulePegText, position108) + add(rulePegText, position122) } { add(ruleAction19, position) } - add(rulefield, position107) + add(rulefield, position121) } return true - l106: - position, tokenIndex = position106, tokenIndex106 + l120: + position, tokenIndex = position120, tokenIndex120 return false }, - /* 10 close <- <(')' sp)> */ + /* 12 close <- <(')' sp)> */ nil, - /* 11 sp <- <(' ' / '\t')*> */ + /* 13 sp <- <(' ' / '\t')*> */ func() bool { { - position117 := position - l118: + position131 := position + l132: { - position119, tokenIndex119 := position, tokenIndex + position133, tokenIndex133 := position, tokenIndex { - position120, tokenIndex120 := position, tokenIndex + position134, tokenIndex134 := position, tokenIndex if buffer[position] != rune(' ') { - goto l121 + goto l135 } position++ - goto l120 - l121: - position, tokenIndex = position120, tokenIndex120 + goto l134 + l135: + position, tokenIndex = position134, tokenIndex134 if buffer[position] != rune('\t') { - goto l119 + goto l133 } position++ } - l120: - goto l118 - l119: - position, tokenIndex = position119, tokenIndex119 + l134: + goto l132 + l133: + position, tokenIndex = position133, tokenIndex133 } - add(rulesp, position117) + add(rulesp, position131) } return true }, - /* 12 comma <- <(sp ',' sp)> */ + /* 14 comma <- <(sp ',' sp)> */ func() bool { - position122, tokenIndex122 := position, tokenIndex + position136, tokenIndex136 := position, tokenIndex { - position123 := position + position137 := position if !_rules[rulesp]() { - goto l122 + goto l136 } if buffer[position] != rune(',') { - goto l122 + goto l136 } position++ if !_rules[rulesp]() { - goto l122 + goto l136 } - add(rulecomma, position123) + add(rulecomma, position137) } return true - l122: - position, tokenIndex = position122, tokenIndex122 + l136: + position, tokenIndex = position136, tokenIndex136 return false }, - /* 13 lbrack <- <('[' sp)> */ + /* 15 lbrack <- <('[' sp)> */ nil, - /* 14 rbrack <- <(sp ']' sp)> */ + /* 16 rbrack <- <(sp ']' sp)> */ nil, - /* 15 newline <- <(sp '\n' sp)> */ + /* 17 newline <- <(sp '\n' sp)> */ func() bool { - position126, tokenIndex126 := position, tokenIndex + position140, tokenIndex140 := position, tokenIndex { - position127 := position + position141 := position if !_rules[rulesp]() { - goto l126 + goto l140 } if buffer[position] != rune('\n') { - goto l126 + goto l140 } position++ if !_rules[rulesp]() { - goto l126 + goto l140 } - add(rulenewline, position127) + add(rulenewline, position141) } return true - l126: - position, tokenIndex = position126, tokenIndex126 + l140: + position, tokenIndex = position140, tokenIndex140 return false }, nil, - /* 18 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 20 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 19 Action1 <- <{ p.endCall() }> */ + /* 21 Action1 <- <{ p.endCall() }> */ nil, - /* 20 Action2 <- <{ p.addBTWN() }> */ + /* 22 Action2 <- <{ p.addBTWN() }> */ nil, - /* 21 Action3 <- <{ p.addLTE() }> */ + /* 23 Action3 <- <{ p.addLTE() }> */ nil, - /* 22 Action4 <- <{ p.addGTE() }> */ + /* 24 Action4 <- <{ p.addGTE() }> */ nil, - /* 23 Action5 <- <{ p.addEQ() }> */ + /* 25 Action5 <- <{ p.addEQ() }> */ nil, - /* 24 Action6 <- <{ p.addNEQ() }> */ + /* 26 Action6 <- <{ p.addNEQ() }> */ nil, - /* 25 Action7 <- <{ p.addLT() }> */ + /* 27 Action7 <- <{ p.addLT() }> */ nil, - /* 26 Action8 <- <{ p.addGT() }> */ + /* 28 Action8 <- <{ p.addGT() }> */ nil, - /* 27 Action9 <- <{ p.startList() }> */ + /* 29 Action9 <- <{ p.startList() }> */ nil, - /* 28 Action10 <- <{ p.endList() }> */ + /* 30 Action10 <- <{ p.endList() }> */ nil, - /* 29 Action11 <- <{ p.addVal(nil) }> */ + /* 31 Action11 <- <{ p.addVal(nil) }> */ nil, - /* 30 Action12 <- <{ p.addVal(true) }> */ + /* 32 Action12 <- <{ p.addVal(true) }> */ nil, - /* 31 Action13 <- <{ p.addVal(false) }> */ + /* 33 Action13 <- <{ p.addVal(false) }> */ nil, - /* 32 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 34 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 33 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 35 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 34 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 36 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 35 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 37 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 36 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 38 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 37 Action19 <- <{ p.addField(buffer[begin:end]) }> */ + /* 39 Action19 <- <{ p.addField(buffer[begin:end]) }> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 8b4884fb1..a38d451d0 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -6,11 +6,18 @@ import ( func TestPEG(t *testing.T) { p := PQL{Buffer: ` -SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="zoo9")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]} +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]} p.Init() err := p.Parse() if err != nil { t.Fatalf("parse error: %v", err) } p.Execute() + + p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`} + p.Init() + err = p.Parse() + if err == nil { + t.Fatalf("should have been an error because of the interior unescaped double quote") + } } From c66cb59f1dcd157d76d7da2e3a31ed4235adc7fa Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 13 Jun 2018 07:03:30 -0500 Subject: [PATCH 03/33] remove -switch option from peg generator --- Makefile | 2 +- pql/pql.peg.go | 1109 ++++++++++++++++++++++++------------------------ 2 files changed, 552 insertions(+), 559 deletions(-) diff --git a/Makefile b/Makefile index c7aefa3e0..363bcd6b6 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ generate-stringer: go generate github.com/pilosa/pilosa generate-pql: require-peg - cd pql && peg -inline -switch pql.peg && cd .. + cd pql && peg -inline pql.peg && cd .. # `go generate` all needed packages generate: generate-protoc generate-stringer generate-pql diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 466f0ed4a..02ce64918 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,6 +1,6 @@ package pql -//go:generate peg -inline -switch pql.peg +//go:generate peg -inline pql.peg import ( "fmt" @@ -25,8 +25,8 @@ const ( rulevalue rulelist ruleitem - ruledquotedstring - rulesquotedstring + ruledoublequotedstring + rulesinglequotedstring rulefield ruleclose rulesp @@ -68,8 +68,8 @@ var rul3s = [...]string{ "value", "list", "item", - "dquotedstring", - "squotedstring", + "doublequotedstring", + "singlequotedstring", "field", "close", "sp", @@ -643,55 +643,51 @@ func (p *PQL) Init() { goto l31 l36: position, tokenIndex = position31, tokenIndex31 - { - switch buffer[position] { - case '>': - if buffer[position] != rune('>') { - goto l25 - } - position++ - { - add(ruleAction8, position) - } - break - case '<': - if buffer[position] != rune('<') { - goto l25 - } - position++ - { - add(ruleAction7, position) - } - break - case '!': - if buffer[position] != rune('!') { - goto l25 - } - position++ - if buffer[position] != rune('=') { - goto l25 - } - position++ - { - add(ruleAction6, position) - } - break - default: - if buffer[position] != rune('=') { - goto l25 - } - position++ - if buffer[position] != rune('=') { - goto l25 - } - position++ - { - add(ruleAction5, position) - } - break - } + if buffer[position] != rune('=') { + goto l38 + } + position++ + if buffer[position] != rune('=') { + goto l38 + } + position++ + { + add(ruleAction5, position) + } + goto l31 + l38: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('!') { + goto l40 + } + position++ + if buffer[position] != rune('=') { + goto l40 + } + position++ + { + add(ruleAction6, position) + } + goto l31 + l40: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('<') { + goto l42 + } + position++ + { + add(ruleAction7, position) + } + goto l31 + l42: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('>') { + goto l25 + } + position++ + { + add(ruleAction8, position) } - } l31: add(ruleCOND, position30) @@ -707,18 +703,18 @@ func (p *PQL) Init() { add(rulearg, position26) } { - position43, tokenIndex43 := position, tokenIndex + position45, tokenIndex45 := position, tokenIndex if !_rules[rulecomma]() { - goto l43 + goto l45 } if !_rules[ruleargs]() { - goto l43 + goto l45 } - goto l44 - l43: - position, tokenIndex = position43, tokenIndex43 + goto l46 + l45: + position, tokenIndex = position45, tokenIndex45 } - l44: + l46: if !_rules[rulesp]() { goto l25 } @@ -739,669 +735,666 @@ func (p *PQL) Init() { }, /* 3 arg <- <(Call / (field sp '=' sp value) / (field sp COND sp value))> */ nil, - /* 4 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ((&('>') ('>' Action8)) | (&('<') ('<' Action7)) | (&('!') ('!' '=' Action6)) | (&('=') ('=' '=' Action5))))> */ + /* 4 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ('=' '=' Action5) / ('!' '=' Action6) / ('<' Action7) / ('>' Action8))> */ nil, /* 5 open <- <('(' sp)> */ nil, /* 6 value <- <(item / (lbrack Action9 list rbrack Action10))> */ func() bool { - position48, tokenIndex48 := position, tokenIndex + position50, tokenIndex50 := position, tokenIndex { - position49 := position + position51 := position { - position50, tokenIndex50 := position, tokenIndex + position52, tokenIndex52 := position, tokenIndex if !_rules[ruleitem]() { - goto l51 + goto l53 } - goto l50 - l51: - position, tokenIndex = position50, tokenIndex50 + goto l52 + l53: + position, tokenIndex = position52, tokenIndex52 { - position52 := position + position54 := position if buffer[position] != rune('[') { - goto l48 + goto l50 } position++ if !_rules[rulesp]() { - goto l48 + goto l50 } - add(rulelbrack, position52) + add(rulelbrack, position54) } { add(ruleAction9, position) } if !_rules[rulelist]() { - goto l48 + goto l50 } { - position54 := position + position56 := position if !_rules[rulesp]() { - goto l48 + goto l50 } if buffer[position] != rune(']') { - goto l48 + goto l50 } position++ if !_rules[rulesp]() { - goto l48 + goto l50 } - add(rulerbrack, position54) + add(rulerbrack, position56) } { add(ruleAction10, position) } } - l50: - add(rulevalue, position49) + l52: + add(rulevalue, position51) } return true - l48: - position, tokenIndex = position48, tokenIndex48 + l50: + position, tokenIndex = position50, tokenIndex50 return false }, /* 7 list <- <(item (comma list)?)> */ func() bool { - position56, tokenIndex56 := position, tokenIndex + position58, tokenIndex58 := position, tokenIndex { - position57 := position + position59 := position if !_rules[ruleitem]() { - goto l56 + goto l58 } { - position58, tokenIndex58 := position, tokenIndex + position60, tokenIndex60 := position, tokenIndex if !_rules[rulecomma]() { - goto l58 + goto l60 } if !_rules[rulelist]() { - goto l58 + goto l60 } - goto l59 - l58: - position, tokenIndex = position58, tokenIndex58 + goto l61 + l60: + position, tokenIndex = position60, tokenIndex60 } - l59: - add(rulelist, position57) + l61: + add(rulelist, position59) } return true - l56: - position, tokenIndex = position56, tokenIndex56 + l58: + position, tokenIndex = position58, tokenIndex58 return false }, - /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / ((&('\'') ('\'' '\'' Action18)) | (&('"') ('"' '"' Action17)) | (&('-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '_' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') (<((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> Action16))))> */ + /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action16) / ('"' '"' Action17) / ('\'' '\'' Action18))> */ func() bool { - position60, tokenIndex60 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex { - position61 := position + position63 := position { - position62, tokenIndex62 := position, tokenIndex + position64, tokenIndex64 := position, tokenIndex if buffer[position] != rune('n') { - goto l63 + goto l65 } position++ if buffer[position] != rune('u') { - goto l63 + goto l65 } position++ if buffer[position] != rune('l') { - goto l63 + goto l65 } position++ if buffer[position] != rune('l') { - goto l63 + goto l65 } position++ { add(ruleAction11, position) } - goto l62 - l63: - position, tokenIndex = position62, tokenIndex62 + goto l64 + l65: + position, tokenIndex = position64, tokenIndex64 if buffer[position] != rune('t') { - goto l65 + goto l67 } position++ if buffer[position] != rune('r') { - goto l65 + goto l67 } position++ if buffer[position] != rune('u') { - goto l65 + goto l67 } position++ if buffer[position] != rune('e') { - goto l65 + goto l67 } position++ { add(ruleAction12, position) } - goto l62 - l65: - position, tokenIndex = position62, tokenIndex62 + goto l64 + l67: + position, tokenIndex = position64, tokenIndex64 if buffer[position] != rune('f') { - goto l67 + goto l69 } position++ if buffer[position] != rune('a') { - goto l67 + goto l69 } position++ if buffer[position] != rune('l') { - goto l67 + goto l69 } position++ if buffer[position] != rune('s') { - goto l67 + goto l69 } position++ if buffer[position] != rune('e') { - goto l67 + goto l69 } position++ { add(ruleAction13, position) } - goto l62 - l67: - position, tokenIndex = position62, tokenIndex62 + goto l64 + l69: + position, tokenIndex = position64, tokenIndex64 { - position70 := position + position72 := position { - position71, tokenIndex71 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if buffer[position] != rune('-') { - goto l71 + goto l73 } position++ - goto l72 - l71: - position, tokenIndex = position71, tokenIndex71 + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 } - l72: + l74: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l69 + goto l71 } position++ - l73: + l75: { - position74, tokenIndex74 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l74 + goto l76 } position++ - goto l73 - l74: - position, tokenIndex = position74, tokenIndex74 + goto l75 + l76: + position, tokenIndex = position76, tokenIndex76 } { - position75, tokenIndex75 := position, tokenIndex + position77, tokenIndex77 := position, tokenIndex if buffer[position] != rune('.') { - goto l75 + goto l77 } position++ - l77: + l79: { - position78, tokenIndex78 := position, tokenIndex + position80, tokenIndex80 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l78 + goto l80 } position++ - goto l77 - l78: - position, tokenIndex = position78, tokenIndex78 + goto l79 + l80: + position, tokenIndex = position80, tokenIndex80 } - goto l76 - l75: - position, tokenIndex = position75, tokenIndex75 + goto l78 + l77: + position, tokenIndex = position77, tokenIndex77 } - l76: - add(rulePegText, position70) + l78: + add(rulePegText, position72) } { add(ruleAction14, position) } - goto l62 - l69: - position, tokenIndex = position62, tokenIndex62 + goto l64 + l71: + position, tokenIndex = position64, tokenIndex64 { - position81 := position + position83 := position { - position82, tokenIndex82 := position, tokenIndex + position84, tokenIndex84 := position, tokenIndex if buffer[position] != rune('-') { - goto l82 + goto l84 } position++ - goto l83 - l82: - position, tokenIndex = position82, tokenIndex82 + goto l85 + l84: + position, tokenIndex = position84, tokenIndex84 } - l83: + l85: if buffer[position] != rune('.') { - goto l80 + goto l82 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l80 + goto l82 } position++ - l84: + l86: { - position85, tokenIndex85 := position, tokenIndex + position87, tokenIndex87 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l85 + goto l87 } position++ - goto l84 - l85: - position, tokenIndex = position85, tokenIndex85 + goto l86 + l87: + position, tokenIndex = position87, tokenIndex87 } - add(rulePegText, position81) + add(rulePegText, position83) } { add(ruleAction15, position) } - goto l62 - l80: - position, tokenIndex = position62, tokenIndex62 + goto l64 + l82: + position, tokenIndex = position64, tokenIndex64 { - switch buffer[position] { - case '\'': - if buffer[position] != rune('\'') { - goto l60 + position90 := position + { + position93, tokenIndex93 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l94 } position++ - { - position88 := position - { - position89 := position - l90: - { - position91, tokenIndex91 := position, tokenIndex - { - position92, tokenIndex92 := position, tokenIndex - { - position94, tokenIndex94 := position, tokenIndex - { - switch buffer[position] { - case '\n': - if buffer[position] != rune('\n') { - goto l94 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l94 - } - position++ - break - default: - if buffer[position] != rune('\'') { - goto l94 - } - position++ - break - } - } - - goto l93 - l94: - position, tokenIndex = position94, tokenIndex94 - } - if !matchDot() { - goto l93 - } - goto l92 - l93: - position, tokenIndex = position92, tokenIndex92 - if buffer[position] != rune('\\') { - goto l96 - } - position++ - if buffer[position] != rune('n') { - goto l96 - } - position++ - goto l92 - l96: - position, tokenIndex = position92, tokenIndex92 - if buffer[position] != rune('\\') { - goto l97 - } - position++ - if buffer[position] != rune('"') { - goto l97 - } - position++ - goto l92 - l97: - position, tokenIndex = position92, tokenIndex92 - if buffer[position] != rune('\\') { - goto l98 - } - position++ - if buffer[position] != rune('\'') { - goto l98 - } - position++ - goto l92 - l98: - position, tokenIndex = position92, tokenIndex92 - if buffer[position] != rune('\\') { - goto l91 - } - position++ - if buffer[position] != rune('\\') { - goto l91 - } - position++ - } - l92: - goto l90 - l91: - position, tokenIndex = position91, tokenIndex91 - } - add(rulesquotedstring, position89) - } - add(rulePegText, position88) - } - if buffer[position] != rune('\'') { - goto l60 + goto l93 + l94: + position, tokenIndex = position93, tokenIndex93 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l95 } position++ - { - add(ruleAction18, position) - } - break - case '"': - if buffer[position] != rune('"') { - goto l60 + goto l93 + l95: + position, tokenIndex = position93, tokenIndex93 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l96 } position++ - { - position100 := position - { - position101 := position - l102: - { - position103, tokenIndex103 := position, tokenIndex - { - position104, tokenIndex104 := position, tokenIndex - { - position106, tokenIndex106 := position, tokenIndex - { - switch buffer[position] { - case '\n': - if buffer[position] != rune('\n') { - goto l106 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l106 - } - position++ - break - default: - if buffer[position] != rune('"') { - goto l106 - } - position++ - break - } - } - - goto l105 - l106: - position, tokenIndex = position106, tokenIndex106 - } - if !matchDot() { - goto l105 - } - goto l104 - l105: - position, tokenIndex = position104, tokenIndex104 - if buffer[position] != rune('\\') { - goto l108 - } - position++ - if buffer[position] != rune('n') { - goto l108 - } - position++ - goto l104 - l108: - position, tokenIndex = position104, tokenIndex104 - if buffer[position] != rune('\\') { - goto l109 - } - position++ - if buffer[position] != rune('"') { - goto l109 - } - position++ - goto l104 - l109: - position, tokenIndex = position104, tokenIndex104 - if buffer[position] != rune('\\') { - goto l110 - } - position++ - if buffer[position] != rune('\'') { - goto l110 - } - position++ - goto l104 - l110: - position, tokenIndex = position104, tokenIndex104 - if buffer[position] != rune('\\') { - goto l103 - } - position++ - if buffer[position] != rune('\\') { - goto l103 - } - position++ - } - l104: - goto l102 - l103: - position, tokenIndex = position103, tokenIndex103 - } - add(ruledquotedstring, position101) - } - add(rulePegText, position100) - } - if buffer[position] != rune('"') { - goto l60 + goto l93 + l96: + position, tokenIndex = position93, tokenIndex93 + if buffer[position] != rune('-') { + goto l97 } position++ - { - add(ruleAction17, position) + goto l93 + l97: + position, tokenIndex = position93, tokenIndex93 + if buffer[position] != rune('_') { + goto l98 } - break - default: - { - position112 := position - { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l60 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l60 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l60 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l60 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l60 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 - } - position++ - break - } - } - - l113: - { - position114, tokenIndex114 := position, tokenIndex - { - switch buffer[position] { - case ':': - if buffer[position] != rune(':') { - goto l114 - } - position++ - break - case '_': - if buffer[position] != rune('_') { - goto l114 - } - position++ - break - case '-': - if buffer[position] != rune('-') { - goto l114 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l114 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l114 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l114 - } - position++ - break - } - } - - goto l113 - l114: - position, tokenIndex = position114, tokenIndex114 - } - add(rulePegText, position112) + position++ + goto l93 + l98: + position, tokenIndex = position93, tokenIndex93 + if buffer[position] != rune(':') { + goto l89 } - { - add(ruleAction16, position) - } - break + position++ } + l93: + l91: + { + position92, tokenIndex92 := position, tokenIndex + { + position99, tokenIndex99 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l100 + } + position++ + goto l99 + l100: + position, tokenIndex = position99, tokenIndex99 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l101 + } + position++ + goto l99 + l101: + position, tokenIndex = position99, tokenIndex99 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l102 + } + position++ + goto l99 + l102: + position, tokenIndex = position99, tokenIndex99 + if buffer[position] != rune('-') { + goto l103 + } + position++ + goto l99 + l103: + position, tokenIndex = position99, tokenIndex99 + if buffer[position] != rune('_') { + goto l104 + } + position++ + goto l99 + l104: + position, tokenIndex = position99, tokenIndex99 + if buffer[position] != rune(':') { + goto l92 + } + position++ + } + l99: + goto l91 + l92: + position, tokenIndex = position92, tokenIndex92 + } + add(rulePegText, position90) + } + { + add(ruleAction16, position) + } + goto l64 + l89: + position, tokenIndex = position64, tokenIndex64 + if buffer[position] != rune('"') { + goto l106 + } + position++ + { + position107 := position + { + position108 := position + l109: + { + position110, tokenIndex110 := position, tokenIndex + { + position111, tokenIndex111 := position, tokenIndex + { + position113, tokenIndex113 := position, tokenIndex + { + position114, tokenIndex114 := position, tokenIndex + if buffer[position] != rune('"') { + goto l115 + } + position++ + goto l114 + l115: + position, tokenIndex = position114, tokenIndex114 + if buffer[position] != rune('\\') { + goto l116 + } + position++ + goto l114 + l116: + position, tokenIndex = position114, tokenIndex114 + if buffer[position] != rune('\n') { + goto l113 + } + position++ + } + l114: + goto l112 + l113: + position, tokenIndex = position113, tokenIndex113 + } + if !matchDot() { + goto l112 + } + goto l111 + l112: + position, tokenIndex = position111, tokenIndex111 + if buffer[position] != rune('\\') { + goto l117 + } + position++ + if buffer[position] != rune('n') { + goto l117 + } + position++ + goto l111 + l117: + position, tokenIndex = position111, tokenIndex111 + if buffer[position] != rune('\\') { + goto l118 + } + position++ + if buffer[position] != rune('"') { + goto l118 + } + position++ + goto l111 + l118: + position, tokenIndex = position111, tokenIndex111 + if buffer[position] != rune('\\') { + goto l119 + } + position++ + if buffer[position] != rune('\'') { + goto l119 + } + position++ + goto l111 + l119: + position, tokenIndex = position111, tokenIndex111 + if buffer[position] != rune('\\') { + goto l110 + } + position++ + if buffer[position] != rune('\\') { + goto l110 + } + position++ + } + l111: + goto l109 + l110: + position, tokenIndex = position110, tokenIndex110 + } + add(ruledoublequotedstring, position108) + } + add(rulePegText, position107) + } + if buffer[position] != rune('"') { + goto l106 + } + position++ + { + add(ruleAction17, position) + } + goto l64 + l106: + position, tokenIndex = position64, tokenIndex64 + if buffer[position] != rune('\'') { + goto l62 + } + position++ + { + position121 := position + { + position122 := position + l123: + { + position124, tokenIndex124 := position, tokenIndex + { + position125, tokenIndex125 := position, tokenIndex + { + position127, tokenIndex127 := position, tokenIndex + { + position128, tokenIndex128 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l129 + } + position++ + goto l128 + l129: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('\\') { + goto l130 + } + position++ + goto l128 + l130: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('\n') { + goto l127 + } + position++ + } + l128: + goto l126 + l127: + position, tokenIndex = position127, tokenIndex127 + } + if !matchDot() { + goto l126 + } + goto l125 + l126: + position, tokenIndex = position125, tokenIndex125 + if buffer[position] != rune('\\') { + goto l131 + } + position++ + if buffer[position] != rune('n') { + goto l131 + } + position++ + goto l125 + l131: + position, tokenIndex = position125, tokenIndex125 + if buffer[position] != rune('\\') { + goto l132 + } + position++ + if buffer[position] != rune('"') { + goto l132 + } + position++ + goto l125 + l132: + position, tokenIndex = position125, tokenIndex125 + if buffer[position] != rune('\\') { + goto l133 + } + position++ + if buffer[position] != rune('\'') { + goto l133 + } + position++ + goto l125 + l133: + position, tokenIndex = position125, tokenIndex125 + if buffer[position] != rune('\\') { + goto l124 + } + position++ + if buffer[position] != rune('\\') { + goto l124 + } + position++ + } + l125: + goto l123 + l124: + position, tokenIndex = position124, tokenIndex124 + } + add(rulesinglequotedstring, position122) + } + add(rulePegText, position121) + } + if buffer[position] != rune('\'') { + goto l62 + } + position++ + { + add(ruleAction18, position) } - } - l62: - add(ruleitem, position61) + l64: + add(ruleitem, position63) } return true - l60: - position, tokenIndex = position60, tokenIndex60 + l62: + position, tokenIndex = position62, tokenIndex62 return false }, - /* 9 dquotedstring <- <((!((&('\n') '\n') | (&('\\') '\\') | (&('"') '"')) .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 9 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 10 squotedstring <- <((!((&('\n') '\n') | (&('\\') '\\') | (&('\'') '\'')) .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 10 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 11 field <- <(<(([a-z] / [A-Z]) ((&('_') '_') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))*)> Action19)> */ + /* 11 field <- <(<(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> Action19)> */ func() bool { - position120, tokenIndex120 := position, tokenIndex + position137, tokenIndex137 := position, tokenIndex { - position121 := position + position138 := position { - position122 := position + position139 := position { - position123, tokenIndex123 := position, tokenIndex + position140, tokenIndex140 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l124 + goto l141 } position++ - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 + goto l140 + l141: + position, tokenIndex = position140, tokenIndex140 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l120 + goto l137 } position++ } - l123: - l125: + l140: + l142: { - position126, tokenIndex126 := position, tokenIndex + position143, tokenIndex143 := position, tokenIndex { - switch buffer[position] { - case '_': - if buffer[position] != rune('_') { - goto l126 - } - position++ - break - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l126 - } - position++ - break - case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l126 - } - position++ - break - default: - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l126 - } - position++ - break + position144, tokenIndex144 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l145 } + position++ + goto l144 + l145: + position, tokenIndex = position144, tokenIndex144 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l146 + } + position++ + goto l144 + l146: + position, tokenIndex = position144, tokenIndex144 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l147 + } + position++ + goto l144 + l147: + position, tokenIndex = position144, tokenIndex144 + if buffer[position] != rune('_') { + goto l143 + } + position++ } - - goto l125 - l126: - position, tokenIndex = position126, tokenIndex126 + l144: + goto l142 + l143: + position, tokenIndex = position143, tokenIndex143 } - add(rulePegText, position122) + add(rulePegText, position139) } { add(ruleAction19, position) } - add(rulefield, position121) + add(rulefield, position138) } return true - l120: - position, tokenIndex = position120, tokenIndex120 + l137: + position, tokenIndex = position137, tokenIndex137 return false }, /* 12 close <- <(')' sp)> */ @@ -1409,53 +1402,53 @@ func (p *PQL) Init() { /* 13 sp <- <(' ' / '\t')*> */ func() bool { { - position131 := position - l132: + position151 := position + l152: { - position133, tokenIndex133 := position, tokenIndex + position153, tokenIndex153 := position, tokenIndex { - position134, tokenIndex134 := position, tokenIndex + position154, tokenIndex154 := position, tokenIndex if buffer[position] != rune(' ') { - goto l135 + goto l155 } position++ - goto l134 - l135: - position, tokenIndex = position134, tokenIndex134 + goto l154 + l155: + position, tokenIndex = position154, tokenIndex154 if buffer[position] != rune('\t') { - goto l133 + goto l153 } position++ } - l134: - goto l132 - l133: - position, tokenIndex = position133, tokenIndex133 + l154: + goto l152 + l153: + position, tokenIndex = position153, tokenIndex153 } - add(rulesp, position131) + add(rulesp, position151) } return true }, /* 14 comma <- <(sp ',' sp)> */ func() bool { - position136, tokenIndex136 := position, tokenIndex + position156, tokenIndex156 := position, tokenIndex { - position137 := position + position157 := position if !_rules[rulesp]() { - goto l136 + goto l156 } if buffer[position] != rune(',') { - goto l136 + goto l156 } position++ if !_rules[rulesp]() { - goto l136 + goto l156 } - add(rulecomma, position137) + add(rulecomma, position157) } return true - l136: - position, tokenIndex = position136, tokenIndex136 + l156: + position, tokenIndex = position156, tokenIndex156 return false }, /* 15 lbrack <- <('[' sp)> */ @@ -1464,24 +1457,24 @@ func (p *PQL) Init() { nil, /* 17 newline <- <(sp '\n' sp)> */ func() bool { - position140, tokenIndex140 := position, tokenIndex + position160, tokenIndex160 := position, tokenIndex { - position141 := position + position161 := position if !_rules[rulesp]() { - goto l140 + goto l160 } if buffer[position] != rune('\n') { - goto l140 + goto l160 } position++ if !_rules[rulesp]() { - goto l140 + goto l160 } - add(rulenewline, position141) + add(rulenewline, position161) } return true - l140: - position, tokenIndex = position140, tokenIndex140 + l160: + position, tokenIndex = position160, tokenIndex160 return false }, nil, From 2ddc2ceeebd5a5dfe8edeb1efb93426d636d885c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 14 Jun 2018 11:58:19 -0500 Subject: [PATCH 04/33] add old parser implementation to pql/internal/oldpql for comparison fuzz testing --- pql/internal/oldpql/ast.go | 272 +++++++++++++++++++++++ pql/internal/oldpql/ast_test.go | 69 ++++++ pql/internal/oldpql/doc.go | 18 ++ pql/internal/oldpql/parser.go | 329 ++++++++++++++++++++++++++++ pql/internal/oldpql/parser_test.go | 194 ++++++++++++++++ pql/internal/oldpql/scanner.go | 303 +++++++++++++++++++++++++ pql/internal/oldpql/scanner_test.go | 74 +++++++ pql/internal/oldpql/token.go | 111 ++++++++++ 8 files changed, 1370 insertions(+) create mode 100644 pql/internal/oldpql/ast.go create mode 100644 pql/internal/oldpql/ast_test.go create mode 100644 pql/internal/oldpql/doc.go create mode 100644 pql/internal/oldpql/parser.go create mode 100644 pql/internal/oldpql/parser_test.go create mode 100644 pql/internal/oldpql/scanner.go create mode 100644 pql/internal/oldpql/scanner_test.go create mode 100644 pql/internal/oldpql/token.go diff --git a/pql/internal/oldpql/ast.go b/pql/internal/oldpql/ast.go new file mode 100644 index 000000000..bee778905 --- /dev/null +++ b/pql/internal/oldpql/ast.go @@ -0,0 +1,272 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "bytes" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// Query represents a PQL query. +type Query struct { + Calls []*Call +} + +// WriteCallN returns the number of mutating calls. +func (q *Query) WriteCallN() int { + var n int + for _, call := range q.Calls { + switch call.Name { + case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs": + n++ + } + } + return n +} + +// String returns a string representation of the query. +func (q *Query) String() string { + a := make([]string, len(q.Calls)) + for i, call := range q.Calls { + a[i] = call.String() + } + return strings.Join(a, "\n") +} + +// Call represents a function call in the AST. +type Call struct { + Name string + Args map[string]interface{} + Children []*Call +} + +// UintArg is for reading the value at key from call.Args as a uint64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a uint64 or an int64 and +// then cast to a uint64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) UintArg(key string) (uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return uint64(tval), true, nil + case uint64: + return tval, true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval) + } +} + +// UintSliceArg reads the value at key from call.Args as a slice of uint64. If +// the key is not in Call.Args, the value of the returned bool will be false, +// and the error will be nil. If the value is a slice of int64 it will convert +// it to []uint64. Otherwise, if it is not a []uint64 it will return an error. +func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return nil, false, nil + } + + switch tval := val.(type) { + case []uint64: + return tval, true, nil + case []int64: + ret := make([]uint64, len(tval)) + for i, v := range tval { + ret[i] = uint64(v) + } + return ret, true, nil + default: + return nil, true, fmt.Errorf("unexpected type %T in UintSliceArg, val %v", tval, tval) + } +} + +// Keys returns a list of argument keys in sorted order. +func (c *Call) Keys() []string { + a := make([]string, 0, len(c.Args)) + for k := range c.Args { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// Clone returns a copy of c. +func (c *Call) Clone() *Call { + if c == nil { + return nil + } + + other := &Call{ + Name: c.Name, + Args: CopyArgs(c.Args), + } + if c.Children != nil { + other.Children = make([]*Call, len(c.Children)) + for i := range c.Children { + other.Children[i] = c.Children[i].Clone() + } + } + return other +} + +// String returns the string representation of the call. +func (c *Call) String() string { + var buf bytes.Buffer + + // Write name. + if c.Name != "" { + buf.WriteString(c.Name) + } else { + buf.WriteString("!UNNAMED") + } + + // Write opening. + buf.WriteByte('(') + + // Write child list. + for i, child := range c.Children { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(child.String()) + } + + // Separate children and args, if necessary. + if len(c.Children) > 0 && len(c.Args) > 0 { + buf.WriteString(", ") + } + + // Write arguments in key order. + for i, key := range c.Keys() { + if i > 0 { + buf.WriteString(", ") + } + // If the Arg value is a Condition, then don't include + // the equal sign in the string representation. + switch v := c.Args[key].(type) { + case *Condition: + fmt.Fprintf(&buf, "%v %s", key, v.String()) + default: + fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v)) + } + } + + // Write closing. + buf.WriteByte(')') + + return buf.String() +} + +// HasConditionArg returns true if any arg is a conditional. +func (c *Call) HasConditionArg() bool { + for _, v := range c.Args { + if _, ok := v.(*Condition); ok { + return true + } + } + return false +} + +// Condition represents an operation & value. +// When used in an argument map it represents a binary expression. +type Condition struct { + Op Token + Value interface{} +} + +// String returns the string representation of the condition. +func (cond *Condition) String() string { + return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value)) +} + +// IntSliceValue reads cond.Value as a slice of uint64. +// If the value is a slice of uint64 it will convert +// it to []int64. Otherwise, if it is not a []int64 it will return an error. +func (cond *Condition) IntSliceValue() ([]int64, error) { + val := cond.Value + + switch tval := val.(type) { + case []interface{}: + ret := make([]int64, len(tval)) + for i, v := range tval { + switch tv := v.(type) { + case int64: + ret[i] = tv + case uint64: + ret[i] = int64(tv) + default: + return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv) + } + } + return ret, nil + default: + return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval) + } +} + +func FormatValue(v interface{}) string { + switch v := v.(type) { + case string: + return fmt.Sprintf("%q", v) + case []interface{}: + return fmt.Sprintf("%s", joinInterfaceSlice(v)) + case []uint64: + return fmt.Sprintf("%s", joinUint64Slice(v)) + case time.Time: + return fmt.Sprintf("\"%s\"", v.Format(TimeFormat)) + case *Condition: + return v.String() + default: + return fmt.Sprintf("%v", v) + } +} + +// CopyArgs returns a copy of m. +func CopyArgs(m map[string]interface{}) map[string]interface{} { + other := make(map[string]interface{}, len(m)) + for k, v := range m { + other[k] = v + } + return other +} + +func joinInterfaceSlice(a []interface{}) string { + other := make([]string, len(a)) + for i := range a { + switch v := a[i].(type) { + case string: + other[i] = fmt.Sprintf("%q", v) + default: + other[i] = fmt.Sprintf("%v", v) + } + } + return "[" + strings.Join(other, ",") + "]" +} + +func joinUint64Slice(a []uint64) string { + other := make([]string, len(a)) + for i := range a { + other[i] = strconv.FormatUint(a[i], 10) + } + return "[" + strings.Join(other, ",") + "]" +} diff --git a/pql/internal/oldpql/ast_test.go b/pql/internal/oldpql/ast_test.go new file mode 100644 index 000000000..1b7c9eba0 --- /dev/null +++ b/pql/internal/oldpql/ast_test.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "reflect" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" +) + +// Ensure call can be converted into a string. +func TestCall_String(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + c := &pql.Call{Name: "Bitmap"} + if s := c.String(); s != `Bitmap()` { + t.Fatalf("unexpected string: %s", s) + } + }) + t.Run("With Args", func(t *testing.T) { + c := &pql.Call{ + Name: "Range", + Args: map[string]interface{}{ + "other": "f", + "field0": &pql.Condition{Op: pql.GTE, Value: 10}, + }, + } + if s := c.String(); s != `Range(field0 >= 10, other="f")` { + t.Fatalf("unexpected string: %s", s) + } + }) +} + +// Ensure condition can handle values for BETWEEN operator. +func TestCondition_Value(t *testing.T) { + t.Run("Between Values", func(t *testing.T) { + for _, tt := range []struct { + val []interface{} + exp []int64 + }{ + {[]interface{}{int64(4), int64(8)}, []int64{4, 8}}, + {[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}}, + {[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}}, + } { + c := &pql.Condition{ + Op: pql.BETWEEN, + Value: tt.val, + } + v, err := c.IntSliceValue() + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v, tt.exp) { + t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v) + } + } + }) +} diff --git a/pql/internal/oldpql/doc.go b/pql/internal/oldpql/doc.go new file mode 100644 index 000000000..3e5bd4876 --- /dev/null +++ b/pql/internal/oldpql/doc.go @@ -0,0 +1,18 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +package oldpql defines the Pilosa Query Language. +*/ +package oldpql diff --git a/pql/internal/oldpql/parser.go b/pql/internal/oldpql/parser.go new file mode 100644 index 000000000..d54033ee7 --- /dev/null +++ b/pql/internal/oldpql/parser.go @@ -0,0 +1,329 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "fmt" + "io" + "strconv" + "strings" +) + +// TimeFormat is the go-style time format used to parse string dates. +const TimeFormat = "2006-01-02T15:04" + +// Parser represents a parser for the PQL language. +type Parser struct { + scanner *bufScanner +} + +// NewParser returns a new instance of Parser. +func NewParser(r io.Reader) *Parser { + return &Parser{ + scanner: newBufScanner(r), + } +} + +// ParseString parses s into a query. +func ParseString(s string) (*Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Parse parses the next node in the query. +func (p *Parser) Parse() (*Query, error) { + q := &Query{} + for { + call, err := p.parseCall() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + q.Calls = append(q.Calls, call) + } + + // Require at least one call. + if len(q.Calls) == 0 { + return nil, io.ErrUnexpectedEOF + } + + return q, nil +} + +// parseCall parses the next function call. +func (p *Parser) parseCall() (*Call, error) { + var c Call + + // Read call name. + tok, pos, lit := p.scanIgnoreWhitespace() + if tok == EOF { + return nil, io.EOF + } else if tok != IDENT { + return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} + } + c.Name = lit + + // Scan opening parenthesis. + if err := p.expect(LPAREN); err != nil { + return nil, err + } + + // Parse children first. + children, err := p.parseChildren() + if err != nil { + return nil, err + } + c.Children = children + + // If next token is a closing paren then exit. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + return &c, nil + } else if tok == IDENT { + p.unscan(1) + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + } + + // Parse key/value arguments. + args, err := p.parseArgs() + if err != nil { + return nil, err + } + c.Args = args + + // Scan closing parenthesis. + if err := p.expect(RPAREN); err != nil { + return nil, err + } + + return &c, nil +} + +// parseChildren parses call children. +func (p *Parser) parseChildren() ([]*Call, error) { + var offset int + var children []*Call + for { + // Ensure next two tokens are IDENT+LPAREN. + if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { + p.unscanIgnoreWhitespace(1 + offset) + return children, nil + } + if tok, _, _ := p.scan(); tok != LPAREN { + p.unscanIgnoreWhitespace(2 + offset) + return children, nil + } + + // Push tokens back on scanner and parse as a call. + p.unscan(2) + child, err := p.parseCall() + if err != nil { + return nil, err + } + children = append(children, child) + + // Exit if closing paren. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + p.unscan(1) + return children, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) + } + + // Make sure comma is unscanned. + offset = 1 + } +} + +// parseArgs parses key/value arguments. +func (p *Parser) parseArgs() (map[string]interface{}, error) { + args := make(map[string]interface{}) + for { + // Parse key. + tok, pos, lit := p.scanIgnoreWhitespace() + if tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != IDENT { + return nil, parseErrorf(pos, "expected argument key, found %q", lit) + } + key := lit + + // Expect '=' or a comparison next. + var op Token + switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { + case ASSIGN: + case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: + op = tok + default: + return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) + } + + // Parse value. + var value interface{} + tok, pos, lit = p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + value = true + } else if lit == "false" { + value = false + } else if lit == "null" { + value = nil + } else { + value = lit + } + case STRING: + value = lit + case INTEGER: + v, err := strconv.ParseInt(lit, 10, 64) + if err != nil { + return nil, err + } + value = v + case FLOAT: + v, err := strconv.ParseFloat(lit, 64) + if err != nil { + return nil, err + } + value = v + case LBRACK: + v, err := p.parseList() + if err != nil { + return nil, err + } + value = v + default: + return nil, parseErrorf(pos, "invalid argument value: %q", lit) + } + + // Ensure key doesn't already exist. + if _, ok := args[key]; ok { + return nil, parseErrorf(pos, "argument key already used: %s", key) + } + + // If op is specified then create a condition. + if op != 0 { + value = &Condition{Op: op, Value: value} + } + + // Add key/value pair to arguments. + args[key] = value + + // Exit if closing paren. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) + } + } +} + +// parseList parses a list of primitives. This is used by the TopN() filters. +func (p *Parser) parseList() ([]interface{}, error) { + var values []interface{} + for { + // Read next value. + tok, pos, lit := p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + values = append(values, true) + } else if lit == "false" { + values = append(values, false) + } else { + values = append(values, lit) + } + case STRING: + values = append(values, lit) + case INTEGER: + v, err := strconv.ParseInt(lit, 10, 64) + if err != nil { + return nil, err + } + values = append(values, v) + default: + return nil, parseErrorf(pos, "invalid list value: %q", lit) + } + + // Expect a comma or closing bracket next. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { + break + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma, found %q", lit) + } + } + return values, nil +} + +// scan returns the next token from the scanner. +func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } + +// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. +func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { + tok, pos, lit = p.scan() + if tok == WS { + tok, pos, lit = p.scan() + } + return +} + +// unscan returns the last n tokens back to the scanner. +func (p *Parser) unscan(n int) { + for i := 0; i < n; i++ { + p.scanner.unscan() + } +} + +// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. +func (p *Parser) unscanIgnoreWhitespace(n int) { + for i := 0; i < n; { + p.scanner.unscan() + if tok, _, _ := p.scanner.curr(); tok != WS { + i++ + } + } +} + +// expect returns an error if the next token is not exp. +func (p *Parser) expect(exp Token) error { + if tok, pos, lit := p.scan(); tok != exp { + return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) + } + return nil +} + +// pos returns the current position. +func (p *Parser) pos() Pos { return p.scanner.pos() } + +// ParseError represents an error that occurred while parsing a PQL query. +type ParseError struct { + Message string + Pos Pos +} + +// Error returns a string representation of e. +func (e *ParseError) Error() string { + return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) +} + +// parseErrorf returns a formatted parse error. +func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { + return &ParseError{ + Message: fmt.Sprintf(format, args...), + Pos: pos, + } +} diff --git a/pql/internal/oldpql/parser_test.go b/pql/internal/oldpql/parser_test.go new file mode 100644 index 000000000..7aa8fafe1 --- /dev/null +++ b/pql/internal/oldpql/parser_test.go @@ -0,0 +1,194 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "reflect" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" + _ "github.com/pilosa/pilosa/test" +) + +// Ensure the parser can parse PQL. +func TestParser_Parse(t *testing.T) { + // Parse with no children or arguments. + t.Run("Empty", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap()`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Bitmap", + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse with only children. + t.Run("ChildrenOnly", func(t *testing.T) { + q, err := pql.ParseString(`Union( Bitmap() , Count() )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Union", + Children: []*pql.Call{ + &pql.Call{Name: "Bitmap"}, + &pql.Call{Name: "Count"}, + }, + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse a single child with a single argument. + t.Run("ChildWithArgument", func(t *testing.T) { + q, err := pql.ParseString(`Count( Bitmap( id=100))`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Count", + Children: []*pql.Call{ + {Name: "Bitmap", Args: map[string]interface{}{"id": int64(100)}}, + }, + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse with only arguments. + t.Run("ArgumentsOnly", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": "value", + "foo": "bar", + "age": int64(12), + "bool0": true, + "bool1": false, + "x": nil, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with float arguments. + t.Run("WithFloatArgs", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": 12.25, + "foo": 13.167, + "bar": 2., + "baz": 0.9, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with float arguments. + t.Run("WithNegativeArgs", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": -12.25, + "foo": int64(-13), + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with both child calls and arguments. + t.Run("ChildrenAndArguments", func(t *testing.T) { + q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Children: []*pql.Call{{ + Name: "Bitmap", + Args: map[string]interface{}{"id": int64(100), "field": "other"}, + }}, + Args: map[string]interface{}{"n": int64(3), "field": "f"}, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse a list argument. + t.Run("ListArgument", func(t *testing.T) { + q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Args: map[string]interface{}{ + "field": "f", + "ids": []interface{}{int64(0), int64(10), int64(30)}, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with condition arguments. + t.Run("WithCondition", func(t *testing.T) { + q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": "foo", + "x": &pql.Condition{Op: pql.EQ, Value: 12.25}, + "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, + "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, + "m": &pql.Condition{Op: pql.NEQ, Value: nil}, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + +} diff --git a/pql/internal/oldpql/scanner.go b/pql/internal/oldpql/scanner.go new file mode 100644 index 000000000..dd7f9126d --- /dev/null +++ b/pql/internal/oldpql/scanner.go @@ -0,0 +1,303 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "bufio" + "bytes" + "io" + "unicode" +) + +// Scanner represents a PQL lexical scanner. +type Scanner struct { + r io.RuneScanner + pos Pos +} + +// NewScanner returns a new instance of Scanner. +func NewScanner(r io.Reader) *Scanner { + return &Scanner{r: bufio.NewReader(r)} +} + +// Scan returns the next token and position from the underlying reader. +func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { + pos = s.pos + + // Read next code point. + ch := s.read() + + // If we see whitespace then consume all contiguous whitespace. + // If we see a letter, or certain acceptable special characters, then consume + // as an ident or reserved word. If we see quotes, then scan as string. + if isWhitespace(ch) { + s.unread() + return s.scanWhitespace() + } else if isIdentFirstChar(ch) { + s.unread() + return s.scanIdent() + } else if isDigit(ch) || ch == '-' { + s.unread() + return s.scanNumber() + } else if ch == '"' || ch == '\'' { + s.unread() + return s.scanString() + } + + // Otherwise parse individual characters. + switch ch { + case eof: + return EOF, pos, "" + case '=': + if next := s.read(); next == '=' { + return EQ, pos, "==" + } + s.unread() + return ASSIGN, pos, string(ch) + case '!': + if next := s.read(); next == '=' { + return NEQ, pos, "!=" + } + s.unread() + return ASSIGN, pos, string(ch) + case '<': + if next := s.read(); next == '=' { + return LTE, pos, "<=" + } + s.unread() + return LT, pos, string(ch) + case '>': + next := s.read() + if next == '=' { + return GTE, pos, ">=" + } else if next == '<' { + return BETWEEN, pos, "><" + } + s.unread() + return GT, pos, string(ch) + case ',': + return COMMA, pos, string(ch) + case '(': + return LPAREN, pos, string(ch) + case ')': + return RPAREN, pos, string(ch) + case '[': + return LBRACK, pos, string(ch) + case ']': + return RBRACK, pos, string(ch) + default: + return ILLEGAL, pos, string(ch) + } +} + +// read returns the next code point from the underlying reader and updates the pos. +func (s *Scanner) read() rune { + // Read next rune from underlying reader. + ch, _, err := s.r.ReadRune() + if err != nil { + return eof + } + + // Update position information. + if ch == '\n' { + s.pos.Line++ + s.pos.Char = 0 + } else { + s.pos.Char++ + } + + return ch +} + +// unread pushes the previously read rune back onto the reader. +func (s *Scanner) unread() { + if s.pos.Char == 0 { + s.pos.Line-- + } else { + s.pos.Char-- + } + + s.r.UnreadRune() +} + +// scanWhitespace consumes the current rune and all contiguous whitespace. +func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { + pos = s.pos + + var buf bytes.Buffer + for { + ch := s.read() + if ch == eof { + break + } else if !isWhitespace(ch) { + s.unread() + break + } + buf.WriteRune(ch) + } + + return WS, pos, buf.String() +} + +func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { + pos = s.pos + + var buf bytes.Buffer + for { + ch := s.read() + if ch == eof { + break + } else if !isIdentChar(ch) { + s.unread() + break + } + buf.WriteRune(ch) + } + lit = buf.String() + + // If the literal matches a keyword then return that keyword. + if tok = Lookup(lit); tok != IDENT { + return tok, pos, lit + } + + return IDENT, pos, lit +} + +// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. +func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { + pos = s.pos + tok = INTEGER + + var buf bytes.Buffer + var seenDot bool + first := true + for { + ch := s.read() + if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { + s.unread() + break + } + if ch == '.' { + seenDot = true + tok = FLOAT + } + buf.WriteRune(ch) + first = false + } + return tok, pos, buf.String() +} + +// scanString consumes a single-quoted or double-quoted string. +func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { + pos = s.pos + + // This must be either a single- or double-quote. + ending := s.read() + + var buf bytes.Buffer + for { + ch := s.read() + if ch == ending { + break + } else if ch == '\n' || ch == eof { + return BADSTRING, pos, buf.String() + } else if ch == '\\' { + next := s.read() + if next == 'n' { + buf.WriteRune('\n') + } else if next == '\\' { + buf.WriteRune('\\') + } else if next == '"' { + buf.WriteRune('"') + } else if next == '\'' { + buf.WriteRune('\'') + } else { + return BADSTRING, pos, buf.String() + } + } else { + buf.WriteRune(ch) + } + } + + return STRING, pos, buf.String() +} + +// bufScanner represents a wrapper for scanner to add a buffer. +// It provides a fixed-length circular buffer that can be unread. +type bufScanner struct { + s *Scanner + i int // buffer index + n int // buffer size + buf [8]struct { + tok Token + pos Pos + lit string + } +} + +// newBufScanner returns a new buffered scanner for a reader. +func newBufScanner(r io.Reader) *bufScanner { + return &bufScanner{s: NewScanner(r)} +} + +// Scan reads the next token from the scanner. +func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { + // If we have unread tokens then read them off the buffer first. + if s.n > 0 { + s.n-- + return s.curr() + } + + // Move buffer position forward and save the token. + s.i = (s.i + 1) % len(s.buf) + buf := &s.buf[s.i] + buf.tok, buf.pos, buf.lit = s.s.Scan() + + return s.curr() +} + +// unscan pushes the previously token back onto the buffer. +func (s *bufScanner) unscan() { s.n++ } + +// curr returns the last read token. +func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { + buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] + return buf.tok, buf.pos, buf.lit +} + +// pos returns the current position. +func (s *bufScanner) pos() Pos { + _, pos, _ := s.curr() + return pos +} + +// isWhitespace returns true if the rune a Unicode space character. +func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } + +// isLetter returns true if the rune is a letter. +func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } + +// isDigit returns true if the rune is a digit. +func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } + +// isIdentChar returns true if the rune can be used in an unquoted identifier. +func isIdentChar(ch rune) bool { + return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' +} + +// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. +func isIdentFirstChar(ch rune) bool { return isLetter(ch) } + +const eof = rune(0) diff --git a/pql/internal/oldpql/scanner_test.go b/pql/internal/oldpql/scanner_test.go new file mode 100644 index 000000000..3a1f462e2 --- /dev/null +++ b/pql/internal/oldpql/scanner_test.go @@ -0,0 +1,74 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "strings" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" +) + +func TestScanner_Scan(t *testing.T) { + var tests = []struct { + name string + s string + tok pql.Token + lit string + pos pql.Pos + }{ + // Special tokens (EOF, ILLEGAL, WS) + {name: "EOF", s: ``, tok: pql.EOF}, + {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, + {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, + {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, + {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, + + {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, + {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, + {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, + {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, + {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, + {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, + {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, + {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, + {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, + {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, + {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, + {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, + {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, + + {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, + {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, + {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, + + {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, + {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := pql.NewScanner(strings.NewReader(tt.s)) + tok, pos, lit := s.Scan() + if tt.tok != tok { + t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) + } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { + t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) + } else if tt.lit != lit { + t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) + } + }) + } +} diff --git a/pql/internal/oldpql/token.go b/pql/internal/oldpql/token.go new file mode 100644 index 000000000..4da3b8505 --- /dev/null +++ b/pql/internal/oldpql/token.go @@ -0,0 +1,111 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import "strings" + +// Token is a lexical token of the PQL language. +type Token int + +const ( + // Special tokens + ILLEGAL Token = iota + EOF + WS + + literal_beg + IDENT // main + STRING // "foo" + BADSTRING // bad escape or unclosed string + INTEGER // 12345 + FLOAT // 100.2 + literal_end + + keyword_beg + ALL + keyword_end + + ASSIGN // = + EQ // == + NEQ // != + LT // < + LTE // <= + GT // > + GTE // >= + BETWEEN // >< + COMMA // , + LPAREN // ( + RPAREN // ) + LBRACK // ( + RBRACK // ) +) + +var tokens = [...]string{ + ILLEGAL: "ILLEGAL", + EOF: "EOF", + WS: "WS", + + IDENT: "IDENT", + INTEGER: "INTEGER", + FLOAT: "FLOAT", + + ALL: "ALL", + + ASSIGN: "=", + EQ: "==", + NEQ: "!=", + LT: "<", + LTE: "<=", + GT: ">", + GTE: ">=", + BETWEEN: "><", + COMMA: ",", + LPAREN: "(", + RPAREN: ")", + LBRACK: "(", + RBRACK: ")", +} + +var keywords map[string]Token + +func init() { + keywords = make(map[string]Token) + for tok := keyword_beg + 1; tok < keyword_end; tok++ { + keywords[strings.ToLower(tokens[tok])] = tok + } +} + +// String returns the string representation of the token. +func (tok Token) String() string { + if tok >= 0 && tok < Token(len(tokens)) { + return tokens[tok] + } + return "" +} + +// Lookup returns the token associated with a given string. +func Lookup(ident string) Token { + if tok, ok := keywords[strings.ToLower(ident)]; ok { + return tok + } + return IDENT +} + +// Pos specifies the line and character position of a token. +// The Char and Line are both zero-based indexes. +type Pos struct { + Line int + Char int +} From 1e0592074245401992a7f1d9045557b7d2d5ac2e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 14 Jun 2018 17:40:28 -0500 Subject: [PATCH 05/33] fuzz testing and bug fixes --- http/handler_test.go | 2 +- pql/ast.go | 5 +- pql/fuzz/README.txt | 8 + ...02ad499148a94f93101dbebda5111cd061137d28-1 | 1 + ...077a5923c7f6ff1b697b556611a3593e725d515f-1 | 1 + .../0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 | 1 + pql/fuzz/corpus/1 | 1 + pql/fuzz/corpus/10 | 1 + pql/fuzz/corpus/11 | 1 + .../11f674c766421132650bcbf8ccc265a013a3409f | 1 + pql/fuzz/corpus/12 | 1 + pql/fuzz/corpus/13 | 1 + .../131cfdaafbd04db9dd2aa37fb23a656500ed1333 | 1 + pql/fuzz/corpus/14 | 1 + pql/fuzz/corpus/15 | 1 + pql/fuzz/corpus/16 | 1 + pql/fuzz/corpus/17 | 1 + pql/fuzz/corpus/18 | 1 + pql/fuzz/corpus/19 | 1 + pql/fuzz/corpus/2 | 1 + pql/fuzz/corpus/20 | 1 + pql/fuzz/corpus/21 | 1 + pql/fuzz/corpus/22 | 1 + pql/fuzz/corpus/23 | 5 + pql/fuzz/corpus/24 | 1 + pql/fuzz/corpus/25 | 2 + pql/fuzz/corpus/26 | 1 + pql/fuzz/corpus/27 | 1 + .../2751bda09fe203e30e9d5f214f9425e2dface095 | 1 + pql/fuzz/corpus/28 | 1 + pql/fuzz/corpus/29 | 1 + pql/fuzz/corpus/3 | 1 + pql/fuzz/corpus/30 | 1 + pql/fuzz/corpus/31 | 1 + .../338717d7ceeb78f7b8b864547fcb87cd62334783 | 1 + .../33fae0740e470344699582c2c8c6f3825de66007 | 1 + .../374b9d8c1d285b57c3fe1f99b76472714cc2c69c | 2 + .../392027b3a650e05b0bc4ca185143138585702c5c | 1 + .../3c9cda1dd6ed289bdec524bb9f4995a9c175d656 | 1 + pql/fuzz/corpus/4 | 1 + .../452308054231977c3f6e551b72437500215019b5 | 1 + pql/fuzz/corpus/5 | 1 + .../57e5daa393a1de6405e0315abf57cf061bd5dc44 | 1 + .../597ed3d1cef06f73136921bdd89fc2916cdd287c | 1 + .../5e982cd2a4acb990e97675afabce72032c1d08ef | 1 + .../5f6b6920de296ca3a34d3ee14477a9d623d4efc2 | 1 + pql/fuzz/corpus/6 | 1 + .../6078ffa2c7287a2fdbb9bca63274a414fd7bc83d | 1 + ...6711a6c9ab125b4444c9c03b14e49f416f25180c-1 | 1 + pql/fuzz/corpus/7 | 1 + ...7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 | 1 + .../7282523da2bd624500932760375168ac6d95b08b | 1 + ...72fca46b66ab75b1b215d42c1f97a6a601e11383-1 | 1 + ...755ea2169f42a7facac54c6d4228abad4ffdb840-1 | 1 + .../75dcc3426aa51753b37f34acaab56815ae00af91 | 1 + .../7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 | 1 + ...7e03f5068158432ddc5faa0579f6cbfc09718884-1 | 1 + pql/fuzz/corpus/8 | 1 + .../80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 | 1 + pql/fuzz/corpus/9 | 1 + .../9456f79011b99928233a5c43c89d9bcabc788a9d | 1 + ...94ebe178c54a1ed5eced6ee363799261b18740c7-1 | 1 + .../9cbc01e0a28e963310a3e6b80eeb094a3de77c06 | 1 + .../9f974590bac2e9aa23f6e93128263403ca9d109f | 1 + .../a5ef2ba5c1423d9d03d8293be378b48af8dee79e | 1 + .../af209066ba9b25655fadd130ec30aa42f9a6c606 | 1 + .../b9258eb89acc5c62232f5e482449cc155a215125 | 1 + .../c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 | 1 + .../c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 | 1 + .../cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c | 1 + ...d20407c02c966d0cac76b72486e892158dce4ba7-1 | 1 + .../d4a4d133499f09ad2d91114f55ed7235e985f7fd | 1 + .../d5dd3b391afdce17c47a2644e536431e3b5b6825 | 1 + ...da588debce70733e48a0f1728ac248ce65e9e8c2-1 | 1 + .../e2c94a638563108995f18d0daadb9d2bd8a5f0c6 | 1 + .../e373d8c28776b2d1c8740807ffbe46cdd0260f98 | 1 + .../ee78db5d4e2231cadcf5957d169657ef4658c343 | 1 + .../f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d | 1 + .../f8f3c39e99db75ff5c8772a9871185a821a75f29 | 1 + .../ff41d50e5926d166b2adc0596339201274509856 | 1 + pql/internal/oldpql/parser_test.go | 1 - pql/internal/oldpql/scanner.go | 2 +- pql/parser_fuzz.go | 115 ++ pql/pql.peg | 17 +- pql/pql.peg.go | 1502 +++++++++-------- pql/pqlpeg_test.go | 23 + 86 files changed, 1072 insertions(+), 686 deletions(-) create mode 100644 pql/fuzz/README.txt create mode 100644 pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 create mode 100644 pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 create mode 100644 pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 create mode 100644 pql/fuzz/corpus/1 create mode 100644 pql/fuzz/corpus/10 create mode 100644 pql/fuzz/corpus/11 create mode 100644 pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f create mode 100644 pql/fuzz/corpus/12 create mode 100644 pql/fuzz/corpus/13 create mode 100644 pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 create mode 100644 pql/fuzz/corpus/14 create mode 100644 pql/fuzz/corpus/15 create mode 100644 pql/fuzz/corpus/16 create mode 100644 pql/fuzz/corpus/17 create mode 100644 pql/fuzz/corpus/18 create mode 100644 pql/fuzz/corpus/19 create mode 100644 pql/fuzz/corpus/2 create mode 100644 pql/fuzz/corpus/20 create mode 100644 pql/fuzz/corpus/21 create mode 100644 pql/fuzz/corpus/22 create mode 100644 pql/fuzz/corpus/23 create mode 100644 pql/fuzz/corpus/24 create mode 100644 pql/fuzz/corpus/25 create mode 100644 pql/fuzz/corpus/26 create mode 100644 pql/fuzz/corpus/27 create mode 100644 pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 create mode 100644 pql/fuzz/corpus/28 create mode 100644 pql/fuzz/corpus/29 create mode 100644 pql/fuzz/corpus/3 create mode 100644 pql/fuzz/corpus/30 create mode 100644 pql/fuzz/corpus/31 create mode 100644 pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 create mode 100644 pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 create mode 100644 pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c create mode 100644 pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c create mode 100644 pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 create mode 100644 pql/fuzz/corpus/4 create mode 100644 pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 create mode 100644 pql/fuzz/corpus/5 create mode 100644 pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 create mode 100644 pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c create mode 100644 pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef create mode 100644 pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 create mode 100644 pql/fuzz/corpus/6 create mode 100644 pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d create mode 100644 pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 create mode 100644 pql/fuzz/corpus/7 create mode 100644 pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 create mode 100644 pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b create mode 100644 pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 create mode 100644 pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 create mode 100644 pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 create mode 100644 pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 create mode 100644 pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 create mode 100644 pql/fuzz/corpus/8 create mode 100644 pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 create mode 100644 pql/fuzz/corpus/9 create mode 100644 pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d create mode 100644 pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 create mode 100644 pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 create mode 100644 pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f create mode 100644 pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e create mode 100644 pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 create mode 100644 pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 create mode 100644 pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 create mode 100644 pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 create mode 100644 pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c create mode 100644 pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 create mode 100644 pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd create mode 100644 pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 create mode 100644 pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 create mode 100644 pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 create mode 100644 pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 create mode 100644 pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 create mode 100644 pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d create mode 100644 pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 create mode 100644 pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 create mode 100644 pql/parser_fuzz.go diff --git a/http/handler_test.go b/http/handler_test.go index a6bd2b98e..5f7913f94 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -653,7 +653,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near PegText (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near open (line 1 symbol 7 - line 1 symbol 8):\n\"(\"\n"}`+"\n" { t.Fatalf("unexpected body: \n%s", body) } } diff --git a/pql/ast.go b/pql/ast.go index 7a91ef987..6d1b206b0 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -72,11 +72,8 @@ func (q *Query) addVal(val interface{}) { return } if q.lastCond != ILLEGAL { - if val != nil || q.lastCond != NEQ { - panic(fmt.Sprintf("can't add val %s with condition %s", val, q.lastCond)) - } call.Args[q.lastField] = &Condition{ - Op: NEQ, + Op: q.lastCond, Value: val, } } else { diff --git a/pql/fuzz/README.txt b/pql/fuzz/README.txt new file mode 100644 index 000000000..e94c88830 --- /dev/null +++ b/pql/fuzz/README.txt @@ -0,0 +1,8 @@ +See https://github.com/dvyukov/go-fuzz + + +Quickstart: + +go get -u github.com/dvyukov/go-fuzz/... +go-fuzz-build github.com/pilosa/pilosa/pql +go-fuzz -bin=./pql-fuzz.zip -workdir=$GOPATH/src/github.com/pilosa/pilosa/pql/fuzz diff --git a/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 b/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 new file mode 100644 index 000000000..b38d50137 --- /dev/null +++ b/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 @@ -0,0 +1 @@ +e(rT03 \ No newline at end of file diff --git a/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 b/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 new file mode 100644 index 000000000..c4507e8e4 --- /dev/null +++ b/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 @@ -0,0 +1 @@ +e(d=f2002-01-01T03:00 \ No newline at end of file diff --git a/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 b/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 new file mode 100644 index 000000000..be461611e --- /dev/null +++ b/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 @@ -0,0 +1 @@ +e(other!=-2) \ No newline at end of file diff --git a/pql/fuzz/corpus/1 b/pql/fuzz/corpus/1 new file mode 100644 index 000000000..a8ccc9f85 --- /dev/null +++ b/pql/fuzz/corpus/1 @@ -0,0 +1 @@ +Bitmap() \ No newline at end of file diff --git a/pql/fuzz/corpus/10 b/pql/fuzz/corpus/10 new file mode 100644 index 000000000..21ff4c59c --- /dev/null +++ b/pql/fuzz/corpus/10 @@ -0,0 +1 @@ +Bitmap(row=10, field=f) \ No newline at end of file diff --git a/pql/fuzz/corpus/11 b/pql/fuzz/corpus/11 new file mode 100644 index 000000000..7636ec48c --- /dev/null +++ b/pql/fuzz/corpus/11 @@ -0,0 +1 @@ +Difference(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f b/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f new file mode 100644 index 000000000..425e9d1d3 --- /dev/null +++ b/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f @@ -0,0 +1 @@ +Range(foo<0) \ No newline at end of file diff --git a/pql/fuzz/corpus/12 b/pql/fuzz/corpus/12 new file mode 100644 index 000000000..0d59771c6 --- /dev/null +++ b/pql/fuzz/corpus/12 @@ -0,0 +1 @@ +Difference() \ No newline at end of file diff --git a/pql/fuzz/corpus/13 b/pql/fuzz/corpus/13 new file mode 100644 index 000000000..d5102d6fe --- /dev/null +++ b/pql/fuzz/corpus/13 @@ -0,0 +1 @@ +Intersect(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 b/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 new file mode 100644 index 000000000..d8c6af7ea --- /dev/null +++ b/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 @@ -0,0 +1 @@ +SV(invalid_column_name=10,f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/14 b/pql/fuzz/corpus/14 new file mode 100644 index 000000000..08695e949 --- /dev/null +++ b/pql/fuzz/corpus/14 @@ -0,0 +1 @@ +Intersect() \ No newline at end of file diff --git a/pql/fuzz/corpus/15 b/pql/fuzz/corpus/15 new file mode 100644 index 000000000..2ade2207e --- /dev/null +++ b/pql/fuzz/corpus/15 @@ -0,0 +1 @@ +Union(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/16 b/pql/fuzz/corpus/16 new file mode 100644 index 000000000..c3b496bb4 --- /dev/null +++ b/pql/fuzz/corpus/16 @@ -0,0 +1 @@ +Union() \ No newline at end of file diff --git a/pql/fuzz/corpus/17 b/pql/fuzz/corpus/17 new file mode 100644 index 000000000..55062ba4c --- /dev/null +++ b/pql/fuzz/corpus/17 @@ -0,0 +1 @@ +Xor(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/18 b/pql/fuzz/corpus/18 new file mode 100644 index 000000000..ea7190ed6 --- /dev/null +++ b/pql/fuzz/corpus/18 @@ -0,0 +1 @@ +Count(Bitmap(row=10, field=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/19 b/pql/fuzz/corpus/19 new file mode 100644 index 000000000..bde6e75ac --- /dev/null +++ b/pql/fuzz/corpus/19 @@ -0,0 +1 @@ +SetBit(row=11, field=f, col=1) \ No newline at end of file diff --git a/pql/fuzz/corpus/2 b/pql/fuzz/corpus/2 new file mode 100644 index 000000000..48ffdc060 --- /dev/null +++ b/pql/fuzz/corpus/2 @@ -0,0 +1 @@ +Union( Bitmap() , Count() ) \ No newline at end of file diff --git a/pql/fuzz/corpus/20 b/pql/fuzz/corpus/20 new file mode 100644 index 000000000..76679c897 --- /dev/null +++ b/pql/fuzz/corpus/20 @@ -0,0 +1 @@ +SetValue(col=10, f=25) \ No newline at end of file diff --git a/pql/fuzz/corpus/21 b/pql/fuzz/corpus/21 new file mode 100644 index 000000000..4ad8fba18 --- /dev/null +++ b/pql/fuzz/corpus/21 @@ -0,0 +1 @@ +SetValue(invalid_column_name=10, f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/22 b/pql/fuzz/corpus/22 new file mode 100644 index 000000000..123a4e7b2 --- /dev/null +++ b/pql/fuzz/corpus/22 @@ -0,0 +1 @@ +SetRowAttrs(row=10, field=f, baz=123, bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/23 b/pql/fuzz/corpus/23 new file mode 100644 index 000000000..31333c37b --- /dev/null +++ b/pql/fuzz/corpus/23 @@ -0,0 +1,5 @@ + SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00") + + SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00") + diff --git a/pql/fuzz/corpus/24 b/pql/fuzz/corpus/24 new file mode 100644 index 000000000..527b67ddc --- /dev/null +++ b/pql/fuzz/corpus/24 @@ -0,0 +1 @@ +Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00") \ No newline at end of file diff --git a/pql/fuzz/corpus/25 b/pql/fuzz/corpus/25 new file mode 100644 index 000000000..32c0405c1 --- /dev/null +++ b/pql/fuzz/corpus/25 @@ -0,0 +1,2 @@ + +Range(foo == 20) diff --git a/pql/fuzz/corpus/26 b/pql/fuzz/corpus/26 new file mode 100644 index 000000000..4cad8028b --- /dev/null +++ b/pql/fuzz/corpus/26 @@ -0,0 +1 @@ +Range(other != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/27 b/pql/fuzz/corpus/27 new file mode 100644 index 000000000..c858f1930 --- /dev/null +++ b/pql/fuzz/corpus/27 @@ -0,0 +1 @@ +Range(foo != 20) \ No newline at end of file diff --git a/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 b/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 new file mode 100644 index 000000000..f02ab7e3d --- /dev/null +++ b/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 @@ -0,0 +1 @@ +N(p(d=0,l=other), d=f,n=3) \ No newline at end of file diff --git a/pql/fuzz/corpus/28 b/pql/fuzz/corpus/28 new file mode 100644 index 000000000..212663384 --- /dev/null +++ b/pql/fuzz/corpus/28 @@ -0,0 +1 @@ +Range(other != -20) \ No newline at end of file diff --git a/pql/fuzz/corpus/29 b/pql/fuzz/corpus/29 new file mode 100644 index 000000000..3d2e5b82b --- /dev/null +++ b/pql/fuzz/corpus/29 @@ -0,0 +1 @@ +Range(foo < 20) \ No newline at end of file diff --git a/pql/fuzz/corpus/3 b/pql/fuzz/corpus/3 new file mode 100644 index 000000000..aef5a7a75 --- /dev/null +++ b/pql/fuzz/corpus/3 @@ -0,0 +1 @@ +Count( Bitmap( id=100)) \ No newline at end of file diff --git a/pql/fuzz/corpus/30 b/pql/fuzz/corpus/30 new file mode 100644 index 000000000..3e3f37870 --- /dev/null +++ b/pql/fuzz/corpus/30 @@ -0,0 +1 @@ +Range(foo <= 20) diff --git a/pql/fuzz/corpus/31 b/pql/fuzz/corpus/31 new file mode 100644 index 000000000..13b7e9347 --- /dev/null +++ b/pql/fuzz/corpus/31 @@ -0,0 +1 @@ +SetRowAttrs(row=10, field=f, baz=12.3, bat=.21, bak=-.27, zaz=-0.27 , q=0, zoo="0", do='0') diff --git a/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 b/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 new file mode 100644 index 000000000..a03a91bd9 --- /dev/null +++ b/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 @@ -0,0 +1 @@ +t( p( )) \ No newline at end of file diff --git a/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 b/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 new file mode 100644 index 000000000..23e1bc1de --- /dev/null +++ b/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 @@ -0,0 +1 @@ +SetRowAttrs(row=10,field=f,baz=123,bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c b/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c new file mode 100644 index 000000000..c65d51e92 --- /dev/null +++ b/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c @@ -0,0 +1,2 @@ + +e(o == 0) diff --git a/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c b/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c new file mode 100644 index 000000000..2b2542414 --- /dev/null +++ b/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c @@ -0,0 +1 @@ +e(r!=-2) \ No newline at end of file diff --git a/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 b/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 new file mode 100644 index 000000000..822aa982c --- /dev/null +++ b/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 @@ -0,0 +1 @@ +MyCall( y=-12.25, o= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/4 b/pql/fuzz/corpus/4 new file mode 100644 index 000000000..22982532c --- /dev/null +++ b/pql/fuzz/corpus/4 @@ -0,0 +1 @@ +MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null ) \ No newline at end of file diff --git a/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 b/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 new file mode 100644 index 000000000..65f6e27b8 --- /dev/null +++ b/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 @@ -0,0 +1 @@ +t(p(w=1,l=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/5 b/pql/fuzz/corpus/5 new file mode 100644 index 000000000..8075673eb --- /dev/null +++ b/pql/fuzz/corpus/5 @@ -0,0 +1 @@ +MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9) \ No newline at end of file diff --git a/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 b/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 new file mode 100644 index 000000000..d2042629e --- /dev/null +++ b/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 @@ -0,0 +1 @@ +e(row=1,field=f,start="1999-12-31T00:00",end="2002-01-01T03:00") \ No newline at end of file diff --git a/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c b/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c new file mode 100644 index 000000000..b22ab81d2 --- /dev/null +++ b/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c @@ -0,0 +1 @@ +MyCall(ke=foo, x =5, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef b/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef new file mode 100644 index 000000000..a7c359cfc --- /dev/null +++ b/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef @@ -0,0 +1 @@ +SetValue(invalid_column_name=10,f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 b/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 new file mode 100644 index 000000000..e8d9b2dc2 --- /dev/null +++ b/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 @@ -0,0 +1 @@ +Range(other!=null) \ No newline at end of file diff --git a/pql/fuzz/corpus/6 b/pql/fuzz/corpus/6 new file mode 100644 index 000000000..919a949ac --- /dev/null +++ b/pql/fuzz/corpus/6 @@ -0,0 +1 @@ +MyCall( key=-12.25, foo= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d b/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d new file mode 100644 index 000000000..87168a0b5 --- /dev/null +++ b/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d @@ -0,0 +1 @@ +tRowAttrs(row=1, field=f, baz=13,bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 b/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 new file mode 100644 index 000000000..00c36326c --- /dev/null +++ b/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 @@ -0,0 +1 @@ +e(w=: \ No newline at end of file diff --git a/pql/fuzz/corpus/7 b/pql/fuzz/corpus/7 new file mode 100644 index 000000000..b5a946470 --- /dev/null +++ b/pql/fuzz/corpus/7 @@ -0,0 +1 @@ +TopN(field="f", ids=[0,10,30]) \ No newline at end of file diff --git a/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 b/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 new file mode 100644 index 000000000..9170e620a --- /dev/null +++ b/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 @@ -0,0 +1 @@ +n(p() , C \ No newline at end of file diff --git a/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b b/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b new file mode 100644 index 000000000..966f30ab4 --- /dev/null +++ b/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b @@ -0,0 +1 @@ +t(p(d=100)) \ No newline at end of file diff --git a/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 b/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 new file mode 100644 index 000000000..229ba77a9 --- /dev/null +++ b/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 @@ -0,0 +1 @@ +U(B(,C \ No newline at end of file diff --git a/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 b/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 new file mode 100644 index 000000000..882c1dac8 --- /dev/null +++ b/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 @@ -0,0 +1 @@ +e(w=12002 \ No newline at end of file diff --git a/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 b/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 new file mode 100644 index 000000000..201e6ddaa --- /dev/null +++ b/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 @@ -0,0 +1 @@ +e(o <= 0) diff --git a/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 b/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 new file mode 100644 index 000000000..394c6b092 --- /dev/null +++ b/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 @@ -0,0 +1 @@ +t(p(w=0), p(w=1)) \ No newline at end of file diff --git a/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 b/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 new file mode 100644 index 000000000..0bf263b2e --- /dev/null +++ b/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 @@ -0,0 +1 @@ +n(p(),C( \ No newline at end of file diff --git a/pql/fuzz/corpus/8 b/pql/fuzz/corpus/8 new file mode 100644 index 000000000..29ce05cfd --- /dev/null +++ b/pql/fuzz/corpus/8 @@ -0,0 +1 @@ +TopN(Bitmap(id=100, field=other), field=f, n=3) \ No newline at end of file diff --git a/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 b/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 new file mode 100644 index 000000000..dd54822fe --- /dev/null +++ b/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 @@ -0,0 +1 @@ +C(y=12.25,o=13.167,r=2.,z=0.9) \ No newline at end of file diff --git a/pql/fuzz/corpus/9 b/pql/fuzz/corpus/9 new file mode 100644 index 000000000..870c1835c --- /dev/null +++ b/pql/fuzz/corpus/9 @@ -0,0 +1 @@ +MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d b/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d new file mode 100644 index 000000000..f7c988077 --- /dev/null +++ b/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d @@ -0,0 +1 @@ +t(p( )) \ No newline at end of file diff --git a/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 b/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 new file mode 100644 index 000000000..10e6841d0 --- /dev/null +++ b/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 @@ -0,0 +1 @@ +e(invalid_column_name<0,f=0) \ No newline at end of file diff --git a/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 b/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 new file mode 100644 index 000000000..090a8a693 --- /dev/null +++ b/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 @@ -0,0 +1 @@ +tV(f=5) \ No newline at end of file diff --git a/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f b/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f new file mode 100644 index 000000000..4b2b7b2bb --- /dev/null +++ b/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f @@ -0,0 +1 @@ +t(Ba(w=0,d=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e b/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e new file mode 100644 index 000000000..dcf964fd5 --- /dev/null +++ b/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e @@ -0,0 +1 @@ +Range(o < 0) \ No newline at end of file diff --git a/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 b/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 new file mode 100644 index 000000000..74ac16027 --- /dev/null +++ b/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 @@ -0,0 +1 @@ +n() \ No newline at end of file diff --git a/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 b/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 new file mode 100644 index 000000000..a9bb8167b --- /dev/null +++ b/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 @@ -0,0 +1 @@ +Intersect(Bitmap(row=10),Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 b/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 new file mode 100644 index 000000000..420a255a2 --- /dev/null +++ b/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 @@ -0,0 +1 @@ +Cl( k=-12.25, f= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 b/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 new file mode 100644 index 000000000..990d4e833 --- /dev/null +++ b/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 @@ -0,0 +1 @@ +e(o<0) \ No newline at end of file diff --git a/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c b/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c new file mode 100644 index 000000000..6e6fab1ca --- /dev/null +++ b/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c @@ -0,0 +1 @@ +Difference(Bitmap(row=10),Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 b/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 new file mode 100644 index 000000000..789a07fc4 --- /dev/null +++ b/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 @@ -0,0 +1 @@ +j(w=10375035658,t=R) \ No newline at end of file diff --git a/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd b/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd new file mode 100644 index 000000000..95edd6a6f --- /dev/null +++ b/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd @@ -0,0 +1 @@ +Range(other!=l) \ No newline at end of file diff --git a/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 b/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 new file mode 100644 index 000000000..e0dfe5315 --- /dev/null +++ b/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 @@ -0,0 +1 @@ +e(o<=0) diff --git a/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 b/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 new file mode 100644 index 000000000..3c37e86b7 --- /dev/null +++ b/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 @@ -0,0 +1 @@ +e(w=T \ No newline at end of file diff --git a/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 b/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 new file mode 100644 index 000000000..cdc7903a6 --- /dev/null +++ b/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 @@ -0,0 +1 @@ +l(key=oo, x == 12.25, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 b/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 new file mode 100644 index 000000000..a692598ed --- /dev/null +++ b/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 @@ -0,0 +1 @@ +SB(ow=1, f=f, c=1) \ No newline at end of file diff --git a/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 b/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 new file mode 100644 index 000000000..beb610cd7 --- /dev/null +++ b/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 @@ -0,0 +1 @@ +Setalue(invalidcolumnnamf=0) \ No newline at end of file diff --git a/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d b/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d new file mode 100644 index 000000000..5cac44705 --- /dev/null +++ b/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d @@ -0,0 +1 @@ +N(field="f",ids=[0,10,30]) \ No newline at end of file diff --git a/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 b/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 new file mode 100644 index 000000000..b13f3aff7 --- /dev/null +++ b/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 @@ -0,0 +1 @@ +U( B() , C() ) \ No newline at end of file diff --git a/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 b/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 new file mode 100644 index 000000000..adca39514 --- /dev/null +++ b/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 @@ -0,0 +1 @@ +Range(r!=null) \ No newline at end of file diff --git a/pql/internal/oldpql/parser_test.go b/pql/internal/oldpql/parser_test.go index 7aa8fafe1..31613429c 100644 --- a/pql/internal/oldpql/parser_test.go +++ b/pql/internal/oldpql/parser_test.go @@ -190,5 +190,4 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) - } diff --git a/pql/internal/oldpql/scanner.go b/pql/internal/oldpql/scanner.go index dd7f9126d..27e321a0c 100644 --- a/pql/internal/oldpql/scanner.go +++ b/pql/internal/oldpql/scanner.go @@ -71,7 +71,7 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { return NEQ, pos, "!=" } s.unread() - return ASSIGN, pos, string(ch) + return ILLEGAL, pos, string(ch) case '<': if next := s.read(); next == '=' { return LTE, pos, "<=" diff --git a/pql/parser_fuzz.go b/pql/parser_fuzz.go new file mode 100644 index 000000000..ffcea44ab --- /dev/null +++ b/pql/parser_fuzz.go @@ -0,0 +1,115 @@ +// +build gofuzz + +package pql + +import ( + "bytes" + "fmt" + "reflect" + + "github.com/pilosa/pilosa/pql/internal/oldpql" + "github.com/pkg/errors" +) + +func Fuzz(data []byte) int { + p1 := NewParser(bytes.NewReader(data)) + q1, err1 := p1.Parse() + p2 := oldpql.NewParser(bytes.NewReader(data)) + q2, err2 := p2.Parse() + if err1 != nil && err2 != nil { + return 0 // both error - this is fine + } + if err1 != nil || err2 != nil { + // error in one but not both - need to know this + panic(fmt.Sprintf("Query: '%s' errored one but not both.\n%v\n%v\n", data, err1, err2)) + } + + // if parsers got different results + if err := queriesEqual(q1, q2); err != nil { + panic(fmt.Sprintf(`Query: '%s' parsed, but got different results: +Result New (string) +%s +Result New (hashv) +%#v +Result Old (string) +%s +Result Old (hashv) +%#v +err: +%v +`, data, q1, q1, q2, q2, err)) + } + + // both queries parsed succesfully and got equivalent results + return 1 +} + +func queriesEqual(q1 *Query, q2 *oldpql.Query) (err error) { + if q1.String() != q2.String() { + defer func() { + // golang black magic + if err == nil { + err = errors.New("string reps unequal") + } else { + err = errors.Wrap(err, "string reps unequal") + } + }() + } + if len(q1.Calls) != len(q2.Calls) { + return errors.Errorf("call lengths unequal: %d and %d", len(q1.Calls), len(q2.Calls)) + } + for i, c1 := range q1.Calls { + c2 := q2.Calls[i] + if err := callsEqual(c1, c2); err != nil { + return errors.Wrapf(err, "calls at %d not equal", i) + } + } + return nil +} + +func callsEqual(c1 *Call, c2 *oldpql.Call) error { + if err := argsEqual(c1.Args, c2.Args); err != nil { + return errors.Wrap(err, "args unequal") + } + if c1.Name != c2.Name { + return errors.Errorf("names unequal '%s' != '%s'", c1.Name, c2.Name) + } + if len(c1.Children) != len(c2.Children) { + return errors.Errorf("different child lengths %d and %d", len(c1.Children), len(c2.Children)) + } + + for i, child1 := range c1.Children { + child2 := c2.Children[i] + if err := callsEqual(child1, child2); err != nil { + return errors.Wrapf(err, "children at %d not equal", i) + } + } + + return nil +} + +func argsEqual(a1 map[string]interface{}, a2 map[string]interface{}) error { + if len(a1) != len(a2) { + return errors.Errorf("lengths unequal %d and %d", len(a1), len(a2)) + } + + for k, v1 := range a1 { + v2 := a1[k] + if c1, ok := v1.(Condition); ok { + if c2, ok := v2.(oldpql.Condition); ok { + if int(c1.Op) != int(c2.Op) { + return errors.Errorf("condition ops unequal %d %d", c1, c2) + } + if !reflect.DeepEqual(c1.Value, c2.Value) { + return errors.Errorf("condition values unequal '%v' '%v'", c1.Value, c2.Value) + } + continue + } + return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2) + } + if !reflect.DeepEqual(v1, v2) { + return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2) + } + } + return nil +} diff --git a/pql/pql.peg b/pql/pql.peg index b909c4450..3602f8cc1 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -6,10 +6,10 @@ type PQL Peg { Calls <- Call* !. -Call <- newline* < [[A-Z]]+ > { p.startCall(buffer[begin:end] ) } open args close newline* { p.endCall() } -args <- arg (comma args)? sp / sp -arg <- ( Call - / field sp '=' sp value +Call <- whitesp < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close whitesp { p.endCall() } +allargs <- Call (comma Call)* (comma args)? / comma? args / sp +args <- arg (comma args)? sp +arg <- ( field sp '=' sp value / field sp COND sp value ) COND <- ( '><' { p.addBTWN() } @@ -25,9 +25,9 @@ value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) list <- item (comma list)? -item <- ( 'null' { p.addVal(nil) } - / 'true' { p.addVal(true) } - / 'false' { p.addVal(false) } +item <- ( 'null' &(comma / sp close) { p.addVal(nil) } + / 'true' &(comma / sp close) { p.addVal(true) } + / 'false' &(comma / sp close) { p.addVal(false) } / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } @@ -44,4 +44,5 @@ sp <- ( ' ' / '\t' )* comma <- sp ',' sp lbrack <- '[' sp rbrack <- sp ']' sp -newline <- sp '\n' sp \ No newline at end of file +whitesp <- ( ' ' / '\t' / '\n' )* +IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / '-' / '_' / '.')* \ No newline at end of file diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 02ce64918..e4680d613 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -18,6 +18,7 @@ const ( ruleUnknown pegRule = iota ruleCalls ruleCall + ruleallargs ruleargs rulearg ruleCOND @@ -33,7 +34,8 @@ const ( rulecomma rulelbrack rulerbrack - rulenewline + rulewhitesp + ruleIDENT rulePegText ruleAction0 ruleAction1 @@ -61,6 +63,7 @@ var rul3s = [...]string{ "Unknown", "Calls", "Call", + "allargs", "args", "arg", "COND", @@ -76,7 +79,8 @@ var rul3s = [...]string{ "comma", "lbrack", "rbrack", - "newline", + "whitesp", + "IDENT", "PegText", "Action0", "Action1", @@ -214,7 +218,7 @@ type PQL struct { Buffer string buffer []rune - rules [40]func() bool + rules [42]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -451,67 +455,92 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(newline* <([a-z] / [A-Z])+> Action0 open args close newline* Action1)> */ + /* 1 Call <- <(whitesp Action0 open allargs comma? close whitesp Action1)> */ func() bool { position5, tokenIndex5 := position, tokenIndex { position6 := position - l7: - { - position8, tokenIndex8 := position, tokenIndex - if !_rules[rulenewline]() { - goto l8 - } - goto l7 - l8: - position, tokenIndex = position8, tokenIndex8 + if !_rules[rulewhitesp]() { + goto l5 } { - position9 := position + position7 := position { - position12, tokenIndex12 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l13 - } - position++ - goto l12 - l13: - position, tokenIndex = position12, tokenIndex12 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l5 - } - position++ - } - l12: - l10: - { - position11, tokenIndex11 := position, tokenIndex + position8 := position { - position14, tokenIndex14 := position, tokenIndex + position9, tokenIndex9 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l15 + goto l10 } position++ - goto l14 - l15: - position, tokenIndex = position14, tokenIndex14 + goto l9 + l10: + position, tokenIndex = position9, tokenIndex9 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l11 + goto l5 } position++ } - l14: - goto l10 + l9: l11: - position, tokenIndex = position11, tokenIndex11 + { + position12, tokenIndex12 := position, tokenIndex + { + position13, tokenIndex13 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l14 + } + position++ + goto l13 + l14: + position, tokenIndex = position13, tokenIndex13 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l15 + } + position++ + goto l13 + l15: + position, tokenIndex = position13, tokenIndex13 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l16 + } + position++ + goto l13 + l16: + position, tokenIndex = position13, tokenIndex13 + if buffer[position] != rune('-') { + goto l17 + } + position++ + goto l13 + l17: + position, tokenIndex = position13, tokenIndex13 + if buffer[position] != rune('_') { + goto l18 + } + position++ + goto l13 + l18: + position, tokenIndex = position13, tokenIndex13 + if buffer[position] != rune('.') { + goto l12 + } + position++ + } + l13: + goto l11 + l12: + position, tokenIndex = position12, tokenIndex12 + } + add(ruleIDENT, position8) } - add(rulePegText, position9) + add(rulePegText, position7) } { add(ruleAction0, position) } { - position17 := position + position20 := position if buffer[position] != rune('(') { goto l5 } @@ -519,31 +548,82 @@ func (p *PQL) Init() { if !_rules[rulesp]() { goto l5 } - add(ruleopen, position17) + add(ruleopen, position20) } - if !_rules[ruleargs]() { + { + position21 := position + { + position22, tokenIndex22 := position, tokenIndex + if !_rules[ruleCall]() { + goto l23 + } + l24: + { + position25, tokenIndex25 := position, tokenIndex + if !_rules[rulecomma]() { + goto l25 + } + if !_rules[ruleCall]() { + goto l25 + } + goto l24 + l25: + position, tokenIndex = position25, tokenIndex25 + } + { + position26, tokenIndex26 := position, tokenIndex + if !_rules[rulecomma]() { + goto l26 + } + if !_rules[ruleargs]() { + goto l26 + } + goto l27 + l26: + position, tokenIndex = position26, tokenIndex26 + } + l27: + goto l22 + l23: + position, tokenIndex = position22, tokenIndex22 + { + position29, tokenIndex29 := position, tokenIndex + if !_rules[rulecomma]() { + goto l29 + } + goto l30 + l29: + position, tokenIndex = position29, tokenIndex29 + } + l30: + if !_rules[ruleargs]() { + goto l28 + } + goto l22 + l28: + position, tokenIndex = position22, tokenIndex22 + if !_rules[rulesp]() { + goto l5 + } + } + l22: + add(ruleallargs, position21) + } + { + position31, tokenIndex31 := position, tokenIndex + if !_rules[rulecomma]() { + goto l31 + } + goto l32 + l31: + position, tokenIndex = position31, tokenIndex31 + } + l32: + if !_rules[ruleclose]() { goto l5 } - { - position18 := position - if buffer[position] != rune(')') { - goto l5 - } - position++ - if !_rules[rulesp]() { - goto l5 - } - add(ruleclose, position18) - } - l19: - { - position20, tokenIndex20 := position, tokenIndex - if !_rules[rulenewline]() { - goto l20 - } - goto l19 - l20: - position, tokenIndex = position20, tokenIndex20 + if !_rules[rulewhitesp]() { + goto l5 } { add(ruleAction1, position) @@ -555,968 +635,1048 @@ func (p *PQL) Init() { position, tokenIndex = position5, tokenIndex5 return false }, - /* 2 args <- <((arg (comma args)? sp) / sp)> */ + /* 2 allargs <- <((Call (comma Call)* (comma args)?) / (comma? args) / sp)> */ + nil, + /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position22, tokenIndex22 := position, tokenIndex + position35, tokenIndex35 := position, tokenIndex { - position23 := position + position36 := position { - position24, tokenIndex24 := position, tokenIndex + position37 := position { - position26 := position - { - position27, tokenIndex27 := position, tokenIndex - if !_rules[ruleCall]() { - goto l28 - } - goto l27 - l28: - position, tokenIndex = position27, tokenIndex27 - if !_rules[rulefield]() { - goto l29 - } - if !_rules[rulesp]() { - goto l29 - } - if buffer[position] != rune('=') { - goto l29 - } - position++ - if !_rules[rulesp]() { - goto l29 - } - if !_rules[rulevalue]() { - goto l29 - } - goto l27 - l29: - position, tokenIndex = position27, tokenIndex27 - if !_rules[rulefield]() { - goto l25 - } - if !_rules[rulesp]() { - goto l25 - } - { - position30 := position - { - position31, tokenIndex31 := position, tokenIndex - if buffer[position] != rune('>') { - goto l32 - } - position++ - if buffer[position] != rune('<') { - goto l32 - } - position++ - { - add(ruleAction2, position) - } - goto l31 - l32: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('<') { - goto l34 - } - position++ - if buffer[position] != rune('=') { - goto l34 - } - position++ - { - add(ruleAction3, position) - } - goto l31 - l34: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('>') { - goto l36 - } - position++ - if buffer[position] != rune('=') { - goto l36 - } - position++ - { - add(ruleAction4, position) - } - goto l31 - l36: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('=') { - goto l38 - } - position++ - if buffer[position] != rune('=') { - goto l38 - } - position++ - { - add(ruleAction5, position) - } - goto l31 - l38: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('!') { - goto l40 - } - position++ - if buffer[position] != rune('=') { - goto l40 - } - position++ - { - add(ruleAction6, position) - } - goto l31 - l40: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('<') { - goto l42 - } - position++ - { - add(ruleAction7, position) - } - goto l31 - l42: - position, tokenIndex = position31, tokenIndex31 - if buffer[position] != rune('>') { - goto l25 - } - position++ - { - add(ruleAction8, position) - } - } - l31: - add(ruleCOND, position30) - } - if !_rules[rulesp]() { - goto l25 - } - if !_rules[rulevalue]() { - goto l25 - } + position38, tokenIndex38 := position, tokenIndex + if !_rules[rulefield]() { + goto l39 } - l27: - add(rulearg, position26) - } - { - position45, tokenIndex45 := position, tokenIndex - if !_rules[rulecomma]() { - goto l45 + if !_rules[rulesp]() { + goto l39 } - if !_rules[ruleargs]() { - goto l45 - } - goto l46 - l45: - position, tokenIndex = position45, tokenIndex45 - } - l46: - if !_rules[rulesp]() { - goto l25 - } - goto l24 - l25: - position, tokenIndex = position24, tokenIndex24 - if !_rules[rulesp]() { - goto l22 - } - } - l24: - add(ruleargs, position23) - } - return true - l22: - position, tokenIndex = position22, tokenIndex22 - return false - }, - /* 3 arg <- <(Call / (field sp '=' sp value) / (field sp COND sp value))> */ - nil, - /* 4 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ('=' '=' Action5) / ('!' '=' Action6) / ('<' Action7) / ('>' Action8))> */ - nil, - /* 5 open <- <('(' sp)> */ - nil, - /* 6 value <- <(item / (lbrack Action9 list rbrack Action10))> */ - func() bool { - position50, tokenIndex50 := position, tokenIndex - { - position51 := position - { - position52, tokenIndex52 := position, tokenIndex - if !_rules[ruleitem]() { - goto l53 - } - goto l52 - l53: - position, tokenIndex = position52, tokenIndex52 - { - position54 := position - if buffer[position] != rune('[') { - goto l50 + if buffer[position] != rune('=') { + goto l39 } position++ if !_rules[rulesp]() { - goto l50 + goto l39 } - add(rulelbrack, position54) + if !_rules[rulevalue]() { + goto l39 + } + goto l38 + l39: + position, tokenIndex = position38, tokenIndex38 + if !_rules[rulefield]() { + goto l35 + } + if !_rules[rulesp]() { + goto l35 + } + { + position40 := position + { + position41, tokenIndex41 := position, tokenIndex + if buffer[position] != rune('>') { + goto l42 + } + position++ + if buffer[position] != rune('<') { + goto l42 + } + position++ + { + add(ruleAction2, position) + } + goto l41 + l42: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('<') { + goto l44 + } + position++ + if buffer[position] != rune('=') { + goto l44 + } + position++ + { + add(ruleAction3, position) + } + goto l41 + l44: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('>') { + goto l46 + } + position++ + if buffer[position] != rune('=') { + goto l46 + } + position++ + { + add(ruleAction4, position) + } + goto l41 + l46: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('=') { + goto l48 + } + position++ + if buffer[position] != rune('=') { + goto l48 + } + position++ + { + add(ruleAction5, position) + } + goto l41 + l48: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('!') { + goto l50 + } + position++ + if buffer[position] != rune('=') { + goto l50 + } + position++ + { + add(ruleAction6, position) + } + goto l41 + l50: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('<') { + goto l52 + } + position++ + { + add(ruleAction7, position) + } + goto l41 + l52: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('>') { + goto l35 + } + position++ + { + add(ruleAction8, position) + } + } + l41: + add(ruleCOND, position40) + } + if !_rules[rulesp]() { + goto l35 + } + if !_rules[rulevalue]() { + goto l35 + } + } + l38: + add(rulearg, position37) + } + { + position55, tokenIndex55 := position, tokenIndex + if !_rules[rulecomma]() { + goto l55 + } + if !_rules[ruleargs]() { + goto l55 + } + goto l56 + l55: + position, tokenIndex = position55, tokenIndex55 + } + l56: + if !_rules[rulesp]() { + goto l35 + } + add(ruleargs, position36) + } + return true + l35: + position, tokenIndex = position35, tokenIndex35 + return false + }, + /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ + nil, + /* 5 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ('=' '=' Action5) / ('!' '=' Action6) / ('<' Action7) / ('>' Action8))> */ + nil, + /* 6 open <- <('(' sp)> */ + nil, + /* 7 value <- <(item / (lbrack Action9 list rbrack Action10))> */ + func() bool { + position60, tokenIndex60 := position, tokenIndex + { + position61 := position + { + position62, tokenIndex62 := position, tokenIndex + if !_rules[ruleitem]() { + goto l63 + } + goto l62 + l63: + position, tokenIndex = position62, tokenIndex62 + { + position64 := position + if buffer[position] != rune('[') { + goto l60 + } + position++ + if !_rules[rulesp]() { + goto l60 + } + add(rulelbrack, position64) } { add(ruleAction9, position) } if !_rules[rulelist]() { - goto l50 + goto l60 } { - position56 := position + position66 := position if !_rules[rulesp]() { - goto l50 + goto l60 } if buffer[position] != rune(']') { - goto l50 + goto l60 } position++ if !_rules[rulesp]() { - goto l50 + goto l60 } - add(rulerbrack, position56) + add(rulerbrack, position66) } { add(ruleAction10, position) } } - l52: - add(rulevalue, position51) + l62: + add(rulevalue, position61) } return true - l50: - position, tokenIndex = position50, tokenIndex50 + l60: + position, tokenIndex = position60, tokenIndex60 return false }, - /* 7 list <- <(item (comma list)?)> */ + /* 8 list <- <(item (comma list)?)> */ func() bool { - position58, tokenIndex58 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex { - position59 := position + position69 := position if !_rules[ruleitem]() { - goto l58 + goto l68 } { - position60, tokenIndex60 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if !_rules[rulecomma]() { - goto l60 + goto l70 } if !_rules[rulelist]() { - goto l60 + goto l70 } - goto l61 - l60: - position, tokenIndex = position60, tokenIndex60 + goto l71 + l70: + position, tokenIndex = position70, tokenIndex70 } - l61: - add(rulelist, position59) + l71: + add(rulelist, position69) } return true - l58: - position, tokenIndex = position58, tokenIndex58 + l68: + position, tokenIndex = position68, tokenIndex68 return false }, - /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action16) / ('"' '"' Action17) / ('\'' '\'' Action18))> */ + /* 9 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action11) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action12) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action16) / ('"' '"' Action17) / ('\'' '\'' Action18))> */ func() bool { - position62, tokenIndex62 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex { - position63 := position + position73 := position { - position64, tokenIndex64 := position, tokenIndex + position74, tokenIndex74 := position, tokenIndex if buffer[position] != rune('n') { - goto l65 + goto l75 } position++ if buffer[position] != rune('u') { - goto l65 + goto l75 } position++ if buffer[position] != rune('l') { - goto l65 + goto l75 } position++ if buffer[position] != rune('l') { - goto l65 + goto l75 } position++ + { + position76, tokenIndex76 := position, tokenIndex + { + position77, tokenIndex77 := position, tokenIndex + if !_rules[rulecomma]() { + goto l78 + } + goto l77 + l78: + position, tokenIndex = position77, tokenIndex77 + if !_rules[rulesp]() { + goto l75 + } + if !_rules[ruleclose]() { + goto l75 + } + } + l77: + position, tokenIndex = position76, tokenIndex76 + } { add(ruleAction11, position) } - goto l64 - l65: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l75: + position, tokenIndex = position74, tokenIndex74 if buffer[position] != rune('t') { - goto l67 + goto l80 } position++ if buffer[position] != rune('r') { - goto l67 + goto l80 } position++ if buffer[position] != rune('u') { - goto l67 + goto l80 } position++ if buffer[position] != rune('e') { - goto l67 + goto l80 } position++ + { + position81, tokenIndex81 := position, tokenIndex + { + position82, tokenIndex82 := position, tokenIndex + if !_rules[rulecomma]() { + goto l83 + } + goto l82 + l83: + position, tokenIndex = position82, tokenIndex82 + if !_rules[rulesp]() { + goto l80 + } + if !_rules[ruleclose]() { + goto l80 + } + } + l82: + position, tokenIndex = position81, tokenIndex81 + } { add(ruleAction12, position) } - goto l64 - l67: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l80: + position, tokenIndex = position74, tokenIndex74 if buffer[position] != rune('f') { - goto l69 + goto l85 } position++ if buffer[position] != rune('a') { - goto l69 + goto l85 } position++ if buffer[position] != rune('l') { - goto l69 + goto l85 } position++ if buffer[position] != rune('s') { - goto l69 + goto l85 } position++ if buffer[position] != rune('e') { - goto l69 + goto l85 } position++ + { + position86, tokenIndex86 := position, tokenIndex + { + position87, tokenIndex87 := position, tokenIndex + if !_rules[rulecomma]() { + goto l88 + } + goto l87 + l88: + position, tokenIndex = position87, tokenIndex87 + if !_rules[rulesp]() { + goto l85 + } + if !_rules[ruleclose]() { + goto l85 + } + } + l87: + position, tokenIndex = position86, tokenIndex86 + } { add(ruleAction13, position) } - goto l64 - l69: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l85: + position, tokenIndex = position74, tokenIndex74 { - position72 := position + position91 := position { - position73, tokenIndex73 := position, tokenIndex + position92, tokenIndex92 := position, tokenIndex if buffer[position] != rune('-') { - goto l73 + goto l92 } position++ - goto l74 - l73: - position, tokenIndex = position73, tokenIndex73 + goto l93 + l92: + position, tokenIndex = position92, tokenIndex92 } - l74: + l93: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l71 + goto l90 } position++ - l75: + l94: { - position76, tokenIndex76 := position, tokenIndex + position95, tokenIndex95 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l76 + goto l95 } position++ - goto l75 - l76: - position, tokenIndex = position76, tokenIndex76 + goto l94 + l95: + position, tokenIndex = position95, tokenIndex95 } { - position77, tokenIndex77 := position, tokenIndex + position96, tokenIndex96 := position, tokenIndex if buffer[position] != rune('.') { - goto l77 + goto l96 } position++ - l79: + l98: { - position80, tokenIndex80 := position, tokenIndex + position99, tokenIndex99 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l80 + goto l99 } position++ - goto l79 - l80: - position, tokenIndex = position80, tokenIndex80 + goto l98 + l99: + position, tokenIndex = position99, tokenIndex99 } - goto l78 - l77: - position, tokenIndex = position77, tokenIndex77 + goto l97 + l96: + position, tokenIndex = position96, tokenIndex96 } - l78: - add(rulePegText, position72) + l97: + add(rulePegText, position91) } { add(ruleAction14, position) } - goto l64 - l71: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l90: + position, tokenIndex = position74, tokenIndex74 { - position83 := position + position102 := position { - position84, tokenIndex84 := position, tokenIndex + position103, tokenIndex103 := position, tokenIndex if buffer[position] != rune('-') { - goto l84 + goto l103 } position++ - goto l85 - l84: - position, tokenIndex = position84, tokenIndex84 + goto l104 + l103: + position, tokenIndex = position103, tokenIndex103 } - l85: + l104: if buffer[position] != rune('.') { - goto l82 + goto l101 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l82 + goto l101 } position++ - l86: + l105: { - position87, tokenIndex87 := position, tokenIndex + position106, tokenIndex106 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l87 + goto l106 } position++ - goto l86 - l87: - position, tokenIndex = position87, tokenIndex87 + goto l105 + l106: + position, tokenIndex = position106, tokenIndex106 } - add(rulePegText, position83) + add(rulePegText, position102) } { add(ruleAction15, position) } - goto l64 - l82: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l101: + position, tokenIndex = position74, tokenIndex74 { - position90 := position + position109 := position { - position93, tokenIndex93 := position, tokenIndex + position112, tokenIndex112 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l94 + goto l113 } position++ - goto l93 - l94: - position, tokenIndex = position93, tokenIndex93 + goto l112 + l113: + position, tokenIndex = position112, tokenIndex112 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l95 + goto l114 } position++ - goto l93 - l95: - position, tokenIndex = position93, tokenIndex93 + goto l112 + l114: + position, tokenIndex = position112, tokenIndex112 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l96 + goto l115 } position++ - goto l93 - l96: - position, tokenIndex = position93, tokenIndex93 + goto l112 + l115: + position, tokenIndex = position112, tokenIndex112 if buffer[position] != rune('-') { - goto l97 + goto l116 } position++ - goto l93 - l97: - position, tokenIndex = position93, tokenIndex93 + goto l112 + l116: + position, tokenIndex = position112, tokenIndex112 if buffer[position] != rune('_') { - goto l98 + goto l117 } position++ - goto l93 - l98: - position, tokenIndex = position93, tokenIndex93 + goto l112 + l117: + position, tokenIndex = position112, tokenIndex112 if buffer[position] != rune(':') { - goto l89 + goto l108 } position++ } - l93: - l91: + l112: + l110: { - position92, tokenIndex92 := position, tokenIndex + position111, tokenIndex111 := position, tokenIndex { - position99, tokenIndex99 := position, tokenIndex + position118, tokenIndex118 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l100 + goto l119 } position++ - goto l99 - l100: - position, tokenIndex = position99, tokenIndex99 + goto l118 + l119: + position, tokenIndex = position118, tokenIndex118 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l101 + goto l120 } position++ - goto l99 - l101: - position, tokenIndex = position99, tokenIndex99 + goto l118 + l120: + position, tokenIndex = position118, tokenIndex118 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l102 + goto l121 } position++ - goto l99 - l102: - position, tokenIndex = position99, tokenIndex99 + goto l118 + l121: + position, tokenIndex = position118, tokenIndex118 if buffer[position] != rune('-') { - goto l103 + goto l122 } position++ - goto l99 - l103: - position, tokenIndex = position99, tokenIndex99 + goto l118 + l122: + position, tokenIndex = position118, tokenIndex118 if buffer[position] != rune('_') { - goto l104 + goto l123 } position++ - goto l99 - l104: - position, tokenIndex = position99, tokenIndex99 + goto l118 + l123: + position, tokenIndex = position118, tokenIndex118 if buffer[position] != rune(':') { - goto l92 + goto l111 } position++ } - l99: - goto l91 - l92: - position, tokenIndex = position92, tokenIndex92 + l118: + goto l110 + l111: + position, tokenIndex = position111, tokenIndex111 } - add(rulePegText, position90) + add(rulePegText, position109) } { add(ruleAction16, position) } - goto l64 - l89: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l108: + position, tokenIndex = position74, tokenIndex74 if buffer[position] != rune('"') { - goto l106 + goto l125 } position++ { - position107 := position + position126 := position { - position108 := position - l109: + position127 := position + l128: { - position110, tokenIndex110 := position, tokenIndex + position129, tokenIndex129 := position, tokenIndex { - position111, tokenIndex111 := position, tokenIndex + position130, tokenIndex130 := position, tokenIndex { - position113, tokenIndex113 := position, tokenIndex + position132, tokenIndex132 := position, tokenIndex { - position114, tokenIndex114 := position, tokenIndex + position133, tokenIndex133 := position, tokenIndex if buffer[position] != rune('"') { - goto l115 + goto l134 } position++ - goto l114 - l115: - position, tokenIndex = position114, tokenIndex114 + goto l133 + l134: + position, tokenIndex = position133, tokenIndex133 if buffer[position] != rune('\\') { - goto l116 + goto l135 } position++ - goto l114 - l116: - position, tokenIndex = position114, tokenIndex114 + goto l133 + l135: + position, tokenIndex = position133, tokenIndex133 if buffer[position] != rune('\n') { - goto l113 + goto l132 } position++ } - l114: - goto l112 - l113: - position, tokenIndex = position113, tokenIndex113 + l133: + goto l131 + l132: + position, tokenIndex = position132, tokenIndex132 } if !matchDot() { - goto l112 + goto l131 } - goto l111 - l112: - position, tokenIndex = position111, tokenIndex111 + goto l130 + l131: + position, tokenIndex = position130, tokenIndex130 if buffer[position] != rune('\\') { - goto l117 + goto l136 } position++ if buffer[position] != rune('n') { - goto l117 + goto l136 } position++ - goto l111 - l117: - position, tokenIndex = position111, tokenIndex111 + goto l130 + l136: + position, tokenIndex = position130, tokenIndex130 if buffer[position] != rune('\\') { - goto l118 + goto l137 } position++ if buffer[position] != rune('"') { - goto l118 + goto l137 } position++ - goto l111 - l118: - position, tokenIndex = position111, tokenIndex111 + goto l130 + l137: + position, tokenIndex = position130, tokenIndex130 if buffer[position] != rune('\\') { - goto l119 + goto l138 } position++ if buffer[position] != rune('\'') { - goto l119 + goto l138 } position++ - goto l111 - l119: - position, tokenIndex = position111, tokenIndex111 + goto l130 + l138: + position, tokenIndex = position130, tokenIndex130 if buffer[position] != rune('\\') { - goto l110 + goto l129 } position++ if buffer[position] != rune('\\') { - goto l110 + goto l129 } position++ } - l111: - goto l109 - l110: - position, tokenIndex = position110, tokenIndex110 + l130: + goto l128 + l129: + position, tokenIndex = position129, tokenIndex129 } - add(ruledoublequotedstring, position108) + add(ruledoublequotedstring, position127) } - add(rulePegText, position107) + add(rulePegText, position126) } if buffer[position] != rune('"') { - goto l106 + goto l125 } position++ { add(ruleAction17, position) } - goto l64 - l106: - position, tokenIndex = position64, tokenIndex64 + goto l74 + l125: + position, tokenIndex = position74, tokenIndex74 if buffer[position] != rune('\'') { - goto l62 + goto l72 } position++ { - position121 := position + position140 := position { - position122 := position - l123: + position141 := position + l142: { - position124, tokenIndex124 := position, tokenIndex + position143, tokenIndex143 := position, tokenIndex { - position125, tokenIndex125 := position, tokenIndex + position144, tokenIndex144 := position, tokenIndex { - position127, tokenIndex127 := position, tokenIndex + position146, tokenIndex146 := position, tokenIndex { - position128, tokenIndex128 := position, tokenIndex + position147, tokenIndex147 := position, tokenIndex if buffer[position] != rune('\'') { - goto l129 + goto l148 } position++ - goto l128 - l129: - position, tokenIndex = position128, tokenIndex128 + goto l147 + l148: + position, tokenIndex = position147, tokenIndex147 if buffer[position] != rune('\\') { - goto l130 + goto l149 } position++ - goto l128 - l130: - position, tokenIndex = position128, tokenIndex128 + goto l147 + l149: + position, tokenIndex = position147, tokenIndex147 if buffer[position] != rune('\n') { - goto l127 + goto l146 } position++ } - l128: - goto l126 - l127: - position, tokenIndex = position127, tokenIndex127 + l147: + goto l145 + l146: + position, tokenIndex = position146, tokenIndex146 } if !matchDot() { - goto l126 + goto l145 } - goto l125 - l126: - position, tokenIndex = position125, tokenIndex125 + goto l144 + l145: + position, tokenIndex = position144, tokenIndex144 if buffer[position] != rune('\\') { - goto l131 + goto l150 } position++ if buffer[position] != rune('n') { - goto l131 + goto l150 } position++ - goto l125 - l131: - position, tokenIndex = position125, tokenIndex125 + goto l144 + l150: + position, tokenIndex = position144, tokenIndex144 if buffer[position] != rune('\\') { - goto l132 + goto l151 } position++ if buffer[position] != rune('"') { - goto l132 + goto l151 } position++ - goto l125 - l132: - position, tokenIndex = position125, tokenIndex125 + goto l144 + l151: + position, tokenIndex = position144, tokenIndex144 if buffer[position] != rune('\\') { - goto l133 + goto l152 } position++ if buffer[position] != rune('\'') { - goto l133 + goto l152 } position++ - goto l125 - l133: - position, tokenIndex = position125, tokenIndex125 + goto l144 + l152: + position, tokenIndex = position144, tokenIndex144 if buffer[position] != rune('\\') { - goto l124 + goto l143 } position++ if buffer[position] != rune('\\') { - goto l124 + goto l143 } position++ } - l125: - goto l123 - l124: - position, tokenIndex = position124, tokenIndex124 + l144: + goto l142 + l143: + position, tokenIndex = position143, tokenIndex143 } - add(rulesinglequotedstring, position122) + add(rulesinglequotedstring, position141) } - add(rulePegText, position121) + add(rulePegText, position140) } if buffer[position] != rune('\'') { - goto l62 + goto l72 } position++ { add(ruleAction18, position) } } - l64: - add(ruleitem, position63) + l74: + add(ruleitem, position73) } return true - l62: - position, tokenIndex = position62, tokenIndex62 + l72: + position, tokenIndex = position72, tokenIndex72 return false }, - /* 9 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 10 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 10 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 11 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 11 field <- <(<(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> Action19)> */ - func() bool { - position137, tokenIndex137 := position, tokenIndex - { - position138 := position - { - position139 := position - { - position140, tokenIndex140 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l141 - } - position++ - goto l140 - l141: - position, tokenIndex = position140, tokenIndex140 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l137 - } - position++ - } - l140: - l142: - { - position143, tokenIndex143 := position, tokenIndex - { - position144, tokenIndex144 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l145 - } - position++ - goto l144 - l145: - position, tokenIndex = position144, tokenIndex144 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l146 - } - position++ - goto l144 - l146: - position, tokenIndex = position144, tokenIndex144 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l147 - } - position++ - goto l144 - l147: - position, tokenIndex = position144, tokenIndex144 - if buffer[position] != rune('_') { - goto l143 - } - position++ - } - l144: - goto l142 - l143: - position, tokenIndex = position143, tokenIndex143 - } - add(rulePegText, position139) - } - { - add(ruleAction19, position) - } - add(rulefield, position138) - } - return true - l137: - position, tokenIndex = position137, tokenIndex137 - return false - }, - /* 12 close <- <(')' sp)> */ - nil, - /* 13 sp <- <(' ' / '\t')*> */ - func() bool { - { - position151 := position - l152: - { - position153, tokenIndex153 := position, tokenIndex - { - position154, tokenIndex154 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l155 - } - position++ - goto l154 - l155: - position, tokenIndex = position154, tokenIndex154 - if buffer[position] != rune('\t') { - goto l153 - } - position++ - } - l154: - goto l152 - l153: - position, tokenIndex = position153, tokenIndex153 - } - add(rulesp, position151) - } - return true - }, - /* 14 comma <- <(sp ',' sp)> */ + /* 12 field <- <(<(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> Action19)> */ func() bool { position156, tokenIndex156 := position, tokenIndex { position157 := position - if !_rules[rulesp]() { - goto l156 + { + position158 := position + { + position159, tokenIndex159 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l160 + } + position++ + goto l159 + l160: + position, tokenIndex = position159, tokenIndex159 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l156 + } + position++ + } + l159: + l161: + { + position162, tokenIndex162 := position, tokenIndex + { + position163, tokenIndex163 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l164 + } + position++ + goto l163 + l164: + position, tokenIndex = position163, tokenIndex163 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l165 + } + position++ + goto l163 + l165: + position, tokenIndex = position163, tokenIndex163 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l166 + } + position++ + goto l163 + l166: + position, tokenIndex = position163, tokenIndex163 + if buffer[position] != rune('_') { + goto l162 + } + position++ + } + l163: + goto l161 + l162: + position, tokenIndex = position162, tokenIndex162 + } + add(rulePegText, position158) } - if buffer[position] != rune(',') { - goto l156 + { + add(ruleAction19, position) } - position++ - if !_rules[rulesp]() { - goto l156 - } - add(rulecomma, position157) + add(rulefield, position157) } return true l156: position, tokenIndex = position156, tokenIndex156 return false }, - /* 15 lbrack <- <('[' sp)> */ - nil, - /* 16 rbrack <- <(sp ']' sp)> */ - nil, - /* 17 newline <- <(sp '\n' sp)> */ + /* 13 close <- <(')' sp)> */ func() bool { - position160, tokenIndex160 := position, tokenIndex + position168, tokenIndex168 := position, tokenIndex { - position161 := position - if !_rules[rulesp]() { - goto l160 - } - if buffer[position] != rune('\n') { - goto l160 + position169 := position + if buffer[position] != rune(')') { + goto l168 } position++ if !_rules[rulesp]() { - goto l160 + goto l168 } - add(rulenewline, position161) + add(ruleclose, position169) } return true - l160: - position, tokenIndex = position160, tokenIndex160 + l168: + position, tokenIndex = position168, tokenIndex168 return false }, + /* 14 sp <- <(' ' / '\t')*> */ + func() bool { + { + position171 := position + l172: + { + position173, tokenIndex173 := position, tokenIndex + { + position174, tokenIndex174 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l175 + } + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('\t') { + goto l173 + } + position++ + } + l174: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + add(rulesp, position171) + } + return true + }, + /* 15 comma <- <(sp ',' sp)> */ + func() bool { + position176, tokenIndex176 := position, tokenIndex + { + position177 := position + if !_rules[rulesp]() { + goto l176 + } + if buffer[position] != rune(',') { + goto l176 + } + position++ + if !_rules[rulesp]() { + goto l176 + } + add(rulecomma, position177) + } + return true + l176: + position, tokenIndex = position176, tokenIndex176 + return false + }, + /* 16 lbrack <- <('[' sp)> */ nil, - /* 20 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 17 rbrack <- <(sp ']' sp)> */ nil, - /* 21 Action1 <- <{ p.endCall() }> */ + /* 18 whitesp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position181 := position + l182: + { + position183, tokenIndex183 := position, tokenIndex + { + position184, tokenIndex184 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l185 + } + position++ + goto l184 + l185: + position, tokenIndex = position184, tokenIndex184 + if buffer[position] != rune('\t') { + goto l186 + } + position++ + goto l184 + l186: + position, tokenIndex = position184, tokenIndex184 + if buffer[position] != rune('\n') { + goto l183 + } + position++ + } + l184: + goto l182 + l183: + position, tokenIndex = position183, tokenIndex183 + } + add(rulewhitesp, position181) + } + return true + }, + /* 19 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '-' / '_' / '.')*)> */ nil, - /* 22 Action2 <- <{ p.addBTWN() }> */ nil, - /* 23 Action3 <- <{ p.addLTE() }> */ + /* 22 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 24 Action4 <- <{ p.addGTE() }> */ + /* 23 Action1 <- <{ p.endCall() }> */ nil, - /* 25 Action5 <- <{ p.addEQ() }> */ + /* 24 Action2 <- <{ p.addBTWN() }> */ nil, - /* 26 Action6 <- <{ p.addNEQ() }> */ + /* 25 Action3 <- <{ p.addLTE() }> */ nil, - /* 27 Action7 <- <{ p.addLT() }> */ + /* 26 Action4 <- <{ p.addGTE() }> */ nil, - /* 28 Action8 <- <{ p.addGT() }> */ + /* 27 Action5 <- <{ p.addEQ() }> */ nil, - /* 29 Action9 <- <{ p.startList() }> */ + /* 28 Action6 <- <{ p.addNEQ() }> */ nil, - /* 30 Action10 <- <{ p.endList() }> */ + /* 29 Action7 <- <{ p.addLT() }> */ nil, - /* 31 Action11 <- <{ p.addVal(nil) }> */ + /* 30 Action8 <- <{ p.addGT() }> */ nil, - /* 32 Action12 <- <{ p.addVal(true) }> */ + /* 31 Action9 <- <{ p.startList() }> */ nil, - /* 33 Action13 <- <{ p.addVal(false) }> */ + /* 32 Action10 <- <{ p.endList() }> */ nil, - /* 34 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 33 Action11 <- <{ p.addVal(nil) }> */ nil, - /* 35 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 34 Action12 <- <{ p.addVal(true) }> */ nil, - /* 36 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 35 Action13 <- <{ p.addVal(false) }> */ nil, - /* 37 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 36 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 38 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 37 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 39 Action19 <- <{ p.addField(buffer[begin:end]) }> */ + /* 38 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 39 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 40 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 41 Action19 <- <{ p.addField(buffer[begin:end]) }> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index a38d451d0..2288c3aeb 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -20,4 +20,27 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 if err == nil { t.Fatalf("should have been an error because of the interior unescaped double quote") } + + q, err := ParseString("TopN(Bitmap(id==other), field=f, n=0)") + if err != nil { + t.Fatalf("should have parsed: %v", err) + } + if q.String() != `TopN(Bitmap(id == "other"), field="f", n=0)` { + t.Fatalf("Failed, got: %s", q) + } + + q, err = ParseString("C(a=falsen0)") + if err != nil { + t.Fatalf("falsen0 should have been parsed as a string") + } + + q, err = ParseString("Bitmap(row=4, did==other)") + if err != nil { + t.Fatalf("should have parsed: %v", err) + } + + if q.String() != `Bitmap(did == "other", row=4)` { + t.Fatalf("got %s", q) + } + } From 4b856bea55a4dd32964a66bbc9da506510d778ea Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 15 Jun 2018 12:22:16 -0500 Subject: [PATCH 06/33] change parser for new PQL --- pql/ast.go | 43 +- pql/pql.peg | 26 +- pql/pql.peg.go | 2896 ++++++++++++++++++++++++++++++------------------ 3 files changed, 1908 insertions(+), 1057 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 6d1b206b0..0be4d7034 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -31,6 +31,8 @@ type Query struct { lastCond Token inList bool callStack []*Call + + conditional []string } func (q *Query) startCall(name string) { @@ -43,13 +45,52 @@ func (q *Query) startCall(name string) { calls := q.callStack[len(q.callStack)-2].Children q.callStack[len(q.callStack)-2].Children = append(calls, newCall) } - } func (q *Query) endCall() { q.callStack = q.callStack[:len(q.callStack)-1] } +func (q *Query) addPosNum(key, value string) { + q.addField(key) + q.addNumVal(value) +} + +func (q *Query) addPosStr(key, value string) { + q.addField(key) + q.addVal(value) +} + +func (q *Query) startConditional() { + q.conditional = make([]string, 0) +} + +func (q *Query) condAdd(val string) { + q.conditional = append(q.conditional, val) +} + +func (q *Query) endConditional() { + // do stuff + if len(q.conditional) != 5 { + panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional)) + } + low, _ := strconv.ParseInt(q.conditional[0], 10, 64) + field := q.conditional[2] + high, _ := strconv.ParseInt(q.conditional[4], 10, 64) + + if q.conditional[1] == "<" { + low++ + } + if q.conditional[3] == "<=" { + high++ + } + + call := q.callStack[len(q.callStack)-1] + call.Args[field] = Condition{Op: BETWEEN, Value: []interface{}{low, high}} + + q.conditional = nil +} + func (q *Query) addField(field string) { if q.lastField != "" { panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) diff --git a/pql/pql.peg b/pql/pql.peg index 3602f8cc1..f288dadf6 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -5,8 +5,14 @@ type PQL Peg { } -Calls <- Call* !. -Call <- whitesp < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close whitesp { p.endCall() } +Calls <- whitesp (Call whitesp)* !. +Call <- 'Set' {p.startCall("Set")} open uintcol comma args (comma timestamp)? close {p.endCall()} + / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} + / 'SetColAttrs' {p.startCall("SetColAttrs")} open posfield comma uintcol comma args close {p.endCall()} + / 'ClearBit' {p.startCall("ClearBit")} open uintcol comma args close {p.endCall()} + / 'TopN' {p.startCall("TopN")} open posfield (comma args)? close {p.endCall()} + / 'Range' {p.startCall("Range")} open (arg / conditional) close {p.endCall()} + / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / comma? args / sp args <- arg (comma args)? sp arg <- ( field sp '=' sp value @@ -20,6 +26,7 @@ COND <- ( '><' { p.addBTWN() } / '<' { p.addLT() } / '>' { p.addGT() } ) +conditional <- {p.startConditional()} int ('<=' / '<') fieldExpr ('<=' / '<') int {p.endConditional()} open <- '(' sp value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } @@ -38,11 +45,20 @@ item <- ( 'null' &(comma / sp close) { p.addVal(nil) } doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* -field <- < [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* > { p.addField(buffer[begin:end]) } +fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* +field <- { p.addField(buffer[begin:end]) } +posfield <- { p.addPosStr("_field", buffer[begin:end]) } +uint <- [1-9] [0-9]* / '0' +int <- '-'? [1-9] [0-9]* / '0' +uintrow <- {p.addPosNum("_row", buffer[begin:end])} +uintcol <- {p.addPosNum("_col", buffer[begin:end])} + close <- ')' sp sp <- ( ' ' / '\t' )* -comma <- sp ',' sp +comma <- sp ',' whitesp lbrack <- '[' sp rbrack <- sp ']' sp whitesp <- ( ' ' / '\t' / '\n' )* -IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / '-' / '_' / '.')* \ No newline at end of file +IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / '-' / '_' / '.')* + +timestamp <- <[0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]> {p.addPosStr("_timestamp", buffer[begin:end])} \ No newline at end of file diff --git a/pql/pql.peg.go b/pql/pql.peg.go index e4680d613..b6dc82f5e 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -22,13 +22,20 @@ const ( ruleargs rulearg ruleCOND + ruleconditional ruleopen rulevalue rulelist ruleitem ruledoublequotedstring rulesinglequotedstring + rulefieldExpr rulefield + ruleposfield + ruleuint + ruleint + ruleuintrow + ruleuintcol ruleclose rulesp rulecomma @@ -36,7 +43,7 @@ const ( rulerbrack rulewhitesp ruleIDENT - rulePegText + ruletimestamp ruleAction0 ruleAction1 ruleAction2 @@ -49,6 +56,7 @@ const ( ruleAction9 ruleAction10 ruleAction11 + rulePegText ruleAction12 ruleAction13 ruleAction14 @@ -57,6 +65,24 @@ const ( ruleAction17 ruleAction18 ruleAction19 + ruleAction20 + ruleAction21 + ruleAction22 + ruleAction23 + ruleAction24 + ruleAction25 + ruleAction26 + ruleAction27 + ruleAction28 + ruleAction29 + ruleAction30 + ruleAction31 + ruleAction32 + ruleAction33 + ruleAction34 + ruleAction35 + ruleAction36 + ruleAction37 ) var rul3s = [...]string{ @@ -67,13 +93,20 @@ var rul3s = [...]string{ "args", "arg", "COND", + "conditional", "open", "value", "list", "item", "doublequotedstring", "singlequotedstring", + "fieldExpr", "field", + "posfield", + "uint", + "int", + "uintrow", + "uintcol", "close", "sp", "comma", @@ -81,7 +114,7 @@ var rul3s = [...]string{ "rbrack", "whitesp", "IDENT", - "PegText", + "timestamp", "Action0", "Action1", "Action2", @@ -94,6 +127,7 @@ var rul3s = [...]string{ "Action9", "Action10", "Action11", + "PegText", "Action12", "Action13", "Action14", @@ -102,6 +136,24 @@ var rul3s = [...]string{ "Action17", "Action18", "Action19", + "Action20", + "Action21", + "Action22", + "Action23", + "Action24", + "Action25", + "Action26", + "Action27", + "Action28", + "Action29", + "Action30", + "Action31", + "Action32", + "Action33", + "Action34", + "Action35", + "Action36", + "Action37", } type token32 struct { @@ -218,7 +270,7 @@ type PQL struct { Buffer string buffer []rune - rules [42]func() bool + rules [68]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -311,45 +363,81 @@ func (p *PQL) Execute() { text = string(_buffer[begin:end]) case ruleAction0: - p.startCall(buffer[begin:end]) + p.startCall("Set") case ruleAction1: p.endCall() case ruleAction2: - p.addBTWN() + p.startCall("SetRowAttrs") case ruleAction3: - p.addLTE() + p.endCall() case ruleAction4: - p.addGTE() + p.startCall("SetColAttrs") case ruleAction5: - p.addEQ() + p.endCall() case ruleAction6: - p.addNEQ() + p.startCall("ClearBit") case ruleAction7: - p.addLT() + p.endCall() case ruleAction8: - p.addGT() + p.startCall("TopN") case ruleAction9: - p.startList() + p.endCall() case ruleAction10: - p.endList() + p.startCall("Range") case ruleAction11: - p.addVal(nil) + p.endCall() case ruleAction12: - p.addVal(true) + p.startCall(buffer[begin:end]) case ruleAction13: - p.addVal(false) + p.endCall() case ruleAction14: - p.addNumVal(buffer[begin:end]) + p.addBTWN() case ruleAction15: - p.addNumVal(buffer[begin:end]) + p.addLTE() case ruleAction16: - p.addVal(buffer[begin:end]) + p.addGTE() case ruleAction17: - p.addVal(buffer[begin:end]) + p.addEQ() case ruleAction18: - p.addVal(buffer[begin:end]) + p.addNEQ() case ruleAction19: + p.addLT() + case ruleAction20: + p.addGT() + case ruleAction21: + p.startConditional() + case ruleAction22: + p.endConditional() + case ruleAction23: + p.startList() + case ruleAction24: + p.endList() + case ruleAction25: + p.addVal(nil) + case ruleAction26: + p.addVal(true) + case ruleAction27: + p.addVal(false) + case ruleAction28: + p.addNumVal(buffer[begin:end]) + case ruleAction29: + p.addNumVal(buffer[begin:end]) + case ruleAction30: + p.addVal(buffer[begin:end]) + case ruleAction31: + p.addVal(buffer[begin:end]) + case ruleAction32: + p.addVal(buffer[begin:end]) + case ruleAction33: p.addField(buffer[begin:end]) + case ruleAction34: + p.addPosStr("_field", buffer[begin:end]) + case ruleAction35: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction36: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction37: + p.addPosStr("_timestamp", buffer[begin:end]) } } @@ -424,17 +512,23 @@ func (p *PQL) Init() { _rules = [...]func() bool{ nil, - /* 0 Calls <- <(Call* !.)> */ + /* 0 Calls <- <(whitesp (Call whitesp)* !.)> */ func() bool { position0, tokenIndex0 := position, tokenIndex { position1 := position + if !_rules[rulewhitesp]() { + goto l0 + } l2: { position3, tokenIndex3 := position, tokenIndex if !_rules[ruleCall]() { goto l3 } + if !_rules[rulewhitesp]() { + goto l3 + } goto l2 l3: position, tokenIndex = position3, tokenIndex3 @@ -455,179 +549,665 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(whitesp Action0 open allargs comma? close whitesp Action1)> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' 'B' 'i' 't' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma args)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (arg / conditional) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { position6 := position - if !_rules[rulewhitesp]() { - goto l5 - } { - position7 := position - { - position8 := position - { - position9, tokenIndex9 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l10 - } - position++ - goto l9 - l10: - position, tokenIndex = position9, tokenIndex9 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l5 - } - position++ - } - l9: - l11: - { - position12, tokenIndex12 := position, tokenIndex - { - position13, tokenIndex13 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l14 - } - position++ - goto l13 - l14: - position, tokenIndex = position13, tokenIndex13 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l15 - } - position++ - goto l13 - l15: - position, tokenIndex = position13, tokenIndex13 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l16 - } - position++ - goto l13 - l16: - position, tokenIndex = position13, tokenIndex13 - if buffer[position] != rune('-') { - goto l17 - } - position++ - goto l13 - l17: - position, tokenIndex = position13, tokenIndex13 - if buffer[position] != rune('_') { - goto l18 - } - position++ - goto l13 - l18: - position, tokenIndex = position13, tokenIndex13 - if buffer[position] != rune('.') { - goto l12 - } - position++ - } - l13: - goto l11 - l12: - position, tokenIndex = position12, tokenIndex12 - } - add(ruleIDENT, position8) - } - add(rulePegText, position7) - } - { - add(ruleAction0, position) - } - { - position20 := position - if buffer[position] != rune('(') { - goto l5 + position7, tokenIndex7 := position, tokenIndex + if buffer[position] != rune('S') { + goto l8 } position++ - if !_rules[rulesp]() { + if buffer[position] != rune('e') { + goto l8 + } + position++ + if buffer[position] != rune('t') { + goto l8 + } + position++ + { + add(ruleAction0, position) + } + if !_rules[ruleopen]() { + goto l8 + } + if !_rules[ruleuintcol]() { + goto l8 + } + if !_rules[rulecomma]() { + goto l8 + } + if !_rules[ruleargs]() { + goto l8 + } + { + position10, tokenIndex10 := position, tokenIndex + if !_rules[rulecomma]() { + goto l10 + } + { + position12 := position + { + position13 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if buffer[position] != rune('-') { + goto l10 + } + position++ + { + position14, tokenIndex14 := position, tokenIndex + if buffer[position] != rune('0') { + goto l15 + } + position++ + goto l14 + l15: + position, tokenIndex = position14, tokenIndex14 + if buffer[position] != rune('1') { + goto l10 + } + position++ + } + l14: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if buffer[position] != rune('-') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if buffer[position] != rune('T') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if buffer[position] != rune(':') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l10 + } + position++ + add(rulePegText, position13) + } + { + add(ruleAction37, position) + } + add(ruletimestamp, position12) + } + goto l11 + l10: + position, tokenIndex = position10, tokenIndex10 + } + l11: + if !_rules[ruleclose]() { + goto l8 + } + { + add(ruleAction1, position) + } + goto l7 + l8: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('S') { + goto l18 + } + position++ + if buffer[position] != rune('e') { + goto l18 + } + position++ + if buffer[position] != rune('t') { + goto l18 + } + position++ + if buffer[position] != rune('R') { + goto l18 + } + position++ + if buffer[position] != rune('o') { + goto l18 + } + position++ + if buffer[position] != rune('w') { + goto l18 + } + position++ + if buffer[position] != rune('A') { + goto l18 + } + position++ + if buffer[position] != rune('t') { + goto l18 + } + position++ + if buffer[position] != rune('t') { + goto l18 + } + position++ + if buffer[position] != rune('r') { + goto l18 + } + position++ + if buffer[position] != rune('s') { + goto l18 + } + position++ + { + add(ruleAction2, position) + } + if !_rules[ruleopen]() { + goto l18 + } + if !_rules[ruleposfield]() { + goto l18 + } + if !_rules[rulecomma]() { + goto l18 + } + { + position20 := position + { + position21 := position + if !_rules[ruleuint]() { + goto l18 + } + add(rulePegText, position21) + } + { + add(ruleAction35, position) + } + add(ruleuintrow, position20) + } + if !_rules[rulecomma]() { + goto l18 + } + if !_rules[ruleargs]() { + goto l18 + } + if !_rules[ruleclose]() { + goto l18 + } + { + add(ruleAction3, position) + } + goto l7 + l18: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('S') { + goto l24 + } + position++ + if buffer[position] != rune('e') { + goto l24 + } + position++ + if buffer[position] != rune('t') { + goto l24 + } + position++ + if buffer[position] != rune('C') { + goto l24 + } + position++ + if buffer[position] != rune('o') { + goto l24 + } + position++ + if buffer[position] != rune('l') { + goto l24 + } + position++ + if buffer[position] != rune('A') { + goto l24 + } + position++ + if buffer[position] != rune('t') { + goto l24 + } + position++ + if buffer[position] != rune('t') { + goto l24 + } + position++ + if buffer[position] != rune('r') { + goto l24 + } + position++ + if buffer[position] != rune('s') { + goto l24 + } + position++ + { + add(ruleAction4, position) + } + if !_rules[ruleopen]() { + goto l24 + } + if !_rules[ruleposfield]() { + goto l24 + } + if !_rules[rulecomma]() { + goto l24 + } + if !_rules[ruleuintcol]() { + goto l24 + } + if !_rules[rulecomma]() { + goto l24 + } + if !_rules[ruleargs]() { + goto l24 + } + if !_rules[ruleclose]() { + goto l24 + } + { + add(ruleAction5, position) + } + goto l7 + l24: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('C') { + goto l27 + } + position++ + if buffer[position] != rune('l') { + goto l27 + } + position++ + if buffer[position] != rune('e') { + goto l27 + } + position++ + if buffer[position] != rune('a') { + goto l27 + } + position++ + if buffer[position] != rune('r') { + goto l27 + } + position++ + if buffer[position] != rune('B') { + goto l27 + } + position++ + if buffer[position] != rune('i') { + goto l27 + } + position++ + if buffer[position] != rune('t') { + goto l27 + } + position++ + { + add(ruleAction6, position) + } + if !_rules[ruleopen]() { + goto l27 + } + if !_rules[ruleuintcol]() { + goto l27 + } + if !_rules[rulecomma]() { + goto l27 + } + if !_rules[ruleargs]() { + goto l27 + } + if !_rules[ruleclose]() { + goto l27 + } + { + add(ruleAction7, position) + } + goto l7 + l27: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('T') { + goto l30 + } + position++ + if buffer[position] != rune('o') { + goto l30 + } + position++ + if buffer[position] != rune('p') { + goto l30 + } + position++ + if buffer[position] != rune('N') { + goto l30 + } + position++ + { + add(ruleAction8, position) + } + if !_rules[ruleopen]() { + goto l30 + } + if !_rules[ruleposfield]() { + goto l30 + } + { + position32, tokenIndex32 := position, tokenIndex + if !_rules[rulecomma]() { + goto l32 + } + if !_rules[ruleargs]() { + goto l32 + } + goto l33 + l32: + position, tokenIndex = position32, tokenIndex32 + } + l33: + if !_rules[ruleclose]() { + goto l30 + } + { + add(ruleAction9, position) + } + goto l7 + l30: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l35 + } + position++ + if buffer[position] != rune('a') { + goto l35 + } + position++ + if buffer[position] != rune('n') { + goto l35 + } + position++ + if buffer[position] != rune('g') { + goto l35 + } + position++ + if buffer[position] != rune('e') { + goto l35 + } + position++ + { + add(ruleAction10, position) + } + if !_rules[ruleopen]() { + goto l35 + } + { + position37, tokenIndex37 := position, tokenIndex + if !_rules[rulearg]() { + goto l38 + } + goto l37 + l38: + position, tokenIndex = position37, tokenIndex37 + { + position39 := position + { + add(ruleAction21, position) + } + if !_rules[ruleint]() { + goto l35 + } + { + position41, tokenIndex41 := position, tokenIndex + if buffer[position] != rune('<') { + goto l42 + } + position++ + if buffer[position] != rune('=') { + goto l42 + } + position++ + goto l41 + l42: + position, tokenIndex = position41, tokenIndex41 + if buffer[position] != rune('<') { + goto l35 + } + position++ + } + l41: + if !_rules[rulefieldExpr]() { + goto l35 + } + { + position43, tokenIndex43 := position, tokenIndex + if buffer[position] != rune('<') { + goto l44 + } + position++ + if buffer[position] != rune('=') { + goto l44 + } + position++ + goto l43 + l44: + position, tokenIndex = position43, tokenIndex43 + if buffer[position] != rune('<') { + goto l35 + } + position++ + } + l43: + if !_rules[ruleint]() { + goto l35 + } + { + add(ruleAction22, position) + } + add(ruleconditional, position39) + } + } + l37: + if !_rules[ruleclose]() { + goto l35 + } + { + add(ruleAction11, position) + } + goto l7 + l35: + position, tokenIndex = position7, tokenIndex7 + { + position47 := position + { + position48 := position + { + position49, tokenIndex49 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l50 + } + position++ + goto l49 + l50: + position, tokenIndex = position49, tokenIndex49 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l5 + } + position++ + } + l49: + l51: + { + position52, tokenIndex52 := position, tokenIndex + { + position53, tokenIndex53 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l54 + } + position++ + goto l53 + l54: + position, tokenIndex = position53, tokenIndex53 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l55 + } + position++ + goto l53 + l55: + position, tokenIndex = position53, tokenIndex53 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l56 + } + position++ + goto l53 + l56: + position, tokenIndex = position53, tokenIndex53 + if buffer[position] != rune('-') { + goto l57 + } + position++ + goto l53 + l57: + position, tokenIndex = position53, tokenIndex53 + if buffer[position] != rune('_') { + goto l58 + } + position++ + goto l53 + l58: + position, tokenIndex = position53, tokenIndex53 + if buffer[position] != rune('.') { + goto l52 + } + position++ + } + l53: + goto l51 + l52: + position, tokenIndex = position52, tokenIndex52 + } + add(ruleIDENT, position48) + } + add(rulePegText, position47) + } + { + add(ruleAction12, position) + } + if !_rules[ruleopen]() { goto l5 } - add(ruleopen, position20) - } - { - position21 := position { - position22, tokenIndex22 := position, tokenIndex - if !_rules[ruleCall]() { - goto l23 - } - l24: + position60 := position { - position25, tokenIndex25 := position, tokenIndex - if !_rules[rulecomma]() { - goto l25 - } + position61, tokenIndex61 := position, tokenIndex if !_rules[ruleCall]() { - goto l25 + goto l62 } - goto l24 - l25: - position, tokenIndex = position25, tokenIndex25 - } - { - position26, tokenIndex26 := position, tokenIndex - if !_rules[rulecomma]() { - goto l26 + l63: + { + position64, tokenIndex64 := position, tokenIndex + if !_rules[rulecomma]() { + goto l64 + } + if !_rules[ruleCall]() { + goto l64 + } + goto l63 + l64: + position, tokenIndex = position64, tokenIndex64 } + { + position65, tokenIndex65 := position, tokenIndex + if !_rules[rulecomma]() { + goto l65 + } + if !_rules[ruleargs]() { + goto l65 + } + goto l66 + l65: + position, tokenIndex = position65, tokenIndex65 + } + l66: + goto l61 + l62: + position, tokenIndex = position61, tokenIndex61 + { + position68, tokenIndex68 := position, tokenIndex + if !_rules[rulecomma]() { + goto l68 + } + goto l69 + l68: + position, tokenIndex = position68, tokenIndex68 + } + l69: if !_rules[ruleargs]() { - goto l26 + goto l67 } - goto l27 - l26: - position, tokenIndex = position26, tokenIndex26 - } - l27: - goto l22 - l23: - position, tokenIndex = position22, tokenIndex22 - { - position29, tokenIndex29 := position, tokenIndex - if !_rules[rulecomma]() { - goto l29 + goto l61 + l67: + position, tokenIndex = position61, tokenIndex61 + if !_rules[rulesp]() { + goto l5 } - goto l30 - l29: - position, tokenIndex = position29, tokenIndex29 - } - l30: - if !_rules[ruleargs]() { - goto l28 - } - goto l22 - l28: - position, tokenIndex = position22, tokenIndex22 - if !_rules[rulesp]() { - goto l5 } + l61: + add(ruleallargs, position60) } - l22: - add(ruleallargs, position21) - } - { - position31, tokenIndex31 := position, tokenIndex - if !_rules[rulecomma]() { - goto l31 + { + position70, tokenIndex70 := position, tokenIndex + if !_rules[rulecomma]() { + goto l70 + } + goto l71 + l70: + position, tokenIndex = position70, tokenIndex70 + } + l71: + if !_rules[ruleclose]() { + goto l5 + } + { + add(ruleAction13, position) } - goto l32 - l31: - position, tokenIndex = position31, tokenIndex31 - } - l32: - if !_rules[ruleclose]() { - goto l5 - } - if !_rules[rulewhitesp]() { - goto l5 - } - { - add(ruleAction1, position) } + l7: add(ruleCall, position6) } return true @@ -639,1044 +1219,1258 @@ func (p *PQL) Init() { nil, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position35, tokenIndex35 := position, tokenIndex + position74, tokenIndex74 := position, tokenIndex { - position36 := position - { - position37 := position - { - position38, tokenIndex38 := position, tokenIndex - if !_rules[rulefield]() { - goto l39 - } - if !_rules[rulesp]() { - goto l39 - } - if buffer[position] != rune('=') { - goto l39 - } - position++ - if !_rules[rulesp]() { - goto l39 - } - if !_rules[rulevalue]() { - goto l39 - } - goto l38 - l39: - position, tokenIndex = position38, tokenIndex38 - if !_rules[rulefield]() { - goto l35 - } - if !_rules[rulesp]() { - goto l35 - } - { - position40 := position - { - position41, tokenIndex41 := position, tokenIndex - if buffer[position] != rune('>') { - goto l42 - } - position++ - if buffer[position] != rune('<') { - goto l42 - } - position++ - { - add(ruleAction2, position) - } - goto l41 - l42: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('<') { - goto l44 - } - position++ - if buffer[position] != rune('=') { - goto l44 - } - position++ - { - add(ruleAction3, position) - } - goto l41 - l44: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('>') { - goto l46 - } - position++ - if buffer[position] != rune('=') { - goto l46 - } - position++ - { - add(ruleAction4, position) - } - goto l41 - l46: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('=') { - goto l48 - } - position++ - if buffer[position] != rune('=') { - goto l48 - } - position++ - { - add(ruleAction5, position) - } - goto l41 - l48: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('!') { - goto l50 - } - position++ - if buffer[position] != rune('=') { - goto l50 - } - position++ - { - add(ruleAction6, position) - } - goto l41 - l50: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('<') { - goto l52 - } - position++ - { - add(ruleAction7, position) - } - goto l41 - l52: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('>') { - goto l35 - } - position++ - { - add(ruleAction8, position) - } - } - l41: - add(ruleCOND, position40) - } - if !_rules[rulesp]() { - goto l35 - } - if !_rules[rulevalue]() { - goto l35 - } - } - l38: - add(rulearg, position37) + position75 := position + if !_rules[rulearg]() { + goto l74 } { - position55, tokenIndex55 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex if !_rules[rulecomma]() { - goto l55 + goto l76 } if !_rules[ruleargs]() { - goto l55 + goto l76 } - goto l56 - l55: - position, tokenIndex = position55, tokenIndex55 + goto l77 + l76: + position, tokenIndex = position76, tokenIndex76 } - l56: + l77: if !_rules[rulesp]() { - goto l35 + goto l74 } - add(ruleargs, position36) + add(ruleargs, position75) } return true - l35: - position, tokenIndex = position35, tokenIndex35 + l74: + position, tokenIndex = position74, tokenIndex74 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ - nil, - /* 5 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ('=' '=' Action5) / ('!' '=' Action6) / ('<' Action7) / ('>' Action8))> */ - nil, - /* 6 open <- <('(' sp)> */ - nil, - /* 7 value <- <(item / (lbrack Action9 list rbrack Action10))> */ func() bool { - position60, tokenIndex60 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex { - position61 := position + position79 := position { - position62, tokenIndex62 := position, tokenIndex - if !_rules[ruleitem]() { - goto l63 + position80, tokenIndex80 := position, tokenIndex + if !_rules[rulefield]() { + goto l81 } - goto l62 - l63: - position, tokenIndex = position62, tokenIndex62 - { - position64 := position - if buffer[position] != rune('[') { - goto l60 - } - position++ - if !_rules[rulesp]() { - goto l60 - } - add(rulelbrack, position64) + if !_rules[rulesp]() { + goto l81 } - { - add(ruleAction9, position) - } - if !_rules[rulelist]() { - goto l60 - } - { - position66 := position - if !_rules[rulesp]() { - goto l60 - } - if buffer[position] != rune(']') { - goto l60 - } - position++ - if !_rules[rulesp]() { - goto l60 - } - add(rulerbrack, position66) - } - { - add(ruleAction10, position) - } - } - l62: - add(rulevalue, position61) - } - return true - l60: - position, tokenIndex = position60, tokenIndex60 - return false - }, - /* 8 list <- <(item (comma list)?)> */ - func() bool { - position68, tokenIndex68 := position, tokenIndex - { - position69 := position - if !_rules[ruleitem]() { - goto l68 - } - { - position70, tokenIndex70 := position, tokenIndex - if !_rules[rulecomma]() { - goto l70 - } - if !_rules[rulelist]() { - goto l70 - } - goto l71 - l70: - position, tokenIndex = position70, tokenIndex70 - } - l71: - add(rulelist, position69) - } - return true - l68: - position, tokenIndex = position68, tokenIndex68 - return false - }, - /* 9 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action11) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action12) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action16) / ('"' '"' Action17) / ('\'' '\'' Action18))> */ - func() bool { - position72, tokenIndex72 := position, tokenIndex - { - position73 := position - { - position74, tokenIndex74 := position, tokenIndex - if buffer[position] != rune('n') { - goto l75 + if buffer[position] != rune('=') { + goto l81 } position++ - if buffer[position] != rune('u') { - goto l75 + if !_rules[rulesp]() { + goto l81 } - position++ - if buffer[position] != rune('l') { - goto l75 + if !_rules[rulevalue]() { + goto l81 } - position++ - if buffer[position] != rune('l') { - goto l75 + goto l80 + l81: + position, tokenIndex = position80, tokenIndex80 + if !_rules[rulefield]() { + goto l78 + } + if !_rules[rulesp]() { + goto l78 } - position++ { - position76, tokenIndex76 := position, tokenIndex + position82 := position { - position77, tokenIndex77 := position, tokenIndex - if !_rules[rulecomma]() { - goto l78 + position83, tokenIndex83 := position, tokenIndex + if buffer[position] != rune('>') { + goto l84 } - goto l77 - l78: - position, tokenIndex = position77, tokenIndex77 - if !_rules[rulesp]() { - goto l75 + position++ + if buffer[position] != rune('<') { + goto l84 } - if !_rules[ruleclose]() { - goto l75 + position++ + { + add(ruleAction14, position) } - } - l77: - position, tokenIndex = position76, tokenIndex76 - } - { - add(ruleAction11, position) - } - goto l74 - l75: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('t') { - goto l80 - } - position++ - if buffer[position] != rune('r') { - goto l80 - } - position++ - if buffer[position] != rune('u') { - goto l80 - } - position++ - if buffer[position] != rune('e') { - goto l80 - } - position++ - { - position81, tokenIndex81 := position, tokenIndex - { - position82, tokenIndex82 := position, tokenIndex - if !_rules[rulecomma]() { - goto l83 + goto l83 + l84: + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('<') { + goto l86 } - goto l82 - l83: - position, tokenIndex = position82, tokenIndex82 - if !_rules[rulesp]() { - goto l80 + position++ + if buffer[position] != rune('=') { + goto l86 } - if !_rules[ruleclose]() { - goto l80 + position++ + { + add(ruleAction15, position) } - } - l82: - position, tokenIndex = position81, tokenIndex81 - } - { - add(ruleAction12, position) - } - goto l74 - l80: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('f') { - goto l85 - } - position++ - if buffer[position] != rune('a') { - goto l85 - } - position++ - if buffer[position] != rune('l') { - goto l85 - } - position++ - if buffer[position] != rune('s') { - goto l85 - } - position++ - if buffer[position] != rune('e') { - goto l85 - } - position++ - { - position86, tokenIndex86 := position, tokenIndex - { - position87, tokenIndex87 := position, tokenIndex - if !_rules[rulecomma]() { + goto l83 + l86: + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('>') { goto l88 } - goto l87 + position++ + if buffer[position] != rune('=') { + goto l88 + } + position++ + { + add(ruleAction16, position) + } + goto l83 l88: - position, tokenIndex = position87, tokenIndex87 - if !_rules[rulesp]() { - goto l85 + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('=') { + goto l90 } - if !_rules[ruleclose]() { - goto l85 + position++ + if buffer[position] != rune('=') { + goto l90 } - } - l87: - position, tokenIndex = position86, tokenIndex86 - } - { - add(ruleAction13, position) - } - goto l74 - l85: - position, tokenIndex = position74, tokenIndex74 - { - position91 := position - { - position92, tokenIndex92 := position, tokenIndex - if buffer[position] != rune('-') { + position++ + { + add(ruleAction17, position) + } + goto l83 + l90: + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('!') { goto l92 } position++ - goto l93 - l92: - position, tokenIndex = position92, tokenIndex92 - } - l93: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l90 - } - position++ - l94: - { - position95, tokenIndex95 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l95 + if buffer[position] != rune('=') { + goto l92 } position++ - goto l94 - l95: - position, tokenIndex = position95, tokenIndex95 - } - { - position96, tokenIndex96 := position, tokenIndex - if buffer[position] != rune('.') { - goto l96 - } - position++ - l98: { - position99, tokenIndex99 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l99 - } - position++ - goto l98 - l99: - position, tokenIndex = position99, tokenIndex99 + add(ruleAction18, position) } - goto l97 - l96: - position, tokenIndex = position96, tokenIndex96 - } - l97: - add(rulePegText, position91) - } - { - add(ruleAction14, position) - } - goto l74 - l90: - position, tokenIndex = position74, tokenIndex74 - { - position102 := position - { - position103, tokenIndex103 := position, tokenIndex - if buffer[position] != rune('-') { - goto l103 + goto l83 + l92: + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('<') { + goto l94 } position++ - goto l104 - l103: - position, tokenIndex = position103, tokenIndex103 + { + add(ruleAction19, position) + } + goto l83 + l94: + position, tokenIndex = position83, tokenIndex83 + if buffer[position] != rune('>') { + goto l78 + } + position++ + { + add(ruleAction20, position) + } } - l104: - if buffer[position] != rune('.') { + l83: + add(ruleCOND, position82) + } + if !_rules[rulesp]() { + goto l78 + } + if !_rules[rulevalue]() { + goto l78 + } + } + l80: + add(rulearg, position79) + } + return true + l78: + position, tokenIndex = position78, tokenIndex78 + return false + }, + /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ + nil, + /* 6 conditional <- <(Action21 int (('<' '=') / '<') fieldExpr (('<' '=') / '<') int Action22)> */ + nil, + /* 7 open <- <('(' sp)> */ + func() bool { + position99, tokenIndex99 := position, tokenIndex + { + position100 := position + if buffer[position] != rune('(') { + goto l99 + } + position++ + if !_rules[rulesp]() { + goto l99 + } + add(ruleopen, position100) + } + return true + l99: + position, tokenIndex = position99, tokenIndex99 + return false + }, + /* 8 value <- <(item / (lbrack Action23 list rbrack Action24))> */ + func() bool { + position101, tokenIndex101 := position, tokenIndex + { + position102 := position + { + position103, tokenIndex103 := position, tokenIndex + if !_rules[ruleitem]() { + goto l104 + } + goto l103 + l104: + position, tokenIndex = position103, tokenIndex103 + { + position105 := position + if buffer[position] != rune('[') { goto l101 } position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { + if !_rules[rulesp]() { + goto l101 + } + add(rulelbrack, position105) + } + { + add(ruleAction23, position) + } + if !_rules[rulelist]() { + goto l101 + } + { + position107 := position + if !_rules[rulesp]() { + goto l101 + } + if buffer[position] != rune(']') { goto l101 } position++ - l105: - { - position106, tokenIndex106 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l106 - } - position++ - goto l105 - l106: - position, tokenIndex = position106, tokenIndex106 + if !_rules[rulesp]() { + goto l101 } - add(rulePegText, position102) + add(rulerbrack, position107) } { - add(ruleAction15, position) + add(ruleAction24, position) } - goto l74 - l101: - position, tokenIndex = position74, tokenIndex74 + } + l103: + add(rulevalue, position102) + } + return true + l101: + position, tokenIndex = position101, tokenIndex101 + return false + }, + /* 9 list <- <(item (comma list)?)> */ + func() bool { + position109, tokenIndex109 := position, tokenIndex + { + position110 := position + if !_rules[ruleitem]() { + goto l109 + } + { + position111, tokenIndex111 := position, tokenIndex + if !_rules[rulecomma]() { + goto l111 + } + if !_rules[rulelist]() { + goto l111 + } + goto l112 + l111: + position, tokenIndex = position111, tokenIndex111 + } + l112: + add(rulelist, position110) + } + return true + l109: + position, tokenIndex = position109, tokenIndex109 + return false + }, + /* 10 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action25) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action26) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action27) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action28) / (<('-'? '.' [0-9]+)> Action29) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action30) / ('"' '"' Action31) / ('\'' '\'' Action32))> */ + func() bool { + position113, tokenIndex113 := position, tokenIndex + { + position114 := position + { + position115, tokenIndex115 := position, tokenIndex + if buffer[position] != rune('n') { + goto l116 + } + position++ + if buffer[position] != rune('u') { + goto l116 + } + position++ + if buffer[position] != rune('l') { + goto l116 + } + position++ + if buffer[position] != rune('l') { + goto l116 + } + position++ { - position109 := position + position117, tokenIndex117 := position, tokenIndex { - position112, tokenIndex112 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l113 + position118, tokenIndex118 := position, tokenIndex + if !_rules[rulecomma]() { + goto l119 } - position++ - goto l112 - l113: - position, tokenIndex = position112, tokenIndex112 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l114 - } - position++ - goto l112 - l114: - position, tokenIndex = position112, tokenIndex112 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l115 - } - position++ - goto l112 - l115: - position, tokenIndex = position112, tokenIndex112 - if buffer[position] != rune('-') { + goto l118 + l119: + position, tokenIndex = position118, tokenIndex118 + if !_rules[rulesp]() { goto l116 } - position++ - goto l112 - l116: - position, tokenIndex = position112, tokenIndex112 - if buffer[position] != rune('_') { - goto l117 - } - position++ - goto l112 - l117: - position, tokenIndex = position112, tokenIndex112 - if buffer[position] != rune(':') { - goto l108 + if !_rules[ruleclose]() { + goto l116 + } + } + l118: + position, tokenIndex = position117, tokenIndex117 + } + { + add(ruleAction25, position) + } + goto l115 + l116: + position, tokenIndex = position115, tokenIndex115 + if buffer[position] != rune('t') { + goto l121 + } + position++ + if buffer[position] != rune('r') { + goto l121 + } + position++ + if buffer[position] != rune('u') { + goto l121 + } + position++ + if buffer[position] != rune('e') { + goto l121 + } + position++ + { + position122, tokenIndex122 := position, tokenIndex + { + position123, tokenIndex123 := position, tokenIndex + if !_rules[rulecomma]() { + goto l124 + } + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 + if !_rules[rulesp]() { + goto l121 + } + if !_rules[ruleclose]() { + goto l121 + } + } + l123: + position, tokenIndex = position122, tokenIndex122 + } + { + add(ruleAction26, position) + } + goto l115 + l121: + position, tokenIndex = position115, tokenIndex115 + if buffer[position] != rune('f') { + goto l126 + } + position++ + if buffer[position] != rune('a') { + goto l126 + } + position++ + if buffer[position] != rune('l') { + goto l126 + } + position++ + if buffer[position] != rune('s') { + goto l126 + } + position++ + if buffer[position] != rune('e') { + goto l126 + } + position++ + { + position127, tokenIndex127 := position, tokenIndex + { + position128, tokenIndex128 := position, tokenIndex + if !_rules[rulecomma]() { + goto l129 + } + goto l128 + l129: + position, tokenIndex = position128, tokenIndex128 + if !_rules[rulesp]() { + goto l126 + } + if !_rules[ruleclose]() { + goto l126 + } + } + l128: + position, tokenIndex = position127, tokenIndex127 + } + { + add(ruleAction27, position) + } + goto l115 + l126: + position, tokenIndex = position115, tokenIndex115 + { + position132 := position + { + position133, tokenIndex133 := position, tokenIndex + if buffer[position] != rune('-') { + goto l133 } position++ + goto l134 + l133: + position, tokenIndex = position133, tokenIndex133 } - l112: - l110: - { - position111, tokenIndex111 := position, tokenIndex - { - position118, tokenIndex118 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l119 - } - position++ - goto l118 - l119: - position, tokenIndex = position118, tokenIndex118 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l120 - } - position++ - goto l118 - l120: - position, tokenIndex = position118, tokenIndex118 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l121 - } - position++ - goto l118 - l121: - position, tokenIndex = position118, tokenIndex118 - if buffer[position] != rune('-') { - goto l122 - } - position++ - goto l118 - l122: - position, tokenIndex = position118, tokenIndex118 - if buffer[position] != rune('_') { - goto l123 - } - position++ - goto l118 - l123: - position, tokenIndex = position118, tokenIndex118 - if buffer[position] != rune(':') { - goto l111 - } - position++ - } - l118: - goto l110 - l111: - position, tokenIndex = position111, tokenIndex111 - } - add(rulePegText, position109) - } - { - add(ruleAction16, position) - } - goto l74 - l108: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('"') { - goto l125 - } - position++ - { - position126 := position - { - position127 := position - l128: - { - position129, tokenIndex129 := position, tokenIndex - { - position130, tokenIndex130 := position, tokenIndex - { - position132, tokenIndex132 := position, tokenIndex - { - position133, tokenIndex133 := position, tokenIndex - if buffer[position] != rune('"') { - goto l134 - } - position++ - goto l133 - l134: - position, tokenIndex = position133, tokenIndex133 - if buffer[position] != rune('\\') { - goto l135 - } - position++ - goto l133 - l135: - position, tokenIndex = position133, tokenIndex133 - if buffer[position] != rune('\n') { - goto l132 - } - position++ - } - l133: - goto l131 - l132: - position, tokenIndex = position132, tokenIndex132 - } - if !matchDot() { - goto l131 - } - goto l130 - l131: - position, tokenIndex = position130, tokenIndex130 - if buffer[position] != rune('\\') { - goto l136 - } - position++ - if buffer[position] != rune('n') { - goto l136 - } - position++ - goto l130 - l136: - position, tokenIndex = position130, tokenIndex130 - if buffer[position] != rune('\\') { - goto l137 - } - position++ - if buffer[position] != rune('"') { - goto l137 - } - position++ - goto l130 - l137: - position, tokenIndex = position130, tokenIndex130 - if buffer[position] != rune('\\') { - goto l138 - } - position++ - if buffer[position] != rune('\'') { - goto l138 - } - position++ - goto l130 - l138: - position, tokenIndex = position130, tokenIndex130 - if buffer[position] != rune('\\') { - goto l129 - } - position++ - if buffer[position] != rune('\\') { - goto l129 - } - position++ - } - l130: - goto l128 - l129: - position, tokenIndex = position129, tokenIndex129 - } - add(ruledoublequotedstring, position127) - } - add(rulePegText, position126) - } - if buffer[position] != rune('"') { - goto l125 - } - position++ - { - add(ruleAction17, position) - } - goto l74 - l125: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('\'') { - goto l72 - } - position++ - { - position140 := position - { - position141 := position - l142: - { - position143, tokenIndex143 := position, tokenIndex - { - position144, tokenIndex144 := position, tokenIndex - { - position146, tokenIndex146 := position, tokenIndex - { - position147, tokenIndex147 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l148 - } - position++ - goto l147 - l148: - position, tokenIndex = position147, tokenIndex147 - if buffer[position] != rune('\\') { - goto l149 - } - position++ - goto l147 - l149: - position, tokenIndex = position147, tokenIndex147 - if buffer[position] != rune('\n') { - goto l146 - } - position++ - } - l147: - goto l145 - l146: - position, tokenIndex = position146, tokenIndex146 - } - if !matchDot() { - goto l145 - } - goto l144 - l145: - position, tokenIndex = position144, tokenIndex144 - if buffer[position] != rune('\\') { - goto l150 - } - position++ - if buffer[position] != rune('n') { - goto l150 - } - position++ - goto l144 - l150: - position, tokenIndex = position144, tokenIndex144 - if buffer[position] != rune('\\') { - goto l151 - } - position++ - if buffer[position] != rune('"') { - goto l151 - } - position++ - goto l144 - l151: - position, tokenIndex = position144, tokenIndex144 - if buffer[position] != rune('\\') { - goto l152 - } - position++ - if buffer[position] != rune('\'') { - goto l152 - } - position++ - goto l144 - l152: - position, tokenIndex = position144, tokenIndex144 - if buffer[position] != rune('\\') { - goto l143 - } - position++ - if buffer[position] != rune('\\') { - goto l143 - } - position++ - } - l144: - goto l142 - l143: - position, tokenIndex = position143, tokenIndex143 - } - add(rulesinglequotedstring, position141) - } - add(rulePegText, position140) - } - if buffer[position] != rune('\'') { - goto l72 - } - position++ - { - add(ruleAction18, position) - } - } - l74: - add(ruleitem, position73) - } - return true - l72: - position, tokenIndex = position72, tokenIndex72 - return false - }, - /* 10 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ - nil, - /* 11 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ - nil, - /* 12 field <- <(<(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> Action19)> */ - func() bool { - position156, tokenIndex156 := position, tokenIndex - { - position157 := position - { - position158 := position - { - position159, tokenIndex159 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l160 + l134: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l131 } position++ - goto l159 - l160: - position, tokenIndex = position159, tokenIndex159 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l156 - } - position++ - } - l159: - l161: - { - position162, tokenIndex162 := position, tokenIndex + l135: { - position163, tokenIndex163 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l164 - } - position++ - goto l163 - l164: - position, tokenIndex = position163, tokenIndex163 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l165 - } - position++ - goto l163 - l165: - position, tokenIndex = position163, tokenIndex163 + position136, tokenIndex136 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l166 + goto l136 } position++ - goto l163 - l166: - position, tokenIndex = position163, tokenIndex163 + goto l135 + l136: + position, tokenIndex = position136, tokenIndex136 + } + { + position137, tokenIndex137 := position, tokenIndex + if buffer[position] != rune('.') { + goto l137 + } + position++ + l139: + { + position140, tokenIndex140 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l140 + } + position++ + goto l139 + l140: + position, tokenIndex = position140, tokenIndex140 + } + goto l138 + l137: + position, tokenIndex = position137, tokenIndex137 + } + l138: + add(rulePegText, position132) + } + { + add(ruleAction28, position) + } + goto l115 + l131: + position, tokenIndex = position115, tokenIndex115 + { + position143 := position + { + position144, tokenIndex144 := position, tokenIndex + if buffer[position] != rune('-') { + goto l144 + } + position++ + goto l145 + l144: + position, tokenIndex = position144, tokenIndex144 + } + l145: + if buffer[position] != rune('.') { + goto l142 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l142 + } + position++ + l146: + { + position147, tokenIndex147 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l147 + } + position++ + goto l146 + l147: + position, tokenIndex = position147, tokenIndex147 + } + add(rulePegText, position143) + } + { + add(ruleAction29, position) + } + goto l115 + l142: + position, tokenIndex = position115, tokenIndex115 + { + position150 := position + { + position153, tokenIndex153 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l154 + } + position++ + goto l153 + l154: + position, tokenIndex = position153, tokenIndex153 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l155 + } + position++ + goto l153 + l155: + position, tokenIndex = position153, tokenIndex153 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l156 + } + position++ + goto l153 + l156: + position, tokenIndex = position153, tokenIndex153 + if buffer[position] != rune('-') { + goto l157 + } + position++ + goto l153 + l157: + position, tokenIndex = position153, tokenIndex153 if buffer[position] != rune('_') { - goto l162 + goto l158 + } + position++ + goto l153 + l158: + position, tokenIndex = position153, tokenIndex153 + if buffer[position] != rune(':') { + goto l149 } position++ } - l163: - goto l161 - l162: - position, tokenIndex = position162, tokenIndex162 + l153: + l151: + { + position152, tokenIndex152 := position, tokenIndex + { + position159, tokenIndex159 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l160 + } + position++ + goto l159 + l160: + position, tokenIndex = position159, tokenIndex159 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l161 + } + position++ + goto l159 + l161: + position, tokenIndex = position159, tokenIndex159 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l162 + } + position++ + goto l159 + l162: + position, tokenIndex = position159, tokenIndex159 + if buffer[position] != rune('-') { + goto l163 + } + position++ + goto l159 + l163: + position, tokenIndex = position159, tokenIndex159 + if buffer[position] != rune('_') { + goto l164 + } + position++ + goto l159 + l164: + position, tokenIndex = position159, tokenIndex159 + if buffer[position] != rune(':') { + goto l152 + } + position++ + } + l159: + goto l151 + l152: + position, tokenIndex = position152, tokenIndex152 + } + add(rulePegText, position150) + } + { + add(ruleAction30, position) + } + goto l115 + l149: + position, tokenIndex = position115, tokenIndex115 + if buffer[position] != rune('"') { + goto l166 + } + position++ + { + position167 := position + { + position168 := position + l169: + { + position170, tokenIndex170 := position, tokenIndex + { + position171, tokenIndex171 := position, tokenIndex + { + position173, tokenIndex173 := position, tokenIndex + { + position174, tokenIndex174 := position, tokenIndex + if buffer[position] != rune('"') { + goto l175 + } + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('\\') { + goto l176 + } + position++ + goto l174 + l176: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('\n') { + goto l173 + } + position++ + } + l174: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + if !matchDot() { + goto l172 + } + goto l171 + l172: + position, tokenIndex = position171, tokenIndex171 + if buffer[position] != rune('\\') { + goto l177 + } + position++ + if buffer[position] != rune('n') { + goto l177 + } + position++ + goto l171 + l177: + position, tokenIndex = position171, tokenIndex171 + if buffer[position] != rune('\\') { + goto l178 + } + position++ + if buffer[position] != rune('"') { + goto l178 + } + position++ + goto l171 + l178: + position, tokenIndex = position171, tokenIndex171 + if buffer[position] != rune('\\') { + goto l179 + } + position++ + if buffer[position] != rune('\'') { + goto l179 + } + position++ + goto l171 + l179: + position, tokenIndex = position171, tokenIndex171 + if buffer[position] != rune('\\') { + goto l170 + } + position++ + if buffer[position] != rune('\\') { + goto l170 + } + position++ + } + l171: + goto l169 + l170: + position, tokenIndex = position170, tokenIndex170 + } + add(ruledoublequotedstring, position168) + } + add(rulePegText, position167) + } + if buffer[position] != rune('"') { + goto l166 + } + position++ + { + add(ruleAction31, position) + } + goto l115 + l166: + position, tokenIndex = position115, tokenIndex115 + if buffer[position] != rune('\'') { + goto l113 + } + position++ + { + position181 := position + { + position182 := position + l183: + { + position184, tokenIndex184 := position, tokenIndex + { + position185, tokenIndex185 := position, tokenIndex + { + position187, tokenIndex187 := position, tokenIndex + { + position188, tokenIndex188 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l189 + } + position++ + goto l188 + l189: + position, tokenIndex = position188, tokenIndex188 + if buffer[position] != rune('\\') { + goto l190 + } + position++ + goto l188 + l190: + position, tokenIndex = position188, tokenIndex188 + if buffer[position] != rune('\n') { + goto l187 + } + position++ + } + l188: + goto l186 + l187: + position, tokenIndex = position187, tokenIndex187 + } + if !matchDot() { + goto l186 + } + goto l185 + l186: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('\\') { + goto l191 + } + position++ + if buffer[position] != rune('n') { + goto l191 + } + position++ + goto l185 + l191: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('\\') { + goto l192 + } + position++ + if buffer[position] != rune('"') { + goto l192 + } + position++ + goto l185 + l192: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('\\') { + goto l193 + } + position++ + if buffer[position] != rune('\'') { + goto l193 + } + position++ + goto l185 + l193: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('\\') { + goto l184 + } + position++ + if buffer[position] != rune('\\') { + goto l184 + } + position++ + } + l185: + goto l183 + l184: + position, tokenIndex = position184, tokenIndex184 + } + add(rulesinglequotedstring, position182) + } + add(rulePegText, position181) + } + if buffer[position] != rune('\'') { + goto l113 + } + position++ + { + add(ruleAction32, position) } - add(rulePegText, position158) } - { - add(ruleAction19, position) - } - add(rulefield, position157) + l115: + add(ruleitem, position114) } return true - l156: - position, tokenIndex = position156, tokenIndex156 + l113: + position, tokenIndex = position113, tokenIndex113 return false }, - /* 13 close <- <(')' sp)> */ + /* 11 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + nil, + /* 12 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + nil, + /* 13 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ func() bool { - position168, tokenIndex168 := position, tokenIndex + position197, tokenIndex197 := position, tokenIndex { - position169 := position + position198 := position + { + position199, tokenIndex199 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l200 + } + position++ + goto l199 + l200: + position, tokenIndex = position199, tokenIndex199 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l197 + } + position++ + } + l199: + l201: + { + position202, tokenIndex202 := position, tokenIndex + { + position203, tokenIndex203 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l204 + } + position++ + goto l203 + l204: + position, tokenIndex = position203, tokenIndex203 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l205 + } + position++ + goto l203 + l205: + position, tokenIndex = position203, tokenIndex203 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l206 + } + position++ + goto l203 + l206: + position, tokenIndex = position203, tokenIndex203 + if buffer[position] != rune('_') { + goto l202 + } + position++ + } + l203: + goto l201 + l202: + position, tokenIndex = position202, tokenIndex202 + } + add(rulefieldExpr, position198) + } + return true + l197: + position, tokenIndex = position197, tokenIndex197 + return false + }, + /* 14 field <- <( Action33)> */ + func() bool { + position207, tokenIndex207 := position, tokenIndex + { + position208 := position + { + position209 := position + if !_rules[rulefieldExpr]() { + goto l207 + } + add(rulePegText, position209) + } + { + add(ruleAction33, position) + } + add(rulefield, position208) + } + return true + l207: + position, tokenIndex = position207, tokenIndex207 + return false + }, + /* 15 posfield <- <( Action34)> */ + func() bool { + position211, tokenIndex211 := position, tokenIndex + { + position212 := position + { + position213 := position + if !_rules[rulefieldExpr]() { + goto l211 + } + add(rulePegText, position213) + } + { + add(ruleAction34, position) + } + add(ruleposfield, position212) + } + return true + l211: + position, tokenIndex = position211, tokenIndex211 + return false + }, + /* 16 uint <- <(([1-9] [0-9]*) / '0')> */ + func() bool { + position215, tokenIndex215 := position, tokenIndex + { + position216 := position + { + position217, tokenIndex217 := position, tokenIndex + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l218 + } + position++ + l219: + { + position220, tokenIndex220 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l220 + } + position++ + goto l219 + l220: + position, tokenIndex = position220, tokenIndex220 + } + goto l217 + l218: + position, tokenIndex = position217, tokenIndex217 + if buffer[position] != rune('0') { + goto l215 + } + position++ + } + l217: + add(ruleuint, position216) + } + return true + l215: + position, tokenIndex = position215, tokenIndex215 + return false + }, + /* 17 int <- <(('-'? [1-9] [0-9]*) / '0')> */ + func() bool { + position221, tokenIndex221 := position, tokenIndex + { + position222 := position + { + position223, tokenIndex223 := position, tokenIndex + { + position225, tokenIndex225 := position, tokenIndex + if buffer[position] != rune('-') { + goto l225 + } + position++ + goto l226 + l225: + position, tokenIndex = position225, tokenIndex225 + } + l226: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l224 + } + position++ + l227: + { + position228, tokenIndex228 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l228 + } + position++ + goto l227 + l228: + position, tokenIndex = position228, tokenIndex228 + } + goto l223 + l224: + position, tokenIndex = position223, tokenIndex223 + if buffer[position] != rune('0') { + goto l221 + } + position++ + } + l223: + add(ruleint, position222) + } + return true + l221: + position, tokenIndex = position221, tokenIndex221 + return false + }, + /* 18 uintrow <- <( Action35)> */ + nil, + /* 19 uintcol <- <( Action36)> */ + func() bool { + position230, tokenIndex230 := position, tokenIndex + { + position231 := position + { + position232 := position + if !_rules[ruleuint]() { + goto l230 + } + add(rulePegText, position232) + } + { + add(ruleAction36, position) + } + add(ruleuintcol, position231) + } + return true + l230: + position, tokenIndex = position230, tokenIndex230 + return false + }, + /* 20 close <- <(')' sp)> */ + func() bool { + position234, tokenIndex234 := position, tokenIndex + { + position235 := position if buffer[position] != rune(')') { - goto l168 + goto l234 } position++ if !_rules[rulesp]() { - goto l168 + goto l234 } - add(ruleclose, position169) + add(ruleclose, position235) } return true - l168: - position, tokenIndex = position168, tokenIndex168 + l234: + position, tokenIndex = position234, tokenIndex234 return false }, - /* 14 sp <- <(' ' / '\t')*> */ + /* 21 sp <- <(' ' / '\t')*> */ func() bool { { - position171 := position - l172: + position237 := position + l238: { - position173, tokenIndex173 := position, tokenIndex + position239, tokenIndex239 := position, tokenIndex { - position174, tokenIndex174 := position, tokenIndex + position240, tokenIndex240 := position, tokenIndex if buffer[position] != rune(' ') { - goto l175 + goto l241 } position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 + goto l240 + l241: + position, tokenIndex = position240, tokenIndex240 if buffer[position] != rune('\t') { - goto l173 + goto l239 } position++ } - l174: - goto l172 - l173: - position, tokenIndex = position173, tokenIndex173 + l240: + goto l238 + l239: + position, tokenIndex = position239, tokenIndex239 } - add(rulesp, position171) + add(rulesp, position237) } return true }, - /* 15 comma <- <(sp ',' sp)> */ + /* 22 comma <- <(sp ',' whitesp)> */ func() bool { - position176, tokenIndex176 := position, tokenIndex + position242, tokenIndex242 := position, tokenIndex { - position177 := position + position243 := position if !_rules[rulesp]() { - goto l176 + goto l242 } if buffer[position] != rune(',') { - goto l176 + goto l242 } position++ - if !_rules[rulesp]() { - goto l176 + if !_rules[rulewhitesp]() { + goto l242 } - add(rulecomma, position177) + add(rulecomma, position243) } return true - l176: - position, tokenIndex = position176, tokenIndex176 + l242: + position, tokenIndex = position242, tokenIndex242 return false }, - /* 16 lbrack <- <('[' sp)> */ + /* 23 lbrack <- <('[' sp)> */ nil, - /* 17 rbrack <- <(sp ']' sp)> */ + /* 24 rbrack <- <(sp ']' sp)> */ nil, - /* 18 whitesp <- <(' ' / '\t' / '\n')*> */ + /* 25 whitesp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position181 := position - l182: + position247 := position + l248: { - position183, tokenIndex183 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position184, tokenIndex184 := position, tokenIndex + position250, tokenIndex250 := position, tokenIndex if buffer[position] != rune(' ') { - goto l185 + goto l251 } position++ - goto l184 - l185: - position, tokenIndex = position184, tokenIndex184 + goto l250 + l251: + position, tokenIndex = position250, tokenIndex250 if buffer[position] != rune('\t') { - goto l186 + goto l252 } position++ - goto l184 - l186: - position, tokenIndex = position184, tokenIndex184 + goto l250 + l252: + position, tokenIndex = position250, tokenIndex250 if buffer[position] != rune('\n') { - goto l183 + goto l249 } position++ } - l184: - goto l182 - l183: - position, tokenIndex = position183, tokenIndex183 + l250: + goto l248 + l249: + position, tokenIndex = position249, tokenIndex249 } - add(rulewhitesp, position181) + add(rulewhitesp, position247) } return true }, - /* 19 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '-' / '_' / '.')*)> */ + /* 26 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '-' / '_' / '.')*)> */ + nil, + /* 27 timestamp <- <(<([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> Action37)> */ + nil, + /* 29 Action0 <- <{p.startCall("Set")}> */ + nil, + /* 30 Action1 <- <{p.endCall()}> */ + nil, + /* 31 Action2 <- <{p.startCall("SetRowAttrs")}> */ + nil, + /* 32 Action3 <- <{p.endCall()}> */ + nil, + /* 33 Action4 <- <{p.startCall("SetColAttrs")}> */ + nil, + /* 34 Action5 <- <{p.endCall()}> */ + nil, + /* 35 Action6 <- <{p.startCall("ClearBit")}> */ + nil, + /* 36 Action7 <- <{p.endCall()}> */ + nil, + /* 37 Action8 <- <{p.startCall("TopN")}> */ + nil, + /* 38 Action9 <- <{p.endCall()}> */ + nil, + /* 39 Action10 <- <{p.startCall("Range")}> */ + nil, + /* 40 Action11 <- <{p.endCall()}> */ nil, nil, - /* 22 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 42 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 23 Action1 <- <{ p.endCall() }> */ + /* 43 Action13 <- <{ p.endCall() }> */ nil, - /* 24 Action2 <- <{ p.addBTWN() }> */ + /* 44 Action14 <- <{ p.addBTWN() }> */ nil, - /* 25 Action3 <- <{ p.addLTE() }> */ + /* 45 Action15 <- <{ p.addLTE() }> */ nil, - /* 26 Action4 <- <{ p.addGTE() }> */ + /* 46 Action16 <- <{ p.addGTE() }> */ nil, - /* 27 Action5 <- <{ p.addEQ() }> */ + /* 47 Action17 <- <{ p.addEQ() }> */ nil, - /* 28 Action6 <- <{ p.addNEQ() }> */ + /* 48 Action18 <- <{ p.addNEQ() }> */ nil, - /* 29 Action7 <- <{ p.addLT() }> */ + /* 49 Action19 <- <{ p.addLT() }> */ nil, - /* 30 Action8 <- <{ p.addGT() }> */ + /* 50 Action20 <- <{ p.addGT() }> */ nil, - /* 31 Action9 <- <{ p.startList() }> */ + /* 51 Action21 <- <{p.startConditional()}> */ nil, - /* 32 Action10 <- <{ p.endList() }> */ + /* 52 Action22 <- <{p.endConditional()}> */ nil, - /* 33 Action11 <- <{ p.addVal(nil) }> */ + /* 53 Action23 <- <{ p.startList() }> */ nil, - /* 34 Action12 <- <{ p.addVal(true) }> */ + /* 54 Action24 <- <{ p.endList() }> */ nil, - /* 35 Action13 <- <{ p.addVal(false) }> */ + /* 55 Action25 <- <{ p.addVal(nil) }> */ nil, - /* 36 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 56 Action26 <- <{ p.addVal(true) }> */ nil, - /* 37 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 57 Action27 <- <{ p.addVal(false) }> */ nil, - /* 38 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 58 Action28 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 39 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 59 Action29 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 40 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 60 Action30 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 41 Action19 <- <{ p.addField(buffer[begin:end]) }> */ + /* 61 Action31 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 62 Action32 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 63 Action33 <- <{ p.addField(buffer[begin:end]) }> */ + nil, + /* 64 Action34 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + nil, + /* 65 Action35 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 66 Action36 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + nil, + /* 67 Action37 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules From 23bca175ae7367db3d1534e6ee0d544a76dbeb88 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 15 Jun 2018 15:02:41 -0500 Subject: [PATCH 07/33] add tests, fix tests, fix bugs --- pql/parser_test.go | 10 +- pql/pql.peg | 10 +- pql/pql.peg.go | 733 ++++++++++++++++++++++++++++----------------- pql/pqlpeg_test.go | 158 +++++++++- 4 files changed, 617 insertions(+), 294 deletions(-) diff --git a/pql/parser_test.go b/pql/parser_test.go index 411406815..c7a260b92 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) { // Parse with both child calls and arguments. t.Run("ChildrenAndArguments", func(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) + q, err := pql.ParseString(`TopN(f, Bitmap(id=100, field=other), n=3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -145,7 +145,7 @@ func TestParser_Parse(t *testing.T) { Name: "Bitmap", Args: map[string]interface{}{"id": int64(100), "field": "other"}, }}, - Args: map[string]interface{}{"n": int64(3), "field": "f"}, + Args: map[string]interface{}{"n": int64(3), "_field": "f"}, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) @@ -154,15 +154,15 @@ func TestParser_Parse(t *testing.T) { // Parse a list argument. t.Run("ListArgument", func(t *testing.T) { - q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) + q, err := pql.ParseString(`TopN(f, ids=[0,10,30])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ Name: "TopN", Args: map[string]interface{}{ - "field": "f", - "ids": []interface{}{int64(0), int64(10), int64(30)}, + "_field": "f", + "ids": []interface{}{int64(0), int64(10), int64(30)}, }, }, ) { diff --git a/pql/pql.peg b/pql/pql.peg index f288dadf6..27c15cb25 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -9,11 +9,11 @@ Calls <- whitesp (Call whitesp)* !. Call <- 'Set' {p.startCall("Set")} open uintcol comma args (comma timestamp)? close {p.endCall()} / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} / 'SetColAttrs' {p.startCall("SetColAttrs")} open posfield comma uintcol comma args close {p.endCall()} - / 'ClearBit' {p.startCall("ClearBit")} open uintcol comma args close {p.endCall()} - / 'TopN' {p.startCall("TopN")} open posfield (comma args)? close {p.endCall()} + / 'Clear' {p.startCall("Clear")} open uintcol comma args close {p.endCall()} + / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (arg / conditional) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } -allargs <- Call (comma Call)* (comma args)? / comma? args / sp +allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp arg <- ( field sp '=' sp value / field sp COND sp value @@ -27,7 +27,6 @@ COND <- ( '><' { p.addBTWN() } / '>' { p.addGT() } ) conditional <- {p.startConditional()} int ('<=' / '<') fieldExpr ('<=' / '<') int {p.endConditional()} -open <- '(' sp value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) @@ -53,12 +52,13 @@ int <- '-'? [1-9] [0-9]* / '0' uintrow <- {p.addPosNum("_row", buffer[begin:end])} uintcol <- {p.addPosNum("_col", buffer[begin:end])} +open <- '(' sp close <- ')' sp sp <- ( ' ' / '\t' )* comma <- sp ',' whitesp lbrack <- '[' sp rbrack <- sp ']' sp whitesp <- ( ' ' / '\t' / '\n' )* -IDENT <- [[A-Z]] ([[A-Z]] / [0-9] / '-' / '_' / '.')* +IDENT <- !('Set(' / 'SetRowAttrs(' / 'SetColAttrs(' / 'Clear(' / 'TopN(' / 'Range(') [[A-Z]] ([[A-Z]] / [0-9])* timestamp <- <[0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]> {p.addPosStr("_timestamp", buffer[begin:end])} \ No newline at end of file diff --git a/pql/pql.peg.go b/pql/pql.peg.go index b6dc82f5e..303d6915d 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -23,7 +23,6 @@ const ( rulearg ruleCOND ruleconditional - ruleopen rulevalue rulelist ruleitem @@ -36,6 +35,7 @@ const ( ruleint ruleuintrow ruleuintcol + ruleopen ruleclose rulesp rulecomma @@ -94,7 +94,6 @@ var rul3s = [...]string{ "arg", "COND", "conditional", - "open", "value", "list", "item", @@ -107,6 +106,7 @@ var rul3s = [...]string{ "int", "uintrow", "uintcol", + "open", "close", "sp", "comma", @@ -375,7 +375,7 @@ func (p *PQL) Execute() { case ruleAction5: p.endCall() case ruleAction6: - p.startCall("ClearBit") + p.startCall("Clear") case ruleAction7: p.endCall() case ruleAction8: @@ -549,7 +549,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' 'B' 'i' 't' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma args)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (arg / conditional) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (arg / conditional) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -867,18 +867,6 @@ func (p *PQL) Init() { goto l27 } position++ - if buffer[position] != rune('B') { - goto l27 - } - position++ - if buffer[position] != rune('i') { - goto l27 - } - position++ - if buffer[position] != rune('t') { - goto l27 - } - position++ { add(ruleAction6, position) } @@ -933,7 +921,7 @@ func (p *PQL) Init() { if !_rules[rulecomma]() { goto l32 } - if !_rules[ruleargs]() { + if !_rules[ruleallargs]() { goto l32 } goto l33 @@ -1058,68 +1046,252 @@ func (p *PQL) Init() { position48 := position { position49, tokenIndex49 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { + { + position50, tokenIndex50 := position, tokenIndex + if buffer[position] != rune('S') { + goto l51 + } + position++ + if buffer[position] != rune('e') { + goto l51 + } + position++ + if buffer[position] != rune('t') { + goto l51 + } + position++ + if buffer[position] != rune('(') { + goto l51 + } + position++ goto l50 + l51: + position, tokenIndex = position50, tokenIndex50 + if buffer[position] != rune('S') { + goto l52 + } + position++ + if buffer[position] != rune('e') { + goto l52 + } + position++ + if buffer[position] != rune('t') { + goto l52 + } + position++ + if buffer[position] != rune('R') { + goto l52 + } + position++ + if buffer[position] != rune('o') { + goto l52 + } + position++ + if buffer[position] != rune('w') { + goto l52 + } + position++ + if buffer[position] != rune('A') { + goto l52 + } + position++ + if buffer[position] != rune('t') { + goto l52 + } + position++ + if buffer[position] != rune('t') { + goto l52 + } + position++ + if buffer[position] != rune('r') { + goto l52 + } + position++ + if buffer[position] != rune('s') { + goto l52 + } + position++ + if buffer[position] != rune('(') { + goto l52 + } + position++ + goto l50 + l52: + position, tokenIndex = position50, tokenIndex50 + if buffer[position] != rune('S') { + goto l53 + } + position++ + if buffer[position] != rune('e') { + goto l53 + } + position++ + if buffer[position] != rune('t') { + goto l53 + } + position++ + if buffer[position] != rune('C') { + goto l53 + } + position++ + if buffer[position] != rune('o') { + goto l53 + } + position++ + if buffer[position] != rune('l') { + goto l53 + } + position++ + if buffer[position] != rune('A') { + goto l53 + } + position++ + if buffer[position] != rune('t') { + goto l53 + } + position++ + if buffer[position] != rune('t') { + goto l53 + } + position++ + if buffer[position] != rune('r') { + goto l53 + } + position++ + if buffer[position] != rune('s') { + goto l53 + } + position++ + if buffer[position] != rune('(') { + goto l53 + } + position++ + goto l50 + l53: + position, tokenIndex = position50, tokenIndex50 + if buffer[position] != rune('C') { + goto l54 + } + position++ + if buffer[position] != rune('l') { + goto l54 + } + position++ + if buffer[position] != rune('e') { + goto l54 + } + position++ + if buffer[position] != rune('a') { + goto l54 + } + position++ + if buffer[position] != rune('r') { + goto l54 + } + position++ + if buffer[position] != rune('(') { + goto l54 + } + position++ + goto l50 + l54: + position, tokenIndex = position50, tokenIndex50 + if buffer[position] != rune('T') { + goto l55 + } + position++ + if buffer[position] != rune('o') { + goto l55 + } + position++ + if buffer[position] != rune('p') { + goto l55 + } + position++ + if buffer[position] != rune('N') { + goto l55 + } + position++ + if buffer[position] != rune('(') { + goto l55 + } + position++ + goto l50 + l55: + position, tokenIndex = position50, tokenIndex50 + if buffer[position] != rune('R') { + goto l49 + } + position++ + if buffer[position] != rune('a') { + goto l49 + } + position++ + if buffer[position] != rune('n') { + goto l49 + } + position++ + if buffer[position] != rune('g') { + goto l49 + } + position++ + if buffer[position] != rune('e') { + goto l49 + } + position++ + if buffer[position] != rune('(') { + goto l49 + } + position++ + } + l50: + goto l5 + l49: + position, tokenIndex = position49, tokenIndex49 + } + { + position56, tokenIndex56 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l57 } position++ - goto l49 - l50: - position, tokenIndex = position49, tokenIndex49 + goto l56 + l57: + position, tokenIndex = position56, tokenIndex56 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l49: - l51: + l56: + l58: { - position52, tokenIndex52 := position, tokenIndex + position59, tokenIndex59 := position, tokenIndex { - position53, tokenIndex53 := position, tokenIndex + position60, tokenIndex60 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l54 + goto l61 } position++ - goto l53 - l54: - position, tokenIndex = position53, tokenIndex53 + goto l60 + l61: + position, tokenIndex = position60, tokenIndex60 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l55 + goto l62 } position++ - goto l53 - l55: - position, tokenIndex = position53, tokenIndex53 + goto l60 + l62: + position, tokenIndex = position60, tokenIndex60 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l56 - } - position++ - goto l53 - l56: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('-') { - goto l57 - } - position++ - goto l53 - l57: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('_') { - goto l58 - } - position++ - goto l53 - l58: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('.') { - goto l52 + goto l59 } position++ } - l53: - goto l51 - l52: - position, tokenIndex = position52, tokenIndex52 + l60: + goto l58 + l59: + position, tokenIndex = position59, tokenIndex59 } add(ruleIDENT, position48) } @@ -1131,75 +1303,19 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l5 } - { - position60 := position - { - position61, tokenIndex61 := position, tokenIndex - if !_rules[ruleCall]() { - goto l62 - } - l63: - { - position64, tokenIndex64 := position, tokenIndex - if !_rules[rulecomma]() { - goto l64 - } - if !_rules[ruleCall]() { - goto l64 - } - goto l63 - l64: - position, tokenIndex = position64, tokenIndex64 - } - { - position65, tokenIndex65 := position, tokenIndex - if !_rules[rulecomma]() { - goto l65 - } - if !_rules[ruleargs]() { - goto l65 - } - goto l66 - l65: - position, tokenIndex = position65, tokenIndex65 - } - l66: - goto l61 - l62: - position, tokenIndex = position61, tokenIndex61 - { - position68, tokenIndex68 := position, tokenIndex - if !_rules[rulecomma]() { - goto l68 - } - goto l69 - l68: - position, tokenIndex = position68, tokenIndex68 - } - l69: - if !_rules[ruleargs]() { - goto l67 - } - goto l61 - l67: - position, tokenIndex = position61, tokenIndex61 - if !_rules[rulesp]() { - goto l5 - } - } - l61: - add(ruleallargs, position60) + if !_rules[ruleallargs]() { + goto l5 } { - position70, tokenIndex70 := position, tokenIndex + position64, tokenIndex64 := position, tokenIndex if !_rules[rulecomma]() { - goto l70 + goto l64 } - goto l71 - l70: - position, tokenIndex = position70, tokenIndex70 + goto l65 + l64: + position, tokenIndex = position64, tokenIndex64 } - l71: + l65: if !_rules[ruleclose]() { goto l5 } @@ -1215,205 +1331,241 @@ func (p *PQL) Init() { position, tokenIndex = position5, tokenIndex5 return false }, - /* 2 allargs <- <((Call (comma Call)* (comma args)?) / (comma? args) / sp)> */ - nil, - /* 3 args <- <(arg (comma args)? sp)> */ + /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position74, tokenIndex74 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex { - position75 := position - if !_rules[rulearg]() { - goto l74 - } + position68 := position { - position76, tokenIndex76 := position, tokenIndex - if !_rules[rulecomma]() { - goto l76 + position69, tokenIndex69 := position, tokenIndex + if !_rules[ruleCall]() { + goto l70 } + l71: + { + position72, tokenIndex72 := position, tokenIndex + if !_rules[rulecomma]() { + goto l72 + } + if !_rules[ruleCall]() { + goto l72 + } + goto l71 + l72: + position, tokenIndex = position72, tokenIndex72 + } + { + position73, tokenIndex73 := position, tokenIndex + if !_rules[rulecomma]() { + goto l73 + } + if !_rules[ruleargs]() { + goto l73 + } + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 + } + l74: + goto l69 + l70: + position, tokenIndex = position69, tokenIndex69 if !_rules[ruleargs]() { - goto l76 + goto l75 + } + goto l69 + l75: + position, tokenIndex = position69, tokenIndex69 + if !_rules[rulesp]() { + goto l67 } - goto l77 - l76: - position, tokenIndex = position76, tokenIndex76 } - l77: - if !_rules[rulesp]() { - goto l74 - } - add(ruleargs, position75) + l69: + add(ruleallargs, position68) } return true - l74: - position, tokenIndex = position74, tokenIndex74 + l67: + position, tokenIndex = position67, tokenIndex67 + return false + }, + /* 3 args <- <(arg (comma args)? sp)> */ + func() bool { + position76, tokenIndex76 := position, tokenIndex + { + position77 := position + if !_rules[rulearg]() { + goto l76 + } + { + position78, tokenIndex78 := position, tokenIndex + if !_rules[rulecomma]() { + goto l78 + } + if !_rules[ruleargs]() { + goto l78 + } + goto l79 + l78: + position, tokenIndex = position78, tokenIndex78 + } + l79: + if !_rules[rulesp]() { + goto l76 + } + add(ruleargs, position77) + } + return true + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position78, tokenIndex78 := position, tokenIndex + position80, tokenIndex80 := position, tokenIndex { - position79 := position + position81 := position { - position80, tokenIndex80 := position, tokenIndex + position82, tokenIndex82 := position, tokenIndex if !_rules[rulefield]() { - goto l81 + goto l83 } if !_rules[rulesp]() { - goto l81 + goto l83 } if buffer[position] != rune('=') { - goto l81 + goto l83 } position++ if !_rules[rulesp]() { - goto l81 + goto l83 } if !_rules[rulevalue]() { - goto l81 + goto l83 } - goto l80 - l81: - position, tokenIndex = position80, tokenIndex80 + goto l82 + l83: + position, tokenIndex = position82, tokenIndex82 if !_rules[rulefield]() { - goto l78 + goto l80 } if !_rules[rulesp]() { - goto l78 + goto l80 } { - position82 := position + position84 := position { - position83, tokenIndex83 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex if buffer[position] != rune('>') { - goto l84 + goto l86 } position++ if buffer[position] != rune('<') { - goto l84 + goto l86 } position++ { add(ruleAction14, position) } - goto l83 - l84: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l86: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('<') { - goto l86 + goto l88 } position++ if buffer[position] != rune('=') { - goto l86 + goto l88 } position++ { add(ruleAction15, position) } - goto l83 - l86: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l88: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('>') { - goto l88 + goto l90 } position++ if buffer[position] != rune('=') { - goto l88 + goto l90 } position++ { add(ruleAction16, position) } - goto l83 - l88: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l90: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('=') { - goto l90 + goto l92 } position++ if buffer[position] != rune('=') { - goto l90 + goto l92 } position++ { add(ruleAction17, position) } - goto l83 - l90: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l92: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('!') { - goto l92 + goto l94 } position++ if buffer[position] != rune('=') { - goto l92 + goto l94 } position++ { add(ruleAction18, position) } - goto l83 - l92: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l94: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('<') { - goto l94 + goto l96 } position++ { add(ruleAction19, position) } - goto l83 - l94: - position, tokenIndex = position83, tokenIndex83 + goto l85 + l96: + position, tokenIndex = position85, tokenIndex85 if buffer[position] != rune('>') { - goto l78 + goto l80 } position++ { add(ruleAction20, position) } } - l83: - add(ruleCOND, position82) + l85: + add(ruleCOND, position84) } if !_rules[rulesp]() { - goto l78 + goto l80 } if !_rules[rulevalue]() { - goto l78 + goto l80 } } - l80: - add(rulearg, position79) + l82: + add(rulearg, position81) } return true - l78: - position, tokenIndex = position78, tokenIndex78 + l80: + position, tokenIndex = position80, tokenIndex80 return false }, /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ nil, /* 6 conditional <- <(Action21 int (('<' '=') / '<') fieldExpr (('<' '=') / '<') int Action22)> */ nil, - /* 7 open <- <('(' sp)> */ - func() bool { - position99, tokenIndex99 := position, tokenIndex - { - position100 := position - if buffer[position] != rune('(') { - goto l99 - } - position++ - if !_rules[rulesp]() { - goto l99 - } - add(ruleopen, position100) - } - return true - l99: - position, tokenIndex = position99, tokenIndex99 - return false - }, - /* 8 value <- <(item / (lbrack Action23 list rbrack Action24))> */ + /* 7 value <- <(item / (lbrack Action23 list rbrack Action24))> */ func() bool { position101, tokenIndex101 := position, tokenIndex { @@ -1469,7 +1621,7 @@ func (p *PQL) Init() { position, tokenIndex = position101, tokenIndex101 return false }, - /* 9 list <- <(item (comma list)?)> */ + /* 8 list <- <(item (comma list)?)> */ func() bool { position109, tokenIndex109 := position, tokenIndex { @@ -1497,7 +1649,7 @@ func (p *PQL) Init() { position, tokenIndex = position109, tokenIndex109 return false }, - /* 10 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action25) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action26) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action27) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action28) / (<('-'? '.' [0-9]+)> Action29) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action30) / ('"' '"' Action31) / ('\'' '\'' Action32))> */ + /* 9 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action25) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action26) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action27) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action28) / (<('-'? '.' [0-9]+)> Action29) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action30) / ('"' '"' Action31) / ('\'' '\'' Action32))> */ func() bool { position113, tokenIndex113 := position, tokenIndex { @@ -2057,11 +2209,11 @@ func (p *PQL) Init() { position, tokenIndex = position113, tokenIndex113 return false }, - /* 11 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 10 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 12 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 11 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 13 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ + /* 12 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ func() bool { position197, tokenIndex197 := position, tokenIndex { @@ -2124,7 +2276,7 @@ func (p *PQL) Init() { position, tokenIndex = position197, tokenIndex197 return false }, - /* 14 field <- <( Action33)> */ + /* 13 field <- <( Action33)> */ func() bool { position207, tokenIndex207 := position, tokenIndex { @@ -2146,7 +2298,7 @@ func (p *PQL) Init() { position, tokenIndex = position207, tokenIndex207 return false }, - /* 15 posfield <- <( Action34)> */ + /* 14 posfield <- <( Action34)> */ func() bool { position211, tokenIndex211 := position, tokenIndex { @@ -2168,7 +2320,7 @@ func (p *PQL) Init() { position, tokenIndex = position211, tokenIndex211 return false }, - /* 16 uint <- <(([1-9] [0-9]*) / '0')> */ + /* 15 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { position215, tokenIndex215 := position, tokenIndex { @@ -2206,7 +2358,7 @@ func (p *PQL) Init() { position, tokenIndex = position215, tokenIndex215 return false }, - /* 17 int <- <(('-'? [1-9] [0-9]*) / '0')> */ + /* 16 int <- <(('-'? [1-9] [0-9]*) / '0')> */ func() bool { position221, tokenIndex221 := position, tokenIndex { @@ -2255,9 +2407,9 @@ func (p *PQL) Init() { position, tokenIndex = position221, tokenIndex221 return false }, - /* 18 uintrow <- <( Action35)> */ + /* 17 uintrow <- <( Action35)> */ nil, - /* 19 uintcol <- <( Action36)> */ + /* 18 uintcol <- <( Action36)> */ func() bool { position230, tokenIndex230 := position, tokenIndex { @@ -2279,75 +2431,94 @@ func (p *PQL) Init() { position, tokenIndex = position230, tokenIndex230 return false }, - /* 20 close <- <(')' sp)> */ + /* 19 open <- <('(' sp)> */ func() bool { position234, tokenIndex234 := position, tokenIndex { position235 := position - if buffer[position] != rune(')') { + if buffer[position] != rune('(') { goto l234 } position++ if !_rules[rulesp]() { goto l234 } - add(ruleclose, position235) + add(ruleopen, position235) } return true l234: position, tokenIndex = position234, tokenIndex234 return false }, + /* 20 close <- <(')' sp)> */ + func() bool { + position236, tokenIndex236 := position, tokenIndex + { + position237 := position + if buffer[position] != rune(')') { + goto l236 + } + position++ + if !_rules[rulesp]() { + goto l236 + } + add(ruleclose, position237) + } + return true + l236: + position, tokenIndex = position236, tokenIndex236 + return false + }, /* 21 sp <- <(' ' / '\t')*> */ func() bool { { - position237 := position - l238: + position239 := position + l240: { - position239, tokenIndex239 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex { - position240, tokenIndex240 := position, tokenIndex + position242, tokenIndex242 := position, tokenIndex if buffer[position] != rune(' ') { + goto l243 + } + position++ + goto l242 + l243: + position, tokenIndex = position242, tokenIndex242 + if buffer[position] != rune('\t') { goto l241 } position++ - goto l240 - l241: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('\t') { - goto l239 - } - position++ } - l240: - goto l238 - l239: - position, tokenIndex = position239, tokenIndex239 + l242: + goto l240 + l241: + position, tokenIndex = position241, tokenIndex241 } - add(rulesp, position237) + add(rulesp, position239) } return true }, /* 22 comma <- <(sp ',' whitesp)> */ func() bool { - position242, tokenIndex242 := position, tokenIndex + position244, tokenIndex244 := position, tokenIndex { - position243 := position + position245 := position if !_rules[rulesp]() { - goto l242 + goto l244 } if buffer[position] != rune(',') { - goto l242 + goto l244 } position++ if !_rules[rulewhitesp]() { - goto l242 + goto l244 } - add(rulecomma, position243) + add(rulecomma, position245) } return true - l242: - position, tokenIndex = position242, tokenIndex242 + l244: + position, tokenIndex = position244, tokenIndex244 return false }, /* 23 lbrack <- <('[' sp)> */ @@ -2357,41 +2528,41 @@ func (p *PQL) Init() { /* 25 whitesp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position247 := position - l248: + position249 := position + l250: { - position249, tokenIndex249 := position, tokenIndex + position251, tokenIndex251 := position, tokenIndex { - position250, tokenIndex250 := position, tokenIndex + position252, tokenIndex252 := position, tokenIndex if buffer[position] != rune(' ') { + goto l253 + } + position++ + goto l252 + l253: + position, tokenIndex = position252, tokenIndex252 + if buffer[position] != rune('\t') { + goto l254 + } + position++ + goto l252 + l254: + position, tokenIndex = position252, tokenIndex252 + if buffer[position] != rune('\n') { goto l251 } position++ - goto l250 - l251: - position, tokenIndex = position250, tokenIndex250 - if buffer[position] != rune('\t') { - goto l252 - } - position++ - goto l250 - l252: - position, tokenIndex = position250, tokenIndex250 - if buffer[position] != rune('\n') { - goto l249 - } - position++ } - l250: - goto l248 - l249: - position, tokenIndex = position249, tokenIndex249 + l252: + goto l250 + l251: + position, tokenIndex = position251, tokenIndex251 } - add(rulewhitesp, position247) + add(rulewhitesp, position249) } return true }, - /* 26 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '-' / '_' / '.')*)> */ + /* 26 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, /* 27 timestamp <- <(<([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> Action37)> */ nil, @@ -2407,7 +2578,7 @@ func (p *PQL) Init() { nil, /* 34 Action5 <- <{p.endCall()}> */ nil, - /* 35 Action6 <- <{p.startCall("ClearBit")}> */ + /* 35 Action6 <- <{p.startCall("Clear")}> */ nil, /* 36 Action7 <- <{p.endCall()}> */ nil, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 2288c3aeb..d3b5591e8 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -1,12 +1,13 @@ package pql import ( + "strconv" "testing" ) func TestPEG(t *testing.T) { p := PQL{Buffer: ` -SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]} +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} p.Init() err := p.Parse() if err != nil { @@ -21,11 +22,11 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 t.Fatalf("should have been an error because of the interior unescaped double quote") } - q, err := ParseString("TopN(Bitmap(id==other), field=f, n=0)") + q, err := ParseString("TopN(blah, Bitmap(id==other), field=f, n=0)") if err != nil { t.Fatalf("should have parsed: %v", err) } - if q.String() != `TopN(Bitmap(id == "other"), field="f", n=0)` { + if q.String() != `TopN(Bitmap(id == "other"), _field="blah", field="f", n=0)` { t.Fatalf("Failed, got: %s", q) } @@ -44,3 +45,154 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 } } + +func TestPEGWorking(t *testing.T) { + tests := []struct { + name string + input string + ncalls int + }{ + { + name: "Empty", + input: "", + ncalls: 0}, + { + name: "Set", + input: "Set(1, a=4)", + ncalls: 1}, + { + name: "DoubleSet", + input: "Set(1, a=4)Set(2, a=4)", + ncalls: 2}, + { + name: "DoubleSetSpc", + input: "Set(1, a=4) Set(2, a=4)", + ncalls: 2}, + { + name: "DoubleSetNewline", + input: "Set(1, a=4) \n Set(2, a=4)", + ncalls: 2}, + { + name: "SetWithArbCall", + input: "Set(1, a=4)Blerg(z=ha)", + ncalls: 2}, + { + name: "SetArbSet", + input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)", + ncalls: 3}, + { + name: "ArbSetArb", + input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)", + ncalls: 3}, + { + name: "SetStringArg", + input: "Set(1, a=zoom)", + ncalls: 1}, + { + name: "SetManyArgs", + input: "Set(1, a=4, b=5)", + ncalls: 1}, + { + name: "SetManyMixedArgs", + input: "Set(1, a=4, bsd=haha)", + ncalls: 1}, + { + name: "SetTimestamp", + input: "Set(1, a=4, 2017-04-03T19:34)", + ncalls: 1}, + { + name: "Union()", + input: "Union()", + ncalls: 1}, + { + name: "UnionOneRow", + input: "Union(Row(a=1))", + ncalls: 1}, + { + name: "UnionTwoRows", + input: "Union(Row(a=1), Row(z=44))", + ncalls: 1}, + { + name: "UnionNested", + input: "Union(Intersect(Row(), Union(Row(), Row())), Row())", + ncalls: 1}, + { + name: "TopN no args", + input: "TopN(boondoggle)", + ncalls: 1}, + { + name: "TopN with args", + input: "TopN(boon, doggle=9)", + ncalls: 1}, + { + name: "double quoted args", + input: `B(a="zm''e")`, + ncalls: 1}, + { + name: "single quoted args", + input: `B(a='zm""e')`, + ncalls: 1}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.input) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.input, err) + } + if len(q.Calls) != test.ncalls { + t.Fatalf("wrong number of calls for '%s': %#v", test.input, q.Calls) + } + }) + } +} + +func TestPEGErrors(t *testing.T) { + tests := []struct { + name string + input string + }{ + { + name: "SetEmpty", + input: "Set()"}, + { + name: "SetNoCol", + input: "Set(a=4)"}, + { + name: "SetNoParens", + input: "Set"}, + { + name: "SetBadTimestamp", + input: "Set(1, a=4, 2017-94-03T19:34)"}, + { + name: "SetTimestampNoArg", + input: "Set(1, 2017-04-03T19:34)"}, + { + name: "SetRowAttrsNoField", + input: "SetRowAttrs(a=4)"}, + { + name: "SetColAttrsNoField", + input: "SetColAttrs(a=4)"}, + { + name: "ClearNoCol", + input: "Clear(a=4)"}, + { + name: "SetStartingComma", + input: "Set(, 1, a=4)"}, + { + name: "StartinCommaArb", + input: "Zeeb(, a=4)"}, + { + name: "TopN No Field", + input: "TopN(a=77)"}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.input) + if err == nil { + t.Fatalf("parsing query '%s' - expected error, got: %s", test.input, q) + } + }) + } +} From a6b6442ef43f6cefe0cd912a34d0aa1e9838ada7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 11:32:21 -0500 Subject: [PATCH 08/33] more tests and fix range --- pql/ast.go | 4 + pql/pql.peg | 7 +- pql/pql.peg.go | 2323 +++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 78 ++ 4 files changed, 1277 insertions(+), 1135 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 0be4d7034..c5d40a4b9 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -63,6 +63,10 @@ func (q *Query) addPosStr(key, value string) { func (q *Query) startConditional() { q.conditional = make([]string, 0) + call := q.callStack[len(q.callStack)-1] + if call.Args == nil { + call.Args = make(map[string]interface{}) + } } func (q *Query) condAdd(val string) { diff --git a/pql/pql.peg b/pql/pql.peg index 27c15cb25..1911dc756 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -26,7 +26,11 @@ COND <- ( '><' { p.addBTWN() } / '<' { p.addLT() } / '>' { p.addGT() } ) -conditional <- {p.startConditional()} int ('<=' / '<') fieldExpr ('<=' / '<') int {p.endConditional()} +conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} +condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])} +condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} +condfield <- sp {p.condAdd(buffer[begin:end])} + value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) @@ -48,7 +52,6 @@ fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* field <- { p.addField(buffer[begin:end]) } posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' -int <- '-'? [1-9] [0-9]* / '0' uintrow <- {p.addPosNum("_row", buffer[begin:end])} uintcol <- {p.addPosNum("_col", buffer[begin:end])} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 303d6915d..356b2b1b3 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -23,6 +23,9 @@ const ( rulearg ruleCOND ruleconditional + rulecondint + rulecondLT + rulecondfield rulevalue rulelist ruleitem @@ -32,7 +35,6 @@ const ( rulefield ruleposfield ruleuint - ruleint ruleuintrow ruleuintcol ruleopen @@ -83,6 +85,9 @@ const ( ruleAction35 ruleAction36 ruleAction37 + ruleAction38 + ruleAction39 + ruleAction40 ) var rul3s = [...]string{ @@ -94,6 +99,9 @@ var rul3s = [...]string{ "arg", "COND", "conditional", + "condint", + "condLT", + "condfield", "value", "list", "item", @@ -103,7 +111,6 @@ var rul3s = [...]string{ "field", "posfield", "uint", - "int", "uintrow", "uintcol", "open", @@ -154,6 +161,9 @@ var rul3s = [...]string{ "Action35", "Action36", "Action37", + "Action38", + "Action39", + "Action40", } type token32 struct { @@ -270,7 +280,7 @@ type PQL struct { Buffer string buffer []rune - rules [68]func() bool + rules [73]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -409,34 +419,40 @@ func (p *PQL) Execute() { case ruleAction22: p.endConditional() case ruleAction23: - p.startList() + p.condAdd(buffer[begin:end]) case ruleAction24: - p.endList() + p.condAdd(buffer[begin:end]) case ruleAction25: - p.addVal(nil) + p.condAdd(buffer[begin:end]) case ruleAction26: - p.addVal(true) + p.startList() case ruleAction27: - p.addVal(false) + p.endList() case ruleAction28: - p.addNumVal(buffer[begin:end]) + p.addVal(nil) case ruleAction29: - p.addNumVal(buffer[begin:end]) + p.addVal(true) case ruleAction30: - p.addVal(buffer[begin:end]) + p.addVal(false) case ruleAction31: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction32: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction33: - p.addField(buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction34: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction35: - p.addPosNum("_row", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction36: - p.addPosNum("_col", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction37: + p.addPosStr("_field", buffer[begin:end]) + case ruleAction38: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction39: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction40: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -670,7 +686,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction37, position) + add(ruleAction40, position) } add(ruletimestamp, position12) } @@ -754,7 +770,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction35, position) + add(ruleAction38, position) } add(ruleuintrow, position20) } @@ -977,51 +993,33 @@ func (p *PQL) Init() { { add(ruleAction21, position) } - if !_rules[ruleint]() { + if !_rules[rulecondint]() { + goto l35 + } + if !_rules[rulecondLT]() { goto l35 } { - position41, tokenIndex41 := position, tokenIndex - if buffer[position] != rune('<') { - goto l42 + position41 := position + { + position42 := position + if !_rules[rulefieldExpr]() { + goto l35 + } + add(rulePegText, position42) } - position++ - if buffer[position] != rune('=') { - goto l42 - } - position++ - goto l41 - l42: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('<') { + if !_rules[rulesp]() { goto l35 } - position++ + { + add(ruleAction25, position) + } + add(rulecondfield, position41) } - l41: - if !_rules[rulefieldExpr]() { + if !_rules[rulecondLT]() { goto l35 } - { - position43, tokenIndex43 := position, tokenIndex - if buffer[position] != rune('<') { - goto l44 - } - position++ - if buffer[position] != rune('=') { - goto l44 - } - position++ - goto l43 - l44: - position, tokenIndex = position43, tokenIndex43 - if buffer[position] != rune('<') { - goto l35 - } - position++ - } - l43: - if !_rules[ruleint]() { + if !_rules[rulecondint]() { goto l35 } { @@ -1041,261 +1039,261 @@ func (p *PQL) Init() { l35: position, tokenIndex = position7, tokenIndex7 { - position47 := position + position46 := position { - position48 := position + position47 := position { - position49, tokenIndex49 := position, tokenIndex + position48, tokenIndex48 := position, tokenIndex { - position50, tokenIndex50 := position, tokenIndex + position49, tokenIndex49 := position, tokenIndex if buffer[position] != rune('S') { - goto l51 + goto l50 } position++ if buffer[position] != rune('e') { - goto l51 + goto l50 } position++ if buffer[position] != rune('t') { - goto l51 + goto l50 } position++ if buffer[position] != rune('(') { + goto l50 + } + position++ + goto l49 + l50: + position, tokenIndex = position49, tokenIndex49 + if buffer[position] != rune('S') { goto l51 } position++ - goto l50 - l51: - position, tokenIndex = position50, tokenIndex50 - if buffer[position] != rune('S') { - goto l52 - } - position++ if buffer[position] != rune('e') { - goto l52 + goto l51 } position++ if buffer[position] != rune('t') { - goto l52 + goto l51 } position++ if buffer[position] != rune('R') { - goto l52 + goto l51 } position++ if buffer[position] != rune('o') { - goto l52 + goto l51 } position++ if buffer[position] != rune('w') { - goto l52 + goto l51 } position++ if buffer[position] != rune('A') { - goto l52 + goto l51 } position++ if buffer[position] != rune('t') { - goto l52 + goto l51 } position++ if buffer[position] != rune('t') { - goto l52 + goto l51 } position++ if buffer[position] != rune('r') { - goto l52 + goto l51 } position++ if buffer[position] != rune('s') { - goto l52 + goto l51 } position++ if buffer[position] != rune('(') { + goto l51 + } + position++ + goto l49 + l51: + position, tokenIndex = position49, tokenIndex49 + if buffer[position] != rune('S') { goto l52 } position++ - goto l50 - l52: - position, tokenIndex = position50, tokenIndex50 - if buffer[position] != rune('S') { - goto l53 - } - position++ if buffer[position] != rune('e') { - goto l53 + goto l52 } position++ if buffer[position] != rune('t') { - goto l53 + goto l52 } position++ if buffer[position] != rune('C') { - goto l53 + goto l52 } position++ if buffer[position] != rune('o') { - goto l53 + goto l52 } position++ if buffer[position] != rune('l') { - goto l53 + goto l52 } position++ if buffer[position] != rune('A') { - goto l53 + goto l52 } position++ if buffer[position] != rune('t') { - goto l53 + goto l52 } position++ if buffer[position] != rune('t') { - goto l53 + goto l52 } position++ if buffer[position] != rune('r') { - goto l53 + goto l52 } position++ if buffer[position] != rune('s') { - goto l53 + goto l52 } position++ if buffer[position] != rune('(') { + goto l52 + } + position++ + goto l49 + l52: + position, tokenIndex = position49, tokenIndex49 + if buffer[position] != rune('C') { goto l53 } position++ - goto l50 - l53: - position, tokenIndex = position50, tokenIndex50 - if buffer[position] != rune('C') { - goto l54 - } - position++ if buffer[position] != rune('l') { - goto l54 + goto l53 } position++ if buffer[position] != rune('e') { - goto l54 + goto l53 } position++ if buffer[position] != rune('a') { - goto l54 + goto l53 } position++ if buffer[position] != rune('r') { - goto l54 + goto l53 } position++ if buffer[position] != rune('(') { + goto l53 + } + position++ + goto l49 + l53: + position, tokenIndex = position49, tokenIndex49 + if buffer[position] != rune('T') { goto l54 } position++ - goto l50 - l54: - position, tokenIndex = position50, tokenIndex50 - if buffer[position] != rune('T') { - goto l55 - } - position++ if buffer[position] != rune('o') { - goto l55 + goto l54 } position++ if buffer[position] != rune('p') { - goto l55 + goto l54 } position++ if buffer[position] != rune('N') { - goto l55 + goto l54 } position++ if buffer[position] != rune('(') { - goto l55 + goto l54 } position++ - goto l50 - l55: - position, tokenIndex = position50, tokenIndex50 + goto l49 + l54: + position, tokenIndex = position49, tokenIndex49 if buffer[position] != rune('R') { - goto l49 + goto l48 } position++ if buffer[position] != rune('a') { - goto l49 + goto l48 } position++ if buffer[position] != rune('n') { - goto l49 + goto l48 } position++ if buffer[position] != rune('g') { - goto l49 + goto l48 } position++ if buffer[position] != rune('e') { - goto l49 + goto l48 } position++ if buffer[position] != rune('(') { - goto l49 + goto l48 } position++ } - l50: - goto l5 l49: - position, tokenIndex = position49, tokenIndex49 + goto l5 + l48: + position, tokenIndex = position48, tokenIndex48 } { - position56, tokenIndex56 := position, tokenIndex + position55, tokenIndex55 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l57 + goto l56 } position++ - goto l56 - l57: - position, tokenIndex = position56, tokenIndex56 + goto l55 + l56: + position, tokenIndex = position55, tokenIndex55 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l56: - l58: + l55: + l57: { - position59, tokenIndex59 := position, tokenIndex + position58, tokenIndex58 := position, tokenIndex { - position60, tokenIndex60 := position, tokenIndex + position59, tokenIndex59 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + goto l59 + l60: + position, tokenIndex = position59, tokenIndex59 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l61 } position++ - goto l60 + goto l59 l61: - position, tokenIndex = position60, tokenIndex60 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l62 - } - position++ - goto l60 - l62: - position, tokenIndex = position60, tokenIndex60 + position, tokenIndex = position59, tokenIndex59 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l59 + goto l58 } position++ } - l60: - goto l58 l59: - position, tokenIndex = position59, tokenIndex59 + goto l57 + l58: + position, tokenIndex = position58, tokenIndex58 } - add(ruleIDENT, position48) + add(ruleIDENT, position47) } - add(rulePegText, position47) + add(rulePegText, position46) } { add(ruleAction12, position) @@ -1307,15 +1305,15 @@ func (p *PQL) Init() { goto l5 } { - position64, tokenIndex64 := position, tokenIndex + position63, tokenIndex63 := position, tokenIndex if !_rules[rulecomma]() { - goto l64 + goto l63 } - goto l65 - l64: - position, tokenIndex = position64, tokenIndex64 + goto l64 + l63: + position, tokenIndex = position63, tokenIndex63 } - l65: + l64: if !_rules[ruleclose]() { goto l5 } @@ -1333,1315 +1331,1374 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position67, tokenIndex67 := position, tokenIndex + position66, tokenIndex66 := position, tokenIndex { - position68 := position + position67 := position { - position69, tokenIndex69 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex if !_rules[ruleCall]() { - goto l70 + goto l69 + } + l70: + { + position71, tokenIndex71 := position, tokenIndex + if !_rules[rulecomma]() { + goto l71 + } + if !_rules[ruleCall]() { + goto l71 + } + goto l70 + l71: + position, tokenIndex = position71, tokenIndex71 } - l71: { position72, tokenIndex72 := position, tokenIndex if !_rules[rulecomma]() { goto l72 } - if !_rules[ruleCall]() { + if !_rules[ruleargs]() { goto l72 } - goto l71 + goto l73 l72: position, tokenIndex = position72, tokenIndex72 } - { - position73, tokenIndex73 := position, tokenIndex - if !_rules[rulecomma]() { - goto l73 - } - if !_rules[ruleargs]() { - goto l73 - } - goto l74 - l73: - position, tokenIndex = position73, tokenIndex73 - } - l74: - goto l69 - l70: - position, tokenIndex = position69, tokenIndex69 + l73: + goto l68 + l69: + position, tokenIndex = position68, tokenIndex68 if !_rules[ruleargs]() { - goto l75 + goto l74 } - goto l69 - l75: - position, tokenIndex = position69, tokenIndex69 + goto l68 + l74: + position, tokenIndex = position68, tokenIndex68 if !_rules[rulesp]() { - goto l67 + goto l66 } } - l69: - add(ruleallargs, position68) + l68: + add(ruleallargs, position67) } return true - l67: - position, tokenIndex = position67, tokenIndex67 + l66: + position, tokenIndex = position66, tokenIndex66 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position76, tokenIndex76 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex { - position77 := position + position76 := position if !_rules[rulearg]() { - goto l76 + goto l75 } { - position78, tokenIndex78 := position, tokenIndex + position77, tokenIndex77 := position, tokenIndex if !_rules[rulecomma]() { - goto l78 + goto l77 } if !_rules[ruleargs]() { - goto l78 + goto l77 } - goto l79 - l78: - position, tokenIndex = position78, tokenIndex78 + goto l78 + l77: + position, tokenIndex = position77, tokenIndex77 } - l79: + l78: if !_rules[rulesp]() { - goto l76 + goto l75 } - add(ruleargs, position77) + add(ruleargs, position76) } return true - l76: - position, tokenIndex = position76, tokenIndex76 + l75: + position, tokenIndex = position75, tokenIndex75 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position80, tokenIndex80 := position, tokenIndex + position79, tokenIndex79 := position, tokenIndex { - position81 := position + position80 := position { - position82, tokenIndex82 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex if !_rules[rulefield]() { - goto l83 + goto l82 } if !_rules[rulesp]() { - goto l83 + goto l82 } if buffer[position] != rune('=') { - goto l83 + goto l82 } position++ if !_rules[rulesp]() { - goto l83 + goto l82 } if !_rules[rulevalue]() { - goto l83 + goto l82 } - goto l82 - l83: - position, tokenIndex = position82, tokenIndex82 + goto l81 + l82: + position, tokenIndex = position81, tokenIndex81 if !_rules[rulefield]() { - goto l80 + goto l79 } if !_rules[rulesp]() { - goto l80 + goto l79 } { - position84 := position + position83 := position { - position85, tokenIndex85 := position, tokenIndex + position84, tokenIndex84 := position, tokenIndex if buffer[position] != rune('>') { - goto l86 + goto l85 } position++ if buffer[position] != rune('<') { - goto l86 + goto l85 } position++ { add(ruleAction14, position) } - goto l85 - l86: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l85: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('<') { - goto l88 + goto l87 } position++ if buffer[position] != rune('=') { - goto l88 + goto l87 } position++ { add(ruleAction15, position) } - goto l85 - l88: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l87: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('>') { - goto l90 + goto l89 } position++ if buffer[position] != rune('=') { - goto l90 + goto l89 } position++ { add(ruleAction16, position) } - goto l85 - l90: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l89: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('=') { - goto l92 + goto l91 } position++ if buffer[position] != rune('=') { - goto l92 + goto l91 } position++ { add(ruleAction17, position) } - goto l85 - l92: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l91: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('!') { - goto l94 + goto l93 } position++ if buffer[position] != rune('=') { - goto l94 + goto l93 } position++ { add(ruleAction18, position) } - goto l85 - l94: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l93: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('<') { - goto l96 + goto l95 } position++ { add(ruleAction19, position) } - goto l85 - l96: - position, tokenIndex = position85, tokenIndex85 + goto l84 + l95: + position, tokenIndex = position84, tokenIndex84 if buffer[position] != rune('>') { - goto l80 + goto l79 } position++ { add(ruleAction20, position) } } - l85: - add(ruleCOND, position84) + l84: + add(ruleCOND, position83) } if !_rules[rulesp]() { - goto l80 + goto l79 } if !_rules[rulevalue]() { - goto l80 + goto l79 } } - l82: - add(rulearg, position81) + l81: + add(rulearg, position80) } return true - l80: - position, tokenIndex = position80, tokenIndex80 + l79: + position, tokenIndex = position79, tokenIndex79 return false }, /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ nil, - /* 6 conditional <- <(Action21 int (('<' '=') / '<') fieldExpr (('<' '=') / '<') int Action22)> */ + /* 6 conditional <- <(Action21 condint condLT condfield condLT condint Action22)> */ nil, - /* 7 value <- <(item / (lbrack Action23 list rbrack Action24))> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ func() bool { - position101, tokenIndex101 := position, tokenIndex + position100, tokenIndex100 := position, tokenIndex { - position102 := position + position101 := position { - position103, tokenIndex103 := position, tokenIndex + position102 := position + { + position103, tokenIndex103 := position, tokenIndex + { + position105, tokenIndex105 := position, tokenIndex + if buffer[position] != rune('-') { + goto l105 + } + position++ + goto l106 + l105: + position, tokenIndex = position105, tokenIndex105 + } + l106: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l104 + } + position++ + l107: + { + position108, tokenIndex108 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l108 + } + position++ + goto l107 + l108: + position, tokenIndex = position108, tokenIndex108 + } + goto l103 + l104: + position, tokenIndex = position103, tokenIndex103 + if buffer[position] != rune('0') { + goto l100 + } + position++ + } + l103: + add(rulePegText, position102) + } + if !_rules[rulesp]() { + goto l100 + } + { + add(ruleAction23, position) + } + add(rulecondint, position101) + } + return true + l100: + position, tokenIndex = position100, tokenIndex100 + return false + }, + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ + func() bool { + position110, tokenIndex110 := position, tokenIndex + { + position111 := position + { + position112 := position + { + position113, tokenIndex113 := position, tokenIndex + if buffer[position] != rune('<') { + goto l114 + } + position++ + if buffer[position] != rune('=') { + goto l114 + } + position++ + goto l113 + l114: + position, tokenIndex = position113, tokenIndex113 + if buffer[position] != rune('<') { + goto l110 + } + position++ + } + l113: + add(rulePegText, position112) + } + if !_rules[rulesp]() { + goto l110 + } + { + add(ruleAction24, position) + } + add(rulecondLT, position111) + } + return true + l110: + position, tokenIndex = position110, tokenIndex110 + return false + }, + /* 9 condfield <- <( sp Action25)> */ + nil, + /* 10 value <- <(item / (lbrack Action26 list rbrack Action27))> */ + func() bool { + position117, tokenIndex117 := position, tokenIndex + { + position118 := position + { + position119, tokenIndex119 := position, tokenIndex if !_rules[ruleitem]() { - goto l104 + goto l120 } - goto l103 - l104: - position, tokenIndex = position103, tokenIndex103 + goto l119 + l120: + position, tokenIndex = position119, tokenIndex119 { - position105 := position + position121 := position if buffer[position] != rune('[') { - goto l101 + goto l117 } position++ if !_rules[rulesp]() { - goto l101 + goto l117 } - add(rulelbrack, position105) - } - { - add(ruleAction23, position) - } - if !_rules[rulelist]() { - goto l101 - } - { - position107 := position - if !_rules[rulesp]() { - goto l101 - } - if buffer[position] != rune(']') { - goto l101 - } - position++ - if !_rules[rulesp]() { - goto l101 - } - add(rulerbrack, position107) - } - { - add(ruleAction24, position) - } - } - l103: - add(rulevalue, position102) - } - return true - l101: - position, tokenIndex = position101, tokenIndex101 - return false - }, - /* 8 list <- <(item (comma list)?)> */ - func() bool { - position109, tokenIndex109 := position, tokenIndex - { - position110 := position - if !_rules[ruleitem]() { - goto l109 - } - { - position111, tokenIndex111 := position, tokenIndex - if !_rules[rulecomma]() { - goto l111 - } - if !_rules[rulelist]() { - goto l111 - } - goto l112 - l111: - position, tokenIndex = position111, tokenIndex111 - } - l112: - add(rulelist, position110) - } - return true - l109: - position, tokenIndex = position109, tokenIndex109 - return false - }, - /* 9 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action25) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action26) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action27) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action28) / (<('-'? '.' [0-9]+)> Action29) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action30) / ('"' '"' Action31) / ('\'' '\'' Action32))> */ - func() bool { - position113, tokenIndex113 := position, tokenIndex - { - position114 := position - { - position115, tokenIndex115 := position, tokenIndex - if buffer[position] != rune('n') { - goto l116 - } - position++ - if buffer[position] != rune('u') { - goto l116 - } - position++ - if buffer[position] != rune('l') { - goto l116 - } - position++ - if buffer[position] != rune('l') { - goto l116 - } - position++ - { - position117, tokenIndex117 := position, tokenIndex - { - position118, tokenIndex118 := position, tokenIndex - if !_rules[rulecomma]() { - goto l119 - } - goto l118 - l119: - position, tokenIndex = position118, tokenIndex118 - if !_rules[rulesp]() { - goto l116 - } - if !_rules[ruleclose]() { - goto l116 - } - } - l118: - position, tokenIndex = position117, tokenIndex117 - } - { - add(ruleAction25, position) - } - goto l115 - l116: - position, tokenIndex = position115, tokenIndex115 - if buffer[position] != rune('t') { - goto l121 - } - position++ - if buffer[position] != rune('r') { - goto l121 - } - position++ - if buffer[position] != rune('u') { - goto l121 - } - position++ - if buffer[position] != rune('e') { - goto l121 - } - position++ - { - position122, tokenIndex122 := position, tokenIndex - { - position123, tokenIndex123 := position, tokenIndex - if !_rules[rulecomma]() { - goto l124 - } - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 - if !_rules[rulesp]() { - goto l121 - } - if !_rules[ruleclose]() { - goto l121 - } - } - l123: - position, tokenIndex = position122, tokenIndex122 + add(rulelbrack, position121) } { add(ruleAction26, position) } - goto l115 - l121: - position, tokenIndex = position115, tokenIndex115 - if buffer[position] != rune('f') { - goto l126 + if !_rules[rulelist]() { + goto l117 } - position++ - if buffer[position] != rune('a') { - goto l126 - } - position++ - if buffer[position] != rune('l') { - goto l126 - } - position++ - if buffer[position] != rune('s') { - goto l126 - } - position++ - if buffer[position] != rune('e') { - goto l126 - } - position++ { - position127, tokenIndex127 := position, tokenIndex - { - position128, tokenIndex128 := position, tokenIndex - if !_rules[rulecomma]() { - goto l129 - } - goto l128 - l129: - position, tokenIndex = position128, tokenIndex128 - if !_rules[rulesp]() { - goto l126 - } - if !_rules[ruleclose]() { - goto l126 - } + position123 := position + if !_rules[rulesp]() { + goto l117 } - l128: - position, tokenIndex = position127, tokenIndex127 + if buffer[position] != rune(']') { + goto l117 + } + position++ + if !_rules[rulesp]() { + goto l117 + } + add(rulerbrack, position123) } { add(ruleAction27, position) } - goto l115 - l126: - position, tokenIndex = position115, tokenIndex115 + } + l119: + add(rulevalue, position118) + } + return true + l117: + position, tokenIndex = position117, tokenIndex117 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position125, tokenIndex125 := position, tokenIndex + { + position126 := position + if !_rules[ruleitem]() { + goto l125 + } + { + position127, tokenIndex127 := position, tokenIndex + if !_rules[rulecomma]() { + goto l127 + } + if !_rules[rulelist]() { + goto l127 + } + goto l128 + l127: + position, tokenIndex = position127, tokenIndex127 + } + l128: + add(rulelist, position126) + } + return true + l125: + position, tokenIndex = position125, tokenIndex125 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action28) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action29) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action30) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action31) / (<('-'? '.' [0-9]+)> Action32) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action33) / ('"' '"' Action34) / ('\'' '\'' Action35))> */ + func() bool { + position129, tokenIndex129 := position, tokenIndex + { + position130 := position + { + position131, tokenIndex131 := position, tokenIndex + if buffer[position] != rune('n') { + goto l132 + } + position++ + if buffer[position] != rune('u') { + goto l132 + } + position++ + if buffer[position] != rune('l') { + goto l132 + } + position++ + if buffer[position] != rune('l') { + goto l132 + } + position++ { - position132 := position + position133, tokenIndex133 := position, tokenIndex { - position133, tokenIndex133 := position, tokenIndex - if buffer[position] != rune('-') { - goto l133 + position134, tokenIndex134 := position, tokenIndex + if !_rules[rulecomma]() { + goto l135 } - position++ goto l134 - l133: - position, tokenIndex = position133, tokenIndex133 + l135: + position, tokenIndex = position134, tokenIndex134 + if !_rules[rulesp]() { + goto l132 + } + if !_rules[ruleclose]() { + goto l132 + } } l134: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l131 - } - position++ - l135: - { - position136, tokenIndex136 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l136 - } - position++ - goto l135 - l136: - position, tokenIndex = position136, tokenIndex136 - } - { - position137, tokenIndex137 := position, tokenIndex - if buffer[position] != rune('.') { - goto l137 - } - position++ - l139: - { - position140, tokenIndex140 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l140 - } - position++ - goto l139 - l140: - position, tokenIndex = position140, tokenIndex140 - } - goto l138 - l137: - position, tokenIndex = position137, tokenIndex137 - } - l138: - add(rulePegText, position132) + position, tokenIndex = position133, tokenIndex133 } { add(ruleAction28, position) } - goto l115 - l131: - position, tokenIndex = position115, tokenIndex115 + goto l131 + l132: + position, tokenIndex = position131, tokenIndex131 + if buffer[position] != rune('t') { + goto l137 + } + position++ + if buffer[position] != rune('r') { + goto l137 + } + position++ + if buffer[position] != rune('u') { + goto l137 + } + position++ + if buffer[position] != rune('e') { + goto l137 + } + position++ { - position143 := position + position138, tokenIndex138 := position, tokenIndex { - position144, tokenIndex144 := position, tokenIndex - if buffer[position] != rune('-') { - goto l144 + position139, tokenIndex139 := position, tokenIndex + if !_rules[rulecomma]() { + goto l140 } - position++ - goto l145 - l144: - position, tokenIndex = position144, tokenIndex144 - } - l145: - if buffer[position] != rune('.') { - goto l142 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l142 - } - position++ - l146: - { - position147, tokenIndex147 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l147 + goto l139 + l140: + position, tokenIndex = position139, tokenIndex139 + if !_rules[rulesp]() { + goto l137 + } + if !_rules[ruleclose]() { + goto l137 } - position++ - goto l146 - l147: - position, tokenIndex = position147, tokenIndex147 } - add(rulePegText, position143) + l139: + position, tokenIndex = position138, tokenIndex138 } { add(ruleAction29, position) } - goto l115 - l142: - position, tokenIndex = position115, tokenIndex115 + goto l131 + l137: + position, tokenIndex = position131, tokenIndex131 + if buffer[position] != rune('f') { + goto l142 + } + position++ + if buffer[position] != rune('a') { + goto l142 + } + position++ + if buffer[position] != rune('l') { + goto l142 + } + position++ + if buffer[position] != rune('s') { + goto l142 + } + position++ + if buffer[position] != rune('e') { + goto l142 + } + position++ { - position150 := position + position143, tokenIndex143 := position, tokenIndex { - position153, tokenIndex153 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l154 + position144, tokenIndex144 := position, tokenIndex + if !_rules[rulecomma]() { + goto l145 } - position++ - goto l153 - l154: - position, tokenIndex = position153, tokenIndex153 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l155 + goto l144 + l145: + position, tokenIndex = position144, tokenIndex144 + if !_rules[rulesp]() { + goto l142 } - position++ - goto l153 - l155: - position, tokenIndex = position153, tokenIndex153 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l156 + if !_rules[ruleclose]() { + goto l142 } - position++ - goto l153 - l156: - position, tokenIndex = position153, tokenIndex153 - if buffer[position] != rune('-') { - goto l157 - } - position++ - goto l153 - l157: - position, tokenIndex = position153, tokenIndex153 - if buffer[position] != rune('_') { - goto l158 - } - position++ - goto l153 - l158: - position, tokenIndex = position153, tokenIndex153 - if buffer[position] != rune(':') { - goto l149 - } - position++ } - l153: - l151: - { - position152, tokenIndex152 := position, tokenIndex - { - position159, tokenIndex159 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l160 - } - position++ - goto l159 - l160: - position, tokenIndex = position159, tokenIndex159 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l161 - } - position++ - goto l159 - l161: - position, tokenIndex = position159, tokenIndex159 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l162 - } - position++ - goto l159 - l162: - position, tokenIndex = position159, tokenIndex159 - if buffer[position] != rune('-') { - goto l163 - } - position++ - goto l159 - l163: - position, tokenIndex = position159, tokenIndex159 - if buffer[position] != rune('_') { - goto l164 - } - position++ - goto l159 - l164: - position, tokenIndex = position159, tokenIndex159 - if buffer[position] != rune(':') { - goto l152 - } - position++ - } - l159: - goto l151 - l152: - position, tokenIndex = position152, tokenIndex152 - } - add(rulePegText, position150) + l144: + position, tokenIndex = position143, tokenIndex143 } { add(ruleAction30, position) } - goto l115 - l149: - position, tokenIndex = position115, tokenIndex115 - if buffer[position] != rune('"') { - goto l166 - } - position++ + goto l131 + l142: + position, tokenIndex = position131, tokenIndex131 { - position167 := position + position148 := position { - position168 := position - l169: - { - position170, tokenIndex170 := position, tokenIndex - { - position171, tokenIndex171 := position, tokenIndex - { - position173, tokenIndex173 := position, tokenIndex - { - position174, tokenIndex174 := position, tokenIndex - if buffer[position] != rune('"') { - goto l175 - } - position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('\\') { - goto l176 - } - position++ - goto l174 - l176: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('\n') { - goto l173 - } - position++ - } - l174: - goto l172 - l173: - position, tokenIndex = position173, tokenIndex173 - } - if !matchDot() { - goto l172 - } - goto l171 - l172: - position, tokenIndex = position171, tokenIndex171 - if buffer[position] != rune('\\') { - goto l177 - } - position++ - if buffer[position] != rune('n') { - goto l177 - } - position++ - goto l171 - l177: - position, tokenIndex = position171, tokenIndex171 - if buffer[position] != rune('\\') { - goto l178 - } - position++ - if buffer[position] != rune('"') { - goto l178 - } - position++ - goto l171 - l178: - position, tokenIndex = position171, tokenIndex171 - if buffer[position] != rune('\\') { - goto l179 - } - position++ - if buffer[position] != rune('\'') { - goto l179 - } - position++ - goto l171 - l179: - position, tokenIndex = position171, tokenIndex171 - if buffer[position] != rune('\\') { - goto l170 - } - position++ - if buffer[position] != rune('\\') { - goto l170 - } - position++ - } - l171: - goto l169 - l170: - position, tokenIndex = position170, tokenIndex170 + position149, tokenIndex149 := position, tokenIndex + if buffer[position] != rune('-') { + goto l149 } - add(ruledoublequotedstring, position168) + position++ + goto l150 + l149: + position, tokenIndex = position149, tokenIndex149 } - add(rulePegText, position167) + l150: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l147 + } + position++ + l151: + { + position152, tokenIndex152 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l152 + } + position++ + goto l151 + l152: + position, tokenIndex = position152, tokenIndex152 + } + { + position153, tokenIndex153 := position, tokenIndex + if buffer[position] != rune('.') { + goto l153 + } + position++ + l155: + { + position156, tokenIndex156 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l156 + } + position++ + goto l155 + l156: + position, tokenIndex = position156, tokenIndex156 + } + goto l154 + l153: + position, tokenIndex = position153, tokenIndex153 + } + l154: + add(rulePegText, position148) } - if buffer[position] != rune('"') { - goto l166 - } - position++ { add(ruleAction31, position) } - goto l115 - l166: - position, tokenIndex = position115, tokenIndex115 - if buffer[position] != rune('\'') { - goto l113 - } - position++ + goto l131 + l147: + position, tokenIndex = position131, tokenIndex131 { - position181 := position + position159 := position { - position182 := position - l183: - { - position184, tokenIndex184 := position, tokenIndex - { - position185, tokenIndex185 := position, tokenIndex - { - position187, tokenIndex187 := position, tokenIndex - { - position188, tokenIndex188 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l189 - } - position++ - goto l188 - l189: - position, tokenIndex = position188, tokenIndex188 - if buffer[position] != rune('\\') { - goto l190 - } - position++ - goto l188 - l190: - position, tokenIndex = position188, tokenIndex188 - if buffer[position] != rune('\n') { - goto l187 - } - position++ - } - l188: - goto l186 - l187: - position, tokenIndex = position187, tokenIndex187 - } - if !matchDot() { - goto l186 - } - goto l185 - l186: - position, tokenIndex = position185, tokenIndex185 - if buffer[position] != rune('\\') { - goto l191 - } - position++ - if buffer[position] != rune('n') { - goto l191 - } - position++ - goto l185 - l191: - position, tokenIndex = position185, tokenIndex185 - if buffer[position] != rune('\\') { - goto l192 - } - position++ - if buffer[position] != rune('"') { - goto l192 - } - position++ - goto l185 - l192: - position, tokenIndex = position185, tokenIndex185 - if buffer[position] != rune('\\') { - goto l193 - } - position++ - if buffer[position] != rune('\'') { - goto l193 - } - position++ - goto l185 - l193: - position, tokenIndex = position185, tokenIndex185 - if buffer[position] != rune('\\') { - goto l184 - } - position++ - if buffer[position] != rune('\\') { - goto l184 - } - position++ - } - l185: - goto l183 - l184: - position, tokenIndex = position184, tokenIndex184 + position160, tokenIndex160 := position, tokenIndex + if buffer[position] != rune('-') { + goto l160 } - add(rulesinglequotedstring, position182) + position++ + goto l161 + l160: + position, tokenIndex = position160, tokenIndex160 } - add(rulePegText, position181) + l161: + if buffer[position] != rune('.') { + goto l158 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l158 + } + position++ + l162: + { + position163, tokenIndex163 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 + } + position++ + goto l162 + l163: + position, tokenIndex = position163, tokenIndex163 + } + add(rulePegText, position159) } - if buffer[position] != rune('\'') { - goto l113 - } - position++ { add(ruleAction32, position) } + goto l131 + l158: + position, tokenIndex = position131, tokenIndex131 + { + position166 := position + { + position169, tokenIndex169 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l170 + } + position++ + goto l169 + l170: + position, tokenIndex = position169, tokenIndex169 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l171 + } + position++ + goto l169 + l171: + position, tokenIndex = position169, tokenIndex169 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l172 + } + position++ + goto l169 + l172: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('-') { + goto l173 + } + position++ + goto l169 + l173: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('_') { + goto l174 + } + position++ + goto l169 + l174: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune(':') { + goto l165 + } + position++ + } + l169: + l167: + { + position168, tokenIndex168 := position, tokenIndex + { + position175, tokenIndex175 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l176 + } + position++ + goto l175 + l176: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l177 + } + position++ + goto l175 + l177: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l178 + } + position++ + goto l175 + l178: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('-') { + goto l179 + } + position++ + goto l175 + l179: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('_') { + goto l180 + } + position++ + goto l175 + l180: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune(':') { + goto l168 + } + position++ + } + l175: + goto l167 + l168: + position, tokenIndex = position168, tokenIndex168 + } + add(rulePegText, position166) + } + { + add(ruleAction33, position) + } + goto l131 + l165: + position, tokenIndex = position131, tokenIndex131 + if buffer[position] != rune('"') { + goto l182 + } + position++ + { + position183 := position + { + position184 := position + l185: + { + position186, tokenIndex186 := position, tokenIndex + { + position187, tokenIndex187 := position, tokenIndex + { + position189, tokenIndex189 := position, tokenIndex + { + position190, tokenIndex190 := position, tokenIndex + if buffer[position] != rune('"') { + goto l191 + } + position++ + goto l190 + l191: + position, tokenIndex = position190, tokenIndex190 + if buffer[position] != rune('\\') { + goto l192 + } + position++ + goto l190 + l192: + position, tokenIndex = position190, tokenIndex190 + if buffer[position] != rune('\n') { + goto l189 + } + position++ + } + l190: + goto l188 + l189: + position, tokenIndex = position189, tokenIndex189 + } + if !matchDot() { + goto l188 + } + goto l187 + l188: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l193 + } + position++ + if buffer[position] != rune('n') { + goto l193 + } + position++ + goto l187 + l193: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l194 + } + position++ + if buffer[position] != rune('"') { + goto l194 + } + position++ + goto l187 + l194: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l195 + } + position++ + if buffer[position] != rune('\'') { + goto l195 + } + position++ + goto l187 + l195: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l186 + } + position++ + if buffer[position] != rune('\\') { + goto l186 + } + position++ + } + l187: + goto l185 + l186: + position, tokenIndex = position186, tokenIndex186 + } + add(ruledoublequotedstring, position184) + } + add(rulePegText, position183) + } + if buffer[position] != rune('"') { + goto l182 + } + position++ + { + add(ruleAction34, position) + } + goto l131 + l182: + position, tokenIndex = position131, tokenIndex131 + if buffer[position] != rune('\'') { + goto l129 + } + position++ + { + position197 := position + { + position198 := position + l199: + { + position200, tokenIndex200 := position, tokenIndex + { + position201, tokenIndex201 := position, tokenIndex + { + position203, tokenIndex203 := position, tokenIndex + { + position204, tokenIndex204 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l205 + } + position++ + goto l204 + l205: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\\') { + goto l206 + } + position++ + goto l204 + l206: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\n') { + goto l203 + } + position++ + } + l204: + goto l202 + l203: + position, tokenIndex = position203, tokenIndex203 + } + if !matchDot() { + goto l202 + } + goto l201 + l202: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l207 + } + position++ + if buffer[position] != rune('n') { + goto l207 + } + position++ + goto l201 + l207: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l208 + } + position++ + if buffer[position] != rune('"') { + goto l208 + } + position++ + goto l201 + l208: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l209 + } + position++ + if buffer[position] != rune('\'') { + goto l209 + } + position++ + goto l201 + l209: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l200 + } + position++ + if buffer[position] != rune('\\') { + goto l200 + } + position++ + } + l201: + goto l199 + l200: + position, tokenIndex = position200, tokenIndex200 + } + add(rulesinglequotedstring, position198) + } + add(rulePegText, position197) + } + if buffer[position] != rune('\'') { + goto l129 + } + position++ + { + add(ruleAction35, position) + } } - l115: - add(ruleitem, position114) + l131: + add(ruleitem, position130) } return true - l113: - position, tokenIndex = position113, tokenIndex113 + l129: + position, tokenIndex = position129, tokenIndex129 return false }, - /* 10 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 13 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 11 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 14 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 12 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ + /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ func() bool { - position197, tokenIndex197 := position, tokenIndex + position213, tokenIndex213 := position, tokenIndex { - position198 := position + position214 := position { - position199, tokenIndex199 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l200 + goto l216 } position++ - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l197 + goto l213 } position++ } - l199: - l201: + l215: + l217: { - position202, tokenIndex202 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex { - position203, tokenIndex203 := position, tokenIndex + position219, tokenIndex219 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l204 - } - position++ - goto l203 - l204: - position, tokenIndex = position203, tokenIndex203 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l205 - } - position++ - goto l203 - l205: - position, tokenIndex = position203, tokenIndex203 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l206 - } - position++ - goto l203 - l206: - position, tokenIndex = position203, tokenIndex203 - if buffer[position] != rune('_') { - goto l202 - } - position++ - } - l203: - goto l201 - l202: - position, tokenIndex = position202, tokenIndex202 - } - add(rulefieldExpr, position198) - } - return true - l197: - position, tokenIndex = position197, tokenIndex197 - return false - }, - /* 13 field <- <( Action33)> */ - func() bool { - position207, tokenIndex207 := position, tokenIndex - { - position208 := position - { - position209 := position - if !_rules[rulefieldExpr]() { - goto l207 - } - add(rulePegText, position209) - } - { - add(ruleAction33, position) - } - add(rulefield, position208) - } - return true - l207: - position, tokenIndex = position207, tokenIndex207 - return false - }, - /* 14 posfield <- <( Action34)> */ - func() bool { - position211, tokenIndex211 := position, tokenIndex - { - position212 := position - { - position213 := position - if !_rules[rulefieldExpr]() { - goto l211 - } - add(rulePegText, position213) - } - { - add(ruleAction34, position) - } - add(ruleposfield, position212) - } - return true - l211: - position, tokenIndex = position211, tokenIndex211 - return false - }, - /* 15 uint <- <(([1-9] [0-9]*) / '0')> */ - func() bool { - position215, tokenIndex215 := position, tokenIndex - { - position216 := position - { - position217, tokenIndex217 := position, tokenIndex - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l218 - } - position++ - l219: - { - position220, tokenIndex220 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l220 } position++ goto l219 l220: - position, tokenIndex = position220, tokenIndex220 + position, tokenIndex = position219, tokenIndex219 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l221 + } + position++ + goto l219 + l221: + position, tokenIndex = position219, tokenIndex219 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l222 + } + position++ + goto l219 + l222: + position, tokenIndex = position219, tokenIndex219 + if buffer[position] != rune('_') { + goto l218 + } + position++ } + l219: goto l217 l218: - position, tokenIndex = position217, tokenIndex217 - if buffer[position] != rune('0') { - goto l215 - } - position++ + position, tokenIndex = position218, tokenIndex218 } - l217: - add(ruleuint, position216) + add(rulefieldExpr, position214) } return true - l215: - position, tokenIndex = position215, tokenIndex215 + l213: + position, tokenIndex = position213, tokenIndex213 return false }, - /* 16 int <- <(('-'? [1-9] [0-9]*) / '0')> */ + /* 16 field <- <( Action36)> */ func() bool { - position221, tokenIndex221 := position, tokenIndex + position223, tokenIndex223 := position, tokenIndex { - position222 := position + position224 := position { - position223, tokenIndex223 := position, tokenIndex - { - position225, tokenIndex225 := position, tokenIndex - if buffer[position] != rune('-') { - goto l225 - } - position++ - goto l226 - l225: - position, tokenIndex = position225, tokenIndex225 + position225 := position + if !_rules[rulefieldExpr]() { + goto l223 } - l226: - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l224 - } - position++ - l227: - { - position228, tokenIndex228 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l228 - } - position++ - goto l227 - l228: - position, tokenIndex = position228, tokenIndex228 - } - goto l223 - l224: - position, tokenIndex = position223, tokenIndex223 - if buffer[position] != rune('0') { - goto l221 - } - position++ - } - l223: - add(ruleint, position222) - } - return true - l221: - position, tokenIndex = position221, tokenIndex221 - return false - }, - /* 17 uintrow <- <( Action35)> */ - nil, - /* 18 uintcol <- <( Action36)> */ - func() bool { - position230, tokenIndex230 := position, tokenIndex - { - position231 := position - { - position232 := position - if !_rules[ruleuint]() { - goto l230 - } - add(rulePegText, position232) + add(rulePegText, position225) } { add(ruleAction36, position) } - add(ruleuintcol, position231) + add(rulefield, position224) } return true - l230: - position, tokenIndex = position230, tokenIndex230 + l223: + position, tokenIndex = position223, tokenIndex223 return false }, - /* 19 open <- <('(' sp)> */ + /* 17 posfield <- <( Action37)> */ func() bool { - position234, tokenIndex234 := position, tokenIndex + position227, tokenIndex227 := position, tokenIndex { - position235 := position - if buffer[position] != rune('(') { - goto l234 + position228 := position + { + position229 := position + if !_rules[rulefieldExpr]() { + goto l227 + } + add(rulePegText, position229) } - position++ - if !_rules[rulesp]() { - goto l234 + { + add(ruleAction37, position) } - add(ruleopen, position235) + add(ruleposfield, position228) } return true - l234: - position, tokenIndex = position234, tokenIndex234 + l227: + position, tokenIndex = position227, tokenIndex227 return false }, - /* 20 close <- <(')' sp)> */ + /* 18 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position236, tokenIndex236 := position, tokenIndex + position231, tokenIndex231 := position, tokenIndex { - position237 := position - if buffer[position] != rune(')') { - goto l236 + position232 := position + { + position233, tokenIndex233 := position, tokenIndex + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l234 + } + position++ + l235: + { + position236, tokenIndex236 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l236 + } + position++ + goto l235 + l236: + position, tokenIndex = position236, tokenIndex236 + } + goto l233 + l234: + position, tokenIndex = position233, tokenIndex233 + if buffer[position] != rune('0') { + goto l231 + } + position++ } - position++ - if !_rules[rulesp]() { - goto l236 - } - add(ruleclose, position237) + l233: + add(ruleuint, position232) } return true - l236: - position, tokenIndex = position236, tokenIndex236 + l231: + position, tokenIndex = position231, tokenIndex231 return false }, - /* 21 sp <- <(' ' / '\t')*> */ + /* 19 uintrow <- <( Action38)> */ + nil, + /* 20 uintcol <- <( Action39)> */ func() bool { + position238, tokenIndex238 := position, tokenIndex { position239 := position - l240: { - position241, tokenIndex241 := position, tokenIndex - { - position242, tokenIndex242 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l243 - } - position++ - goto l242 - l243: - position, tokenIndex = position242, tokenIndex242 - if buffer[position] != rune('\t') { - goto l241 - } - position++ + position240 := position + if !_rules[ruleuint]() { + goto l238 } - l242: - goto l240 - l241: - position, tokenIndex = position241, tokenIndex241 + add(rulePegText, position240) } - add(rulesp, position239) + { + add(ruleAction39, position) + } + add(ruleuintcol, position239) } return true + l238: + position, tokenIndex = position238, tokenIndex238 + return false }, - /* 22 comma <- <(sp ',' whitesp)> */ + /* 21 open <- <('(' sp)> */ + func() bool { + position242, tokenIndex242 := position, tokenIndex + { + position243 := position + if buffer[position] != rune('(') { + goto l242 + } + position++ + if !_rules[rulesp]() { + goto l242 + } + add(ruleopen, position243) + } + return true + l242: + position, tokenIndex = position242, tokenIndex242 + return false + }, + /* 22 close <- <(')' sp)> */ func() bool { position244, tokenIndex244 := position, tokenIndex { position245 := position - if !_rules[rulesp]() { - goto l244 - } - if buffer[position] != rune(',') { + if buffer[position] != rune(')') { goto l244 } position++ - if !_rules[rulewhitesp]() { + if !_rules[rulesp]() { goto l244 } - add(rulecomma, position245) + add(ruleclose, position245) } return true l244: position, tokenIndex = position244, tokenIndex244 return false }, - /* 23 lbrack <- <('[' sp)> */ - nil, - /* 24 rbrack <- <(sp ']' sp)> */ - nil, - /* 25 whitesp <- <(' ' / '\t' / '\n')*> */ + /* 23 sp <- <(' ' / '\t')*> */ func() bool { { - position249 := position - l250: + position247 := position + l248: { - position251, tokenIndex251 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position252, tokenIndex252 := position, tokenIndex + position250, tokenIndex250 := position, tokenIndex if buffer[position] != rune(' ') { - goto l253 - } - position++ - goto l252 - l253: - position, tokenIndex = position252, tokenIndex252 - if buffer[position] != rune('\t') { - goto l254 - } - position++ - goto l252 - l254: - position, tokenIndex = position252, tokenIndex252 - if buffer[position] != rune('\n') { goto l251 } position++ + goto l250 + l251: + position, tokenIndex = position250, tokenIndex250 + if buffer[position] != rune('\t') { + goto l249 + } + position++ } - l252: - goto l250 - l251: - position, tokenIndex = position251, tokenIndex251 + l250: + goto l248 + l249: + position, tokenIndex = position249, tokenIndex249 } - add(rulewhitesp, position249) + add(rulesp, position247) } return true }, - /* 26 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 24 comma <- <(sp ',' whitesp)> */ + func() bool { + position252, tokenIndex252 := position, tokenIndex + { + position253 := position + if !_rules[rulesp]() { + goto l252 + } + if buffer[position] != rune(',') { + goto l252 + } + position++ + if !_rules[rulewhitesp]() { + goto l252 + } + add(rulecomma, position253) + } + return true + l252: + position, tokenIndex = position252, tokenIndex252 + return false + }, + /* 25 lbrack <- <('[' sp)> */ nil, - /* 27 timestamp <- <(<([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> Action37)> */ + /* 26 rbrack <- <(sp ']' sp)> */ nil, - /* 29 Action0 <- <{p.startCall("Set")}> */ + /* 27 whitesp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position257 := position + l258: + { + position259, tokenIndex259 := position, tokenIndex + { + position260, tokenIndex260 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l261 + } + position++ + goto l260 + l261: + position, tokenIndex = position260, tokenIndex260 + if buffer[position] != rune('\t') { + goto l262 + } + position++ + goto l260 + l262: + position, tokenIndex = position260, tokenIndex260 + if buffer[position] != rune('\n') { + goto l259 + } + position++ + } + l260: + goto l258 + l259: + position, tokenIndex = position259, tokenIndex259 + } + add(rulewhitesp, position257) + } + return true + }, + /* 28 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, - /* 30 Action1 <- <{p.endCall()}> */ + /* 29 timestamp <- <(<([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> Action40)> */ nil, - /* 31 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 31 Action0 <- <{p.startCall("Set")}> */ nil, - /* 32 Action3 <- <{p.endCall()}> */ + /* 32 Action1 <- <{p.endCall()}> */ nil, - /* 33 Action4 <- <{p.startCall("SetColAttrs")}> */ + /* 33 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 34 Action5 <- <{p.endCall()}> */ + /* 34 Action3 <- <{p.endCall()}> */ nil, - /* 35 Action6 <- <{p.startCall("Clear")}> */ + /* 35 Action4 <- <{p.startCall("SetColAttrs")}> */ nil, - /* 36 Action7 <- <{p.endCall()}> */ + /* 36 Action5 <- <{p.endCall()}> */ nil, - /* 37 Action8 <- <{p.startCall("TopN")}> */ + /* 37 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 38 Action9 <- <{p.endCall()}> */ + /* 38 Action7 <- <{p.endCall()}> */ nil, - /* 39 Action10 <- <{p.startCall("Range")}> */ + /* 39 Action8 <- <{p.startCall("TopN")}> */ nil, - /* 40 Action11 <- <{p.endCall()}> */ + /* 40 Action9 <- <{p.endCall()}> */ + nil, + /* 41 Action10 <- <{p.startCall("Range")}> */ + nil, + /* 42 Action11 <- <{p.endCall()}> */ nil, nil, - /* 42 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 44 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 43 Action13 <- <{ p.endCall() }> */ + /* 45 Action13 <- <{ p.endCall() }> */ nil, - /* 44 Action14 <- <{ p.addBTWN() }> */ + /* 46 Action14 <- <{ p.addBTWN() }> */ nil, - /* 45 Action15 <- <{ p.addLTE() }> */ + /* 47 Action15 <- <{ p.addLTE() }> */ nil, - /* 46 Action16 <- <{ p.addGTE() }> */ + /* 48 Action16 <- <{ p.addGTE() }> */ nil, - /* 47 Action17 <- <{ p.addEQ() }> */ + /* 49 Action17 <- <{ p.addEQ() }> */ nil, - /* 48 Action18 <- <{ p.addNEQ() }> */ + /* 50 Action18 <- <{ p.addNEQ() }> */ nil, - /* 49 Action19 <- <{ p.addLT() }> */ + /* 51 Action19 <- <{ p.addLT() }> */ nil, - /* 50 Action20 <- <{ p.addGT() }> */ + /* 52 Action20 <- <{ p.addGT() }> */ nil, - /* 51 Action21 <- <{p.startConditional()}> */ + /* 53 Action21 <- <{p.startConditional()}> */ nil, - /* 52 Action22 <- <{p.endConditional()}> */ + /* 54 Action22 <- <{p.endConditional()}> */ nil, - /* 53 Action23 <- <{ p.startList() }> */ + /* 55 Action23 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 54 Action24 <- <{ p.endList() }> */ + /* 56 Action24 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 55 Action25 <- <{ p.addVal(nil) }> */ + /* 57 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 56 Action26 <- <{ p.addVal(true) }> */ + /* 58 Action26 <- <{ p.startList() }> */ nil, - /* 57 Action27 <- <{ p.addVal(false) }> */ + /* 59 Action27 <- <{ p.endList() }> */ nil, - /* 58 Action28 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 60 Action28 <- <{ p.addVal(nil) }> */ nil, - /* 59 Action29 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 61 Action29 <- <{ p.addVal(true) }> */ nil, - /* 60 Action30 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 62 Action30 <- <{ p.addVal(false) }> */ nil, - /* 61 Action31 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 63 Action31 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 62 Action32 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 64 Action32 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 63 Action33 <- <{ p.addField(buffer[begin:end]) }> */ + /* 65 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 64 Action34 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 66 Action34 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 65 Action35 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 67 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 66 Action36 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 68 Action36 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 67 Action37 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 69 Action37 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + nil, + /* 70 Action38 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 71 Action39 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + nil, + /* 72 Action40 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index d3b5591e8..6ea4d86c3 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -132,6 +132,78 @@ func TestPEGWorking(t *testing.T) { name: "single quoted args", input: `B(a='zm""e')`, ncalls: 1}, + { + name: "SetRowAttrs", + input: "SetRowAttrs(blah, 9, a=47)", + ncalls: 1}, + { + name: "SetRowAttrs2args", + input: "SetRowAttrs(blah, 9, a=47, b=bval)", + ncalls: 1}, + { + name: "SetColAttrs", + input: "SetColAttrs(blah, 9, a=47)", + ncalls: 1}, + { + name: "SetColAttrs2args", + input: "SetColAttrs(blah, 9, a=47, b=bval)", + ncalls: 1}, + { + name: "Clear", + input: "Clear(1, a=53)", + ncalls: 1}, + { + name: "Clear2args", + input: "Clear(1, a=53, b=33)", + ncalls: 1}, + { + name: "TopN", + input: "TopN(myfield, n=44)", + ncalls: 1}, + { + name: "TopNBitmap", + input: "TopN(myfield, Row(a=47), n=10)", + ncalls: 1}, + { + name: "RangeLT", + input: "Range(a < 4)", + ncalls: 1}, + { + name: "RangeGT", + input: "Range(a > 4)", + ncalls: 1}, + { + name: "RangeLTE", + input: "Range(a <= 4)", + ncalls: 1}, + { + name: "RangeGTE", + input: "Range(a >= 4)", + ncalls: 1}, + { + name: "RangeEQ", + input: "Range(a == 4)", + ncalls: 1}, + { + name: "RangeNEQ", + input: "Range(a != null)", + ncalls: 1}, + { + name: "RangeLTLT", + input: "Range(4 < a < 9)", + ncalls: 1}, + { + name: "RangeLTLTE", + input: "Range(4 < a <= 9)", + ncalls: 1}, + { + name: "RangeLTELT", + input: "Range(4 <= a < 9)", + ncalls: 1}, + { + name: "RangeLTELTE", + input: "Range(4 <= a <= 9)", + ncalls: 1}, } for i, test := range tests { @@ -185,6 +257,12 @@ func TestPEGErrors(t *testing.T) { { name: "TopN No Field", input: "TopN(a=77)"}, + { + name: "SetRowAttrs0args", + input: "SetRowAttrs(blah, 9)"}, + { + name: "Clear0args", + input: "Clear(9)"}, } for i, test := range tests { From 5c081d15183090349c0850eacc9ee23c79a64514 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 11:59:39 -0500 Subject: [PATCH 09/33] more tests, fix bug where Condition wasn't pointer --- pql/ast.go | 2 +- pql/pqlpeg_test.go | 199 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 1 deletion(-) diff --git a/pql/ast.go b/pql/ast.go index c5d40a4b9..602b762e1 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -90,7 +90,7 @@ func (q *Query) endConditional() { } call := q.callStack[len(q.callStack)-1] - call.Args[field] = Condition{Op: BETWEEN, Value: []interface{}{low, high}} + call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} q.conditional = nil } diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 6ea4d86c3..ea7c4a3a1 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -1,6 +1,7 @@ package pql import ( + "reflect" "strconv" "testing" ) @@ -274,3 +275,201 @@ func TestPEGErrors(t *testing.T) { }) } } + +func TestPQLDeepEquality(t *testing.T) { + tests := []struct { + name string + call string + exp *Call + }{ + { + name: "Set", + call: "Set(1, a=7, 2010-07-08T14:44)", + exp: &Call{ + Name: "Set", + Args: map[string]interface{}{ + "a": int64(7), + "_col": int64(1), + "_timestamp": "2010-07-08T14:44", + }, + }}, + { + name: "SetRowAttrs", + call: "SetRowAttrs(myfield, 9, z=4)", + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, + { + name: "SetColAttrs", + call: "SetColAttrs(myfield, 9, z=4)", + exp: &Call{ + Name: "SetColAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_col": int64(9), + }, + }}, + { + name: "Clear", + call: "Clear(1, a=7)", + exp: &Call{ + Name: "Clear", + Args: map[string]interface{}{ + "a": int64(7), + "_col": int64(1), + }, + }}, + { + name: "TopN", + call: "TopN(myfield, Row(), a=7)", + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "RangeEQ", + call: "Range(a==7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: EQ, + Value: int64(7), + }, + }, + }}, + { + name: "RangeLT", + call: "Range(a<7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: LT, + Value: int64(7), + }, + }, + }}, + { + name: "RangeLTE", + call: "Range(a<=7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: LTE, + Value: int64(7), + }, + }, + }}, + { + name: "RangeGTE", + call: "Range(a>=7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: GTE, + Value: int64(7), + }, + }, + }}, + { + name: "RangeGT", + call: "Range(a>7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: GT, + Value: int64(7), + }, + }, + }}, + { + name: "RangeNEQ", + call: "Range(a!=null)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: NEQ, + Value: nil, + }, + }, + }}, + { + name: "RangeLTELT", + call: "Range(4 <= a < 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(4), int64(9)}, + }, + }, + }}, + { + name: "RangeLTLT", + call: "Range(4 < a < 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(9)}, + }, + }, + }}, + { + name: "RangeLTELTE", + call: "Range(4 <= a <= 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(4), int64(10)}, + }, + }, + }}, + { + name: "RangeLTLTE", + call: "Range(4 < a <= 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(10)}, + }, + }, + }}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.call) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.call, err) + } + + if !reflect.DeepEqual(test.exp, q.Calls[0]) { + t.Fatalf("unexpected call:\n%s\ninstead of:\n%s\n'%#v'\ninstead of:\n'%#v'", q.Calls[0], test.exp, q.Calls[0], test.exp) + } + }) + } +} From 08e84aa07ff42eea7fa68eb432105da55dde324f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 19 Jun 2018 14:07:03 -0500 Subject: [PATCH 10/33] time range support --- pql/pql.peg | 9 +- pql/pql.peg.go | 2511 +++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 14 + 3 files changed, 1342 insertions(+), 1192 deletions(-) diff --git a/pql/pql.peg b/pql/pql.peg index 1911dc756..d5f82f49c 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -11,7 +11,7 @@ Call <- 'Set' {p.startCall("Set")} open uintcol comma args (comma timestamp)? c / 'SetColAttrs' {p.startCall("SetColAttrs")} open posfield comma uintcol comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open uintcol comma args close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} - / 'Range' {p.startCall("Range")} open (arg / conditional) close {p.endCall()} + / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp @@ -31,6 +31,8 @@ condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])} condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} condfield <- sp {p.condAdd(buffer[begin:end])} +timerange <- field sp '=' sp value comma {p.addPosStr("_start", buffer[begin:end])} comma {p.addPosStr("_end", buffer[begin:end])} + value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) @@ -64,4 +66,7 @@ rbrack <- sp ']' sp whitesp <- ( ' ' / '\t' / '\n' )* IDENT <- !('Set(' / 'SetRowAttrs(' / 'SetColAttrs(' / 'Clear(' / 'TopN(' / 'Range(') [[A-Z]] ([[A-Z]] / [0-9])* -timestamp <- <[0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]> {p.addPosStr("_timestamp", buffer[begin:end])} \ No newline at end of file + +timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] +timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt +timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 356b2b1b3..467c14883 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -26,6 +26,7 @@ const ( rulecondint rulecondLT rulecondfield + ruletimerange rulevalue rulelist ruleitem @@ -45,6 +46,8 @@ const ( rulerbrack rulewhitesp ruleIDENT + ruletimestampbasicfmt + ruletimestampfmt ruletimestamp ruleAction0 ruleAction1 @@ -88,6 +91,8 @@ const ( ruleAction38 ruleAction39 ruleAction40 + ruleAction41 + ruleAction42 ) var rul3s = [...]string{ @@ -102,6 +107,7 @@ var rul3s = [...]string{ "condint", "condLT", "condfield", + "timerange", "value", "list", "item", @@ -121,6 +127,8 @@ var rul3s = [...]string{ "rbrack", "whitesp", "IDENT", + "timestampbasicfmt", + "timestampfmt", "timestamp", "Action0", "Action1", @@ -164,6 +172,8 @@ var rul3s = [...]string{ "Action38", "Action39", "Action40", + "Action41", + "Action42", } type token32 struct { @@ -280,7 +290,7 @@ type PQL struct { Buffer string buffer []rune - rules [73]func() bool + rules [78]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -425,34 +435,38 @@ func (p *PQL) Execute() { case ruleAction25: p.condAdd(buffer[begin:end]) case ruleAction26: - p.startList() + p.addPosStr("_start", buffer[begin:end]) case ruleAction27: - p.endList() + p.addPosStr("_end", buffer[begin:end]) case ruleAction28: - p.addVal(nil) + p.startList() case ruleAction29: - p.addVal(true) + p.endList() case ruleAction30: - p.addVal(false) + p.addVal(nil) case ruleAction31: - p.addNumVal(buffer[begin:end]) + p.addVal(true) case ruleAction32: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction33: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction34: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction35: p.addVal(buffer[begin:end]) case ruleAction36: - p.addField(buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction37: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction38: - p.addPosNum("_row", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction39: - p.addPosNum("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction40: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction41: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction42: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -565,7 +579,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (arg / conditional) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -608,85 +622,13 @@ func (p *PQL) Init() { position12 := position { position13 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { + if !_rules[ruletimestampfmt]() { goto l10 } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if buffer[position] != rune('-') { - goto l10 - } - position++ - { - position14, tokenIndex14 := position, tokenIndex - if buffer[position] != rune('0') { - goto l15 - } - position++ - goto l14 - l15: - position, tokenIndex = position14, tokenIndex14 - if buffer[position] != rune('1') { - goto l10 - } - position++ - } - l14: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if buffer[position] != rune('-') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if buffer[position] != rune('T') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if buffer[position] != rune(':') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l10 - } - position++ add(rulePegText, position13) } { - add(ruleAction40, position) + add(ruleAction42, position) } add(ruletimestamp, position12) } @@ -705,595 +647,644 @@ func (p *PQL) Init() { l8: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('S') { - goto l18 + goto l16 } position++ if buffer[position] != rune('e') { - goto l18 + goto l16 } position++ if buffer[position] != rune('t') { - goto l18 + goto l16 } position++ if buffer[position] != rune('R') { - goto l18 + goto l16 } position++ if buffer[position] != rune('o') { - goto l18 + goto l16 } position++ if buffer[position] != rune('w') { - goto l18 + goto l16 } position++ if buffer[position] != rune('A') { - goto l18 + goto l16 } position++ if buffer[position] != rune('t') { - goto l18 + goto l16 } position++ if buffer[position] != rune('t') { - goto l18 + goto l16 } position++ if buffer[position] != rune('r') { - goto l18 + goto l16 } position++ if buffer[position] != rune('s') { - goto l18 + goto l16 } position++ { add(ruleAction2, position) } if !_rules[ruleopen]() { - goto l18 + goto l16 } if !_rules[ruleposfield]() { - goto l18 + goto l16 } if !_rules[rulecomma]() { - goto l18 + goto l16 } { - position20 := position + position18 := position { - position21 := position + position19 := position if !_rules[ruleuint]() { - goto l18 + goto l16 } - add(rulePegText, position21) + add(rulePegText, position19) } { - add(ruleAction38, position) + add(ruleAction40, position) } - add(ruleuintrow, position20) + add(ruleuintrow, position18) } if !_rules[rulecomma]() { - goto l18 + goto l16 } if !_rules[ruleargs]() { - goto l18 + goto l16 } if !_rules[ruleclose]() { - goto l18 + goto l16 } { add(ruleAction3, position) } goto l7 - l18: + l16: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('S') { - goto l24 + goto l22 } position++ if buffer[position] != rune('e') { - goto l24 + goto l22 } position++ if buffer[position] != rune('t') { - goto l24 + goto l22 } position++ if buffer[position] != rune('C') { - goto l24 + goto l22 } position++ if buffer[position] != rune('o') { - goto l24 + goto l22 } position++ if buffer[position] != rune('l') { - goto l24 + goto l22 } position++ if buffer[position] != rune('A') { - goto l24 + goto l22 } position++ if buffer[position] != rune('t') { - goto l24 + goto l22 } position++ if buffer[position] != rune('t') { - goto l24 + goto l22 } position++ if buffer[position] != rune('r') { - goto l24 + goto l22 } position++ if buffer[position] != rune('s') { - goto l24 + goto l22 } position++ { add(ruleAction4, position) } if !_rules[ruleopen]() { - goto l24 + goto l22 } if !_rules[ruleposfield]() { - goto l24 + goto l22 } if !_rules[rulecomma]() { - goto l24 + goto l22 } if !_rules[ruleuintcol]() { - goto l24 + goto l22 } if !_rules[rulecomma]() { - goto l24 + goto l22 } if !_rules[ruleargs]() { - goto l24 + goto l22 } if !_rules[ruleclose]() { - goto l24 + goto l22 } { add(ruleAction5, position) } goto l7 - l24: + l22: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('C') { - goto l27 + goto l25 } position++ if buffer[position] != rune('l') { - goto l27 + goto l25 } position++ if buffer[position] != rune('e') { - goto l27 + goto l25 } position++ if buffer[position] != rune('a') { - goto l27 + goto l25 } position++ if buffer[position] != rune('r') { - goto l27 + goto l25 } position++ { add(ruleAction6, position) } if !_rules[ruleopen]() { - goto l27 + goto l25 } if !_rules[ruleuintcol]() { - goto l27 + goto l25 } if !_rules[rulecomma]() { - goto l27 + goto l25 } if !_rules[ruleargs]() { - goto l27 + goto l25 } if !_rules[ruleclose]() { - goto l27 + goto l25 } { add(ruleAction7, position) } goto l7 - l27: + l25: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('T') { - goto l30 + goto l28 } position++ if buffer[position] != rune('o') { - goto l30 + goto l28 } position++ if buffer[position] != rune('p') { - goto l30 + goto l28 } position++ if buffer[position] != rune('N') { - goto l30 + goto l28 } position++ { add(ruleAction8, position) } if !_rules[ruleopen]() { - goto l30 + goto l28 } if !_rules[ruleposfield]() { - goto l30 + goto l28 } { - position32, tokenIndex32 := position, tokenIndex + position30, tokenIndex30 := position, tokenIndex if !_rules[rulecomma]() { - goto l32 + goto l30 } if !_rules[ruleallargs]() { - goto l32 + goto l30 } - goto l33 - l32: - position, tokenIndex = position32, tokenIndex32 + goto l31 + l30: + position, tokenIndex = position30, tokenIndex30 } - l33: + l31: if !_rules[ruleclose]() { - goto l30 + goto l28 } { add(ruleAction9, position) } goto l7 - l30: + l28: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('R') { - goto l35 + goto l33 } position++ if buffer[position] != rune('a') { - goto l35 + goto l33 } position++ if buffer[position] != rune('n') { - goto l35 + goto l33 } position++ if buffer[position] != rune('g') { - goto l35 + goto l33 } position++ if buffer[position] != rune('e') { - goto l35 + goto l33 } position++ { add(ruleAction10, position) } if !_rules[ruleopen]() { - goto l35 + goto l33 } { - position37, tokenIndex37 := position, tokenIndex - if !_rules[rulearg]() { - goto l38 - } - goto l37 - l38: - position, tokenIndex = position37, tokenIndex37 + position35, tokenIndex35 := position, tokenIndex { - position39 := position + position37 := position + if !_rules[rulefield]() { + goto l36 + } + if !_rules[rulesp]() { + goto l36 + } + if buffer[position] != rune('=') { + goto l36 + } + position++ + if !_rules[rulesp]() { + goto l36 + } + if !_rules[rulevalue]() { + goto l36 + } + if !_rules[rulecomma]() { + goto l36 + } + { + position38 := position + if !_rules[ruletimestampfmt]() { + goto l36 + } + add(rulePegText, position38) + } + { + add(ruleAction26, position) + } + if !_rules[rulecomma]() { + goto l36 + } + { + position40 := position + if !_rules[ruletimestampfmt]() { + goto l36 + } + add(rulePegText, position40) + } + { + add(ruleAction27, position) + } + add(ruletimerange, position37) + } + goto l35 + l36: + position, tokenIndex = position35, tokenIndex35 + { + position43 := position { add(ruleAction21, position) } if !_rules[rulecondint]() { - goto l35 + goto l42 } if !_rules[rulecondLT]() { - goto l35 + goto l42 } { - position41 := position + position45 := position { - position42 := position + position46 := position if !_rules[rulefieldExpr]() { - goto l35 + goto l42 } - add(rulePegText, position42) + add(rulePegText, position46) } if !_rules[rulesp]() { - goto l35 + goto l42 } { add(ruleAction25, position) } - add(rulecondfield, position41) + add(rulecondfield, position45) } if !_rules[rulecondLT]() { - goto l35 + goto l42 } if !_rules[rulecondint]() { - goto l35 + goto l42 } { add(ruleAction22, position) } - add(ruleconditional, position39) + add(ruleconditional, position43) + } + goto l35 + l42: + position, tokenIndex = position35, tokenIndex35 + if !_rules[rulearg]() { + goto l33 } } - l37: + l35: if !_rules[ruleclose]() { - goto l35 + goto l33 } { add(ruleAction11, position) } goto l7 - l35: + l33: position, tokenIndex = position7, tokenIndex7 { - position46 := position + position50 := position { - position47 := position + position51 := position { - position48, tokenIndex48 := position, tokenIndex + position52, tokenIndex52 := position, tokenIndex { - position49, tokenIndex49 := position, tokenIndex + position53, tokenIndex53 := position, tokenIndex if buffer[position] != rune('S') { - goto l50 + goto l54 } position++ if buffer[position] != rune('e') { - goto l50 + goto l54 } position++ if buffer[position] != rune('t') { - goto l50 + goto l54 } position++ if buffer[position] != rune('(') { - goto l50 + goto l54 } position++ - goto l49 - l50: - position, tokenIndex = position49, tokenIndex49 + goto l53 + l54: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('S') { - goto l51 + goto l55 } position++ if buffer[position] != rune('e') { - goto l51 + goto l55 } position++ if buffer[position] != rune('t') { - goto l51 + goto l55 } position++ if buffer[position] != rune('R') { - goto l51 + goto l55 } position++ if buffer[position] != rune('o') { - goto l51 + goto l55 } position++ if buffer[position] != rune('w') { - goto l51 + goto l55 } position++ if buffer[position] != rune('A') { - goto l51 + goto l55 } position++ if buffer[position] != rune('t') { - goto l51 + goto l55 } position++ if buffer[position] != rune('t') { - goto l51 + goto l55 } position++ if buffer[position] != rune('r') { - goto l51 + goto l55 } position++ if buffer[position] != rune('s') { - goto l51 + goto l55 } position++ if buffer[position] != rune('(') { - goto l51 + goto l55 } position++ - goto l49 - l51: - position, tokenIndex = position49, tokenIndex49 + goto l53 + l55: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('S') { - goto l52 + goto l56 } position++ if buffer[position] != rune('e') { - goto l52 + goto l56 } position++ if buffer[position] != rune('t') { - goto l52 + goto l56 } position++ if buffer[position] != rune('C') { - goto l52 + goto l56 } position++ if buffer[position] != rune('o') { - goto l52 + goto l56 } position++ if buffer[position] != rune('l') { - goto l52 + goto l56 } position++ if buffer[position] != rune('A') { - goto l52 + goto l56 } position++ if buffer[position] != rune('t') { - goto l52 + goto l56 } position++ if buffer[position] != rune('t') { - goto l52 + goto l56 } position++ if buffer[position] != rune('r') { - goto l52 + goto l56 } position++ if buffer[position] != rune('s') { - goto l52 + goto l56 } position++ if buffer[position] != rune('(') { - goto l52 + goto l56 } position++ - goto l49 - l52: - position, tokenIndex = position49, tokenIndex49 + goto l53 + l56: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('C') { - goto l53 + goto l57 } position++ if buffer[position] != rune('l') { - goto l53 + goto l57 } position++ if buffer[position] != rune('e') { - goto l53 + goto l57 } position++ if buffer[position] != rune('a') { - goto l53 + goto l57 } position++ if buffer[position] != rune('r') { - goto l53 + goto l57 } position++ if buffer[position] != rune('(') { - goto l53 + goto l57 } position++ - goto l49 - l53: - position, tokenIndex = position49, tokenIndex49 + goto l53 + l57: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('T') { - goto l54 + goto l58 } position++ if buffer[position] != rune('o') { - goto l54 + goto l58 } position++ if buffer[position] != rune('p') { - goto l54 + goto l58 } position++ if buffer[position] != rune('N') { - goto l54 + goto l58 } position++ if buffer[position] != rune('(') { - goto l54 + goto l58 } position++ - goto l49 - l54: - position, tokenIndex = position49, tokenIndex49 + goto l53 + l58: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('R') { - goto l48 + goto l52 } position++ if buffer[position] != rune('a') { - goto l48 + goto l52 } position++ if buffer[position] != rune('n') { - goto l48 + goto l52 } position++ if buffer[position] != rune('g') { - goto l48 + goto l52 } position++ if buffer[position] != rune('e') { - goto l48 + goto l52 } position++ if buffer[position] != rune('(') { - goto l48 + goto l52 } position++ } - l49: + l53: goto l5 - l48: - position, tokenIndex = position48, tokenIndex48 + l52: + position, tokenIndex = position52, tokenIndex52 } { - position55, tokenIndex55 := position, tokenIndex + position59, tokenIndex59 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l56 + goto l60 } position++ - goto l55 - l56: - position, tokenIndex = position55, tokenIndex55 + goto l59 + l60: + position, tokenIndex = position59, tokenIndex59 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l55: - l57: + l59: + l61: { - position58, tokenIndex58 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex { - position59, tokenIndex59 := position, tokenIndex + position63, tokenIndex63 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 + goto l64 } position++ - goto l59 - l60: - position, tokenIndex = position59, tokenIndex59 + goto l63 + l64: + position, tokenIndex = position63, tokenIndex63 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l61 + goto l65 } position++ - goto l59 - l61: - position, tokenIndex = position59, tokenIndex59 + goto l63 + l65: + position, tokenIndex = position63, tokenIndex63 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l58 + goto l62 } position++ } - l59: - goto l57 - l58: - position, tokenIndex = position58, tokenIndex58 + l63: + goto l61 + l62: + position, tokenIndex = position62, tokenIndex62 } - add(ruleIDENT, position47) + add(ruleIDENT, position51) } - add(rulePegText, position46) + add(rulePegText, position50) } { add(ruleAction12, position) @@ -1305,15 +1296,15 @@ func (p *PQL) Init() { goto l5 } { - position63, tokenIndex63 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex if !_rules[rulecomma]() { - goto l63 + goto l67 } - goto l64 - l63: - position, tokenIndex = position63, tokenIndex63 + goto l68 + l67: + position, tokenIndex = position67, tokenIndex67 } - l64: + l68: if !_rules[ruleclose]() { goto l5 } @@ -1331,232 +1322,232 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position66, tokenIndex66 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex { - position67 := position + position71 := position { - position68, tokenIndex68 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex if !_rules[ruleCall]() { - goto l69 + goto l73 } - l70: + l74: { - position71, tokenIndex71 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex if !_rules[rulecomma]() { - goto l71 + goto l75 } if !_rules[ruleCall]() { - goto l71 + goto l75 } - goto l70 - l71: - position, tokenIndex = position71, tokenIndex71 + goto l74 + l75: + position, tokenIndex = position75, tokenIndex75 } { - position72, tokenIndex72 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex if !_rules[rulecomma]() { - goto l72 + goto l76 } if !_rules[ruleargs]() { - goto l72 + goto l76 } - goto l73 - l72: - position, tokenIndex = position72, tokenIndex72 + goto l77 + l76: + position, tokenIndex = position76, tokenIndex76 } + l77: + goto l72 l73: - goto l68 - l69: - position, tokenIndex = position68, tokenIndex68 + position, tokenIndex = position72, tokenIndex72 if !_rules[ruleargs]() { - goto l74 + goto l78 } - goto l68 - l74: - position, tokenIndex = position68, tokenIndex68 + goto l72 + l78: + position, tokenIndex = position72, tokenIndex72 if !_rules[rulesp]() { - goto l66 + goto l70 } } - l68: - add(ruleallargs, position67) + l72: + add(ruleallargs, position71) } return true - l66: - position, tokenIndex = position66, tokenIndex66 + l70: + position, tokenIndex = position70, tokenIndex70 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position75, tokenIndex75 := position, tokenIndex + position79, tokenIndex79 := position, tokenIndex { - position76 := position + position80 := position if !_rules[rulearg]() { - goto l75 + goto l79 } { - position77, tokenIndex77 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex if !_rules[rulecomma]() { - goto l77 + goto l81 } if !_rules[ruleargs]() { - goto l77 + goto l81 } - goto l78 - l77: - position, tokenIndex = position77, tokenIndex77 + goto l82 + l81: + position, tokenIndex = position81, tokenIndex81 } - l78: + l82: if !_rules[rulesp]() { - goto l75 + goto l79 } - add(ruleargs, position76) + add(ruleargs, position80) } return true - l75: - position, tokenIndex = position75, tokenIndex75 + l79: + position, tokenIndex = position79, tokenIndex79 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position79, tokenIndex79 := position, tokenIndex + position83, tokenIndex83 := position, tokenIndex { - position80 := position + position84 := position { - position81, tokenIndex81 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex if !_rules[rulefield]() { - goto l82 + goto l86 } if !_rules[rulesp]() { - goto l82 + goto l86 } if buffer[position] != rune('=') { - goto l82 + goto l86 } position++ if !_rules[rulesp]() { - goto l82 + goto l86 } if !_rules[rulevalue]() { - goto l82 + goto l86 } - goto l81 - l82: - position, tokenIndex = position81, tokenIndex81 + goto l85 + l86: + position, tokenIndex = position85, tokenIndex85 if !_rules[rulefield]() { - goto l79 + goto l83 } if !_rules[rulesp]() { - goto l79 + goto l83 } { - position83 := position + position87 := position { - position84, tokenIndex84 := position, tokenIndex + position88, tokenIndex88 := position, tokenIndex if buffer[position] != rune('>') { - goto l85 + goto l89 } position++ if buffer[position] != rune('<') { - goto l85 + goto l89 } position++ { add(ruleAction14, position) } - goto l84 - l85: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l89: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('<') { - goto l87 + goto l91 } position++ if buffer[position] != rune('=') { - goto l87 + goto l91 } position++ { add(ruleAction15, position) } - goto l84 - l87: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l91: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('>') { - goto l89 + goto l93 } position++ if buffer[position] != rune('=') { - goto l89 + goto l93 } position++ { add(ruleAction16, position) } - goto l84 - l89: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l93: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('=') { - goto l91 + goto l95 } position++ if buffer[position] != rune('=') { - goto l91 + goto l95 } position++ { add(ruleAction17, position) } - goto l84 - l91: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l95: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('!') { - goto l93 + goto l97 } position++ if buffer[position] != rune('=') { - goto l93 + goto l97 } position++ { add(ruleAction18, position) } - goto l84 - l93: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l97: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('<') { - goto l95 + goto l99 } position++ { add(ruleAction19, position) } - goto l84 - l95: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l99: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('>') { - goto l79 + goto l83 } position++ { add(ruleAction20, position) } } - l84: - add(ruleCOND, position83) + l88: + add(ruleCOND, position87) } if !_rules[rulesp]() { - goto l79 + goto l83 } if !_rules[rulevalue]() { - goto l79 + goto l83 } } - l81: - add(rulearg, position80) + l85: + add(rulearg, position84) } return true - l79: - position, tokenIndex = position79, tokenIndex79 + l83: + position, tokenIndex = position83, tokenIndex83 return false }, /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ @@ -1565,244 +1556,200 @@ func (p *PQL) Init() { nil, /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ func() bool { - position100, tokenIndex100 := position, tokenIndex + position104, tokenIndex104 := position, tokenIndex { - position101 := position + position105 := position { - position102 := position + position106 := position { - position103, tokenIndex103 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex { - position105, tokenIndex105 := position, tokenIndex + position109, tokenIndex109 := position, tokenIndex if buffer[position] != rune('-') { - goto l105 + goto l109 } position++ - goto l106 - l105: - position, tokenIndex = position105, tokenIndex105 + goto l110 + l109: + position, tokenIndex = position109, tokenIndex109 } - l106: + l110: if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l108 + } + position++ + l111: + { + position112, tokenIndex112 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l112 + } + position++ + goto l111 + l112: + position, tokenIndex = position112, tokenIndex112 + } + goto l107 + l108: + position, tokenIndex = position107, tokenIndex107 + if buffer[position] != rune('0') { goto l104 } position++ - l107: - { - position108, tokenIndex108 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l108 - } - position++ - goto l107 - l108: - position, tokenIndex = position108, tokenIndex108 - } - goto l103 - l104: - position, tokenIndex = position103, tokenIndex103 - if buffer[position] != rune('0') { - goto l100 - } - position++ } - l103: - add(rulePegText, position102) + l107: + add(rulePegText, position106) } if !_rules[rulesp]() { - goto l100 + goto l104 } { add(ruleAction23, position) } - add(rulecondint, position101) + add(rulecondint, position105) } return true - l100: - position, tokenIndex = position100, tokenIndex100 + l104: + position, tokenIndex = position104, tokenIndex104 return false }, /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ func() bool { - position110, tokenIndex110 := position, tokenIndex + position114, tokenIndex114 := position, tokenIndex { - position111 := position + position115 := position { - position112 := position + position116 := position { - position113, tokenIndex113 := position, tokenIndex + position117, tokenIndex117 := position, tokenIndex if buffer[position] != rune('<') { - goto l114 + goto l118 } position++ if buffer[position] != rune('=') { + goto l118 + } + position++ + goto l117 + l118: + position, tokenIndex = position117, tokenIndex117 + if buffer[position] != rune('<') { goto l114 } position++ - goto l113 - l114: - position, tokenIndex = position113, tokenIndex113 - if buffer[position] != rune('<') { - goto l110 - } - position++ } - l113: - add(rulePegText, position112) + l117: + add(rulePegText, position116) } if !_rules[rulesp]() { - goto l110 + goto l114 } { add(ruleAction24, position) } - add(rulecondLT, position111) + add(rulecondLT, position115) } return true - l110: - position, tokenIndex = position110, tokenIndex110 + l114: + position, tokenIndex = position114, tokenIndex114 return false }, /* 9 condfield <- <( sp Action25)> */ nil, - /* 10 value <- <(item / (lbrack Action26 list rbrack Action27))> */ + /* 10 timerange <- <(field sp '=' sp value comma Action26 comma Action27)> */ + nil, + /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ func() bool { - position117, tokenIndex117 := position, tokenIndex + position122, tokenIndex122 := position, tokenIndex { - position118 := position + position123 := position { - position119, tokenIndex119 := position, tokenIndex + position124, tokenIndex124 := position, tokenIndex if !_rules[ruleitem]() { - goto l120 + goto l125 } - goto l119 - l120: - position, tokenIndex = position119, tokenIndex119 + goto l124 + l125: + position, tokenIndex = position124, tokenIndex124 { - position121 := position + position126 := position if buffer[position] != rune('[') { - goto l117 + goto l122 } position++ if !_rules[rulesp]() { - goto l117 + goto l122 } - add(rulelbrack, position121) - } - { - add(ruleAction26, position) - } - if !_rules[rulelist]() { - goto l117 - } - { - position123 := position - if !_rules[rulesp]() { - goto l117 - } - if buffer[position] != rune(']') { - goto l117 - } - position++ - if !_rules[rulesp]() { - goto l117 - } - add(rulerbrack, position123) - } - { - add(ruleAction27, position) - } - } - l119: - add(rulevalue, position118) - } - return true - l117: - position, tokenIndex = position117, tokenIndex117 - return false - }, - /* 11 list <- <(item (comma list)?)> */ - func() bool { - position125, tokenIndex125 := position, tokenIndex - { - position126 := position - if !_rules[ruleitem]() { - goto l125 - } - { - position127, tokenIndex127 := position, tokenIndex - if !_rules[rulecomma]() { - goto l127 - } - if !_rules[rulelist]() { - goto l127 - } - goto l128 - l127: - position, tokenIndex = position127, tokenIndex127 - } - l128: - add(rulelist, position126) - } - return true - l125: - position, tokenIndex = position125, tokenIndex125 - return false - }, - /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action28) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action29) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action30) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action31) / (<('-'? '.' [0-9]+)> Action32) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action33) / ('"' '"' Action34) / ('\'' '\'' Action35))> */ - func() bool { - position129, tokenIndex129 := position, tokenIndex - { - position130 := position - { - position131, tokenIndex131 := position, tokenIndex - if buffer[position] != rune('n') { - goto l132 - } - position++ - if buffer[position] != rune('u') { - goto l132 - } - position++ - if buffer[position] != rune('l') { - goto l132 - } - position++ - if buffer[position] != rune('l') { - goto l132 - } - position++ - { - position133, tokenIndex133 := position, tokenIndex - { - position134, tokenIndex134 := position, tokenIndex - if !_rules[rulecomma]() { - goto l135 - } - goto l134 - l135: - position, tokenIndex = position134, tokenIndex134 - if !_rules[rulesp]() { - goto l132 - } - if !_rules[ruleclose]() { - goto l132 - } - } - l134: - position, tokenIndex = position133, tokenIndex133 + add(rulelbrack, position126) } { add(ruleAction28, position) } - goto l131 - l132: - position, tokenIndex = position131, tokenIndex131 - if buffer[position] != rune('t') { - goto l137 + if !_rules[rulelist]() { + goto l122 } - position++ - if buffer[position] != rune('r') { + { + position128 := position + if !_rules[rulesp]() { + goto l122 + } + if buffer[position] != rune(']') { + goto l122 + } + position++ + if !_rules[rulesp]() { + goto l122 + } + add(rulerbrack, position128) + } + { + add(ruleAction29, position) + } + } + l124: + add(rulevalue, position123) + } + return true + l122: + position, tokenIndex = position122, tokenIndex122 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position130, tokenIndex130 := position, tokenIndex + { + position131 := position + if !_rules[ruleitem]() { + goto l130 + } + { + position132, tokenIndex132 := position, tokenIndex + if !_rules[rulecomma]() { + goto l132 + } + if !_rules[rulelist]() { + goto l132 + } + goto l133 + l132: + position, tokenIndex = position132, tokenIndex132 + } + l133: + add(rulelist, position131) + } + return true + l130: + position, tokenIndex = position130, tokenIndex130 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ + func() bool { + position134, tokenIndex134 := position, tokenIndex + { + position135 := position + { + position136, tokenIndex136 := position, tokenIndex + if buffer[position] != rune('n') { goto l137 } position++ @@ -1810,7 +1757,11 @@ func (p *PQL) Init() { goto l137 } position++ - if buffer[position] != rune('e') { + if buffer[position] != rune('l') { + goto l137 + } + position++ + if buffer[position] != rune('l') { goto l137 } position++ @@ -1835,24 +1786,20 @@ func (p *PQL) Init() { position, tokenIndex = position138, tokenIndex138 } { - add(ruleAction29, position) + add(ruleAction30, position) } - goto l131 + goto l136 l137: - position, tokenIndex = position131, tokenIndex131 - if buffer[position] != rune('f') { + position, tokenIndex = position136, tokenIndex136 + if buffer[position] != rune('t') { goto l142 } position++ - if buffer[position] != rune('a') { + if buffer[position] != rune('r') { goto l142 } position++ - if buffer[position] != rune('l') { - goto l142 - } - position++ - if buffer[position] != rune('s') { + if buffer[position] != rune('u') { goto l142 } position++ @@ -1880,825 +1827,1009 @@ func (p *PQL) Init() { l144: position, tokenIndex = position143, tokenIndex143 } - { - add(ruleAction30, position) - } - goto l131 - l142: - position, tokenIndex = position131, tokenIndex131 - { - position148 := position - { - position149, tokenIndex149 := position, tokenIndex - if buffer[position] != rune('-') { - goto l149 - } - position++ - goto l150 - l149: - position, tokenIndex = position149, tokenIndex149 - } - l150: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l147 - } - position++ - l151: - { - position152, tokenIndex152 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 - } - position++ - goto l151 - l152: - position, tokenIndex = position152, tokenIndex152 - } - { - position153, tokenIndex153 := position, tokenIndex - if buffer[position] != rune('.') { - goto l153 - } - position++ - l155: - { - position156, tokenIndex156 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l156 - } - position++ - goto l155 - l156: - position, tokenIndex = position156, tokenIndex156 - } - goto l154 - l153: - position, tokenIndex = position153, tokenIndex153 - } - l154: - add(rulePegText, position148) - } { add(ruleAction31, position) } - goto l131 - l147: - position, tokenIndex = position131, tokenIndex131 + goto l136 + l142: + position, tokenIndex = position136, tokenIndex136 + if buffer[position] != rune('f') { + goto l147 + } + position++ + if buffer[position] != rune('a') { + goto l147 + } + position++ + if buffer[position] != rune('l') { + goto l147 + } + position++ + if buffer[position] != rune('s') { + goto l147 + } + position++ + if buffer[position] != rune('e') { + goto l147 + } + position++ { - position159 := position + position148, tokenIndex148 := position, tokenIndex { - position160, tokenIndex160 := position, tokenIndex - if buffer[position] != rune('-') { - goto l160 + position149, tokenIndex149 := position, tokenIndex + if !_rules[rulecomma]() { + goto l150 } - position++ - goto l161 - l160: - position, tokenIndex = position160, tokenIndex160 - } - l161: - if buffer[position] != rune('.') { - goto l158 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l158 - } - position++ - l162: - { - position163, tokenIndex163 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 + goto l149 + l150: + position, tokenIndex = position149, tokenIndex149 + if !_rules[rulesp]() { + goto l147 + } + if !_rules[ruleclose]() { + goto l147 } - position++ - goto l162 - l163: - position, tokenIndex = position163, tokenIndex163 } - add(rulePegText, position159) + l149: + position, tokenIndex = position148, tokenIndex148 } { add(ruleAction32, position) } - goto l131 - l158: - position, tokenIndex = position131, tokenIndex131 + goto l136 + l147: + position, tokenIndex = position136, tokenIndex136 { - position166 := position + position153 := position { - position169, tokenIndex169 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l170 - } - position++ - goto l169 - l170: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l171 - } - position++ - goto l169 - l171: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l172 - } - position++ - goto l169 - l172: - position, tokenIndex = position169, tokenIndex169 + position154, tokenIndex154 := position, tokenIndex if buffer[position] != rune('-') { - goto l173 - } - position++ - goto l169 - l173: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune('_') { - goto l174 - } - position++ - goto l169 - l174: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune(':') { - goto l165 + goto l154 } position++ + goto l155 + l154: + position, tokenIndex = position154, tokenIndex154 } - l169: - l167: + l155: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l152 + } + position++ + l156: { - position168, tokenIndex168 := position, tokenIndex - { - position175, tokenIndex175 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l176 - } - position++ - goto l175 - l176: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l177 - } - position++ - goto l175 - l177: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l178 - } - position++ - goto l175 - l178: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('-') { - goto l179 - } - position++ - goto l175 - l179: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('_') { - goto l180 - } - position++ - goto l175 - l180: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune(':') { - goto l168 - } - position++ + position157, tokenIndex157 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l157 } - l175: - goto l167 - l168: - position, tokenIndex = position168, tokenIndex168 + position++ + goto l156 + l157: + position, tokenIndex = position157, tokenIndex157 } - add(rulePegText, position166) + { + position158, tokenIndex158 := position, tokenIndex + if buffer[position] != rune('.') { + goto l158 + } + position++ + l160: + { + position161, tokenIndex161 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l161 + } + position++ + goto l160 + l161: + position, tokenIndex = position161, tokenIndex161 + } + goto l159 + l158: + position, tokenIndex = position158, tokenIndex158 + } + l159: + add(rulePegText, position153) } { add(ruleAction33, position) } - goto l131 - l165: - position, tokenIndex = position131, tokenIndex131 - if buffer[position] != rune('"') { - goto l182 - } - position++ + goto l136 + l152: + position, tokenIndex = position136, tokenIndex136 { - position183 := position + position164 := position { - position184 := position - l185: - { - position186, tokenIndex186 := position, tokenIndex - { - position187, tokenIndex187 := position, tokenIndex - { - position189, tokenIndex189 := position, tokenIndex - { - position190, tokenIndex190 := position, tokenIndex - if buffer[position] != rune('"') { - goto l191 - } - position++ - goto l190 - l191: - position, tokenIndex = position190, tokenIndex190 - if buffer[position] != rune('\\') { - goto l192 - } - position++ - goto l190 - l192: - position, tokenIndex = position190, tokenIndex190 - if buffer[position] != rune('\n') { - goto l189 - } - position++ - } - l190: - goto l188 - l189: - position, tokenIndex = position189, tokenIndex189 - } - if !matchDot() { - goto l188 - } - goto l187 - l188: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l193 - } - position++ - if buffer[position] != rune('n') { - goto l193 - } - position++ - goto l187 - l193: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l194 - } - position++ - if buffer[position] != rune('"') { - goto l194 - } - position++ - goto l187 - l194: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l195 - } - position++ - if buffer[position] != rune('\'') { - goto l195 - } - position++ - goto l187 - l195: - position, tokenIndex = position187, tokenIndex187 - if buffer[position] != rune('\\') { - goto l186 - } - position++ - if buffer[position] != rune('\\') { - goto l186 - } - position++ - } - l187: - goto l185 - l186: - position, tokenIndex = position186, tokenIndex186 + position165, tokenIndex165 := position, tokenIndex + if buffer[position] != rune('-') { + goto l165 } - add(ruledoublequotedstring, position184) + position++ + goto l166 + l165: + position, tokenIndex = position165, tokenIndex165 } - add(rulePegText, position183) + l166: + if buffer[position] != rune('.') { + goto l163 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 + } + position++ + l167: + { + position168, tokenIndex168 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l168 + } + position++ + goto l167 + l168: + position, tokenIndex = position168, tokenIndex168 + } + add(rulePegText, position164) } - if buffer[position] != rune('"') { - goto l182 - } - position++ { add(ruleAction34, position) } - goto l131 - l182: - position, tokenIndex = position131, tokenIndex131 - if buffer[position] != rune('\'') { - goto l129 - } - position++ + goto l136 + l163: + position, tokenIndex = position136, tokenIndex136 { - position197 := position + position171 := position { - position198 := position - l199: - { - position200, tokenIndex200 := position, tokenIndex - { - position201, tokenIndex201 := position, tokenIndex - { - position203, tokenIndex203 := position, tokenIndex - { - position204, tokenIndex204 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l205 - } - position++ - goto l204 - l205: - position, tokenIndex = position204, tokenIndex204 - if buffer[position] != rune('\\') { - goto l206 - } - position++ - goto l204 - l206: - position, tokenIndex = position204, tokenIndex204 - if buffer[position] != rune('\n') { - goto l203 - } - position++ - } - l204: - goto l202 - l203: - position, tokenIndex = position203, tokenIndex203 - } - if !matchDot() { - goto l202 - } - goto l201 - l202: - position, tokenIndex = position201, tokenIndex201 - if buffer[position] != rune('\\') { - goto l207 - } - position++ - if buffer[position] != rune('n') { - goto l207 - } - position++ - goto l201 - l207: - position, tokenIndex = position201, tokenIndex201 - if buffer[position] != rune('\\') { - goto l208 - } - position++ - if buffer[position] != rune('"') { - goto l208 - } - position++ - goto l201 - l208: - position, tokenIndex = position201, tokenIndex201 - if buffer[position] != rune('\\') { - goto l209 - } - position++ - if buffer[position] != rune('\'') { - goto l209 - } - position++ - goto l201 - l209: - position, tokenIndex = position201, tokenIndex201 - if buffer[position] != rune('\\') { - goto l200 - } - position++ - if buffer[position] != rune('\\') { - goto l200 - } - position++ - } - l201: - goto l199 - l200: - position, tokenIndex = position200, tokenIndex200 + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l175 } - add(rulesinglequotedstring, position198) + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l176 + } + position++ + goto l174 + l176: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l177 + } + position++ + goto l174 + l177: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('-') { + goto l178 + } + position++ + goto l174 + l178: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('_') { + goto l179 + } + position++ + goto l174 + l179: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune(':') { + goto l170 + } + position++ } - add(rulePegText, position197) + l174: + l172: + { + position173, tokenIndex173 := position, tokenIndex + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l173 + } + position++ + } + l180: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + add(rulePegText, position171) } - if buffer[position] != rune('\'') { - goto l129 - } - position++ { add(ruleAction35, position) } + goto l136 + l170: + position, tokenIndex = position136, tokenIndex136 + if buffer[position] != rune('"') { + goto l187 + } + position++ + { + position188 := position + { + position189 := position + l190: + { + position191, tokenIndex191 := position, tokenIndex + { + position192, tokenIndex192 := position, tokenIndex + { + position194, tokenIndex194 := position, tokenIndex + { + position195, tokenIndex195 := position, tokenIndex + if buffer[position] != rune('"') { + goto l196 + } + position++ + goto l195 + l196: + position, tokenIndex = position195, tokenIndex195 + if buffer[position] != rune('\\') { + goto l197 + } + position++ + goto l195 + l197: + position, tokenIndex = position195, tokenIndex195 + if buffer[position] != rune('\n') { + goto l194 + } + position++ + } + l195: + goto l193 + l194: + position, tokenIndex = position194, tokenIndex194 + } + if !matchDot() { + goto l193 + } + goto l192 + l193: + position, tokenIndex = position192, tokenIndex192 + if buffer[position] != rune('\\') { + goto l198 + } + position++ + if buffer[position] != rune('n') { + goto l198 + } + position++ + goto l192 + l198: + position, tokenIndex = position192, tokenIndex192 + if buffer[position] != rune('\\') { + goto l199 + } + position++ + if buffer[position] != rune('"') { + goto l199 + } + position++ + goto l192 + l199: + position, tokenIndex = position192, tokenIndex192 + if buffer[position] != rune('\\') { + goto l200 + } + position++ + if buffer[position] != rune('\'') { + goto l200 + } + position++ + goto l192 + l200: + position, tokenIndex = position192, tokenIndex192 + if buffer[position] != rune('\\') { + goto l191 + } + position++ + if buffer[position] != rune('\\') { + goto l191 + } + position++ + } + l192: + goto l190 + l191: + position, tokenIndex = position191, tokenIndex191 + } + add(ruledoublequotedstring, position189) + } + add(rulePegText, position188) + } + if buffer[position] != rune('"') { + goto l187 + } + position++ + { + add(ruleAction36, position) + } + goto l136 + l187: + position, tokenIndex = position136, tokenIndex136 + if buffer[position] != rune('\'') { + goto l134 + } + position++ + { + position202 := position + { + position203 := position + l204: + { + position205, tokenIndex205 := position, tokenIndex + { + position206, tokenIndex206 := position, tokenIndex + { + position208, tokenIndex208 := position, tokenIndex + { + position209, tokenIndex209 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l210 + } + position++ + goto l209 + l210: + position, tokenIndex = position209, tokenIndex209 + if buffer[position] != rune('\\') { + goto l211 + } + position++ + goto l209 + l211: + position, tokenIndex = position209, tokenIndex209 + if buffer[position] != rune('\n') { + goto l208 + } + position++ + } + l209: + goto l207 + l208: + position, tokenIndex = position208, tokenIndex208 + } + if !matchDot() { + goto l207 + } + goto l206 + l207: + position, tokenIndex = position206, tokenIndex206 + if buffer[position] != rune('\\') { + goto l212 + } + position++ + if buffer[position] != rune('n') { + goto l212 + } + position++ + goto l206 + l212: + position, tokenIndex = position206, tokenIndex206 + if buffer[position] != rune('\\') { + goto l213 + } + position++ + if buffer[position] != rune('"') { + goto l213 + } + position++ + goto l206 + l213: + position, tokenIndex = position206, tokenIndex206 + if buffer[position] != rune('\\') { + goto l214 + } + position++ + if buffer[position] != rune('\'') { + goto l214 + } + position++ + goto l206 + l214: + position, tokenIndex = position206, tokenIndex206 + if buffer[position] != rune('\\') { + goto l205 + } + position++ + if buffer[position] != rune('\\') { + goto l205 + } + position++ + } + l206: + goto l204 + l205: + position, tokenIndex = position205, tokenIndex205 + } + add(rulesinglequotedstring, position203) + } + add(rulePegText, position202) + } + if buffer[position] != rune('\'') { + goto l134 + } + position++ + { + add(ruleAction37, position) + } } - l131: - add(ruleitem, position130) + l136: + add(ruleitem, position135) } return true - l129: - position, tokenIndex = position129, tokenIndex129 + l134: + position, tokenIndex = position134, tokenIndex134 return false }, - /* 13 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 14 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, - /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ + /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ func() bool { - position213, tokenIndex213 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex { - position214 := position + position219 := position { - position215, tokenIndex215 := position, tokenIndex + position220, tokenIndex220 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l216 + goto l221 } position++ - goto l215 - l216: - position, tokenIndex = position215, tokenIndex215 + goto l220 + l221: + position, tokenIndex = position220, tokenIndex220 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l213 + goto l218 } position++ } - l215: - l217: + l220: + l222: { - position218, tokenIndex218 := position, tokenIndex + position223, tokenIndex223 := position, tokenIndex { - position219, tokenIndex219 := position, tokenIndex + position224, tokenIndex224 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l220 + goto l225 } position++ - goto l219 - l220: - position, tokenIndex = position219, tokenIndex219 + goto l224 + l225: + position, tokenIndex = position224, tokenIndex224 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l221 + goto l226 } position++ - goto l219 - l221: - position, tokenIndex = position219, tokenIndex219 + goto l224 + l226: + position, tokenIndex = position224, tokenIndex224 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l222 + goto l227 } position++ - goto l219 - l222: - position, tokenIndex = position219, tokenIndex219 + goto l224 + l227: + position, tokenIndex = position224, tokenIndex224 if buffer[position] != rune('_') { - goto l218 + goto l223 } position++ } - l219: - goto l217 - l218: - position, tokenIndex = position218, tokenIndex218 + l224: + goto l222 + l223: + position, tokenIndex = position223, tokenIndex223 } - add(rulefieldExpr, position214) + add(rulefieldExpr, position219) } return true - l213: - position, tokenIndex = position213, tokenIndex213 + l218: + position, tokenIndex = position218, tokenIndex218 return false }, - /* 16 field <- <( Action36)> */ + /* 17 field <- <( Action38)> */ func() bool { - position223, tokenIndex223 := position, tokenIndex + position228, tokenIndex228 := position, tokenIndex { - position224 := position + position229 := position { - position225 := position + position230 := position if !_rules[rulefieldExpr]() { - goto l223 + goto l228 } - add(rulePegText, position225) + add(rulePegText, position230) } { - add(ruleAction36, position) + add(ruleAction38, position) } - add(rulefield, position224) + add(rulefield, position229) } return true - l223: - position, tokenIndex = position223, tokenIndex223 + l228: + position, tokenIndex = position228, tokenIndex228 return false }, - /* 17 posfield <- <( Action37)> */ + /* 18 posfield <- <( Action39)> */ func() bool { - position227, tokenIndex227 := position, tokenIndex + position232, tokenIndex232 := position, tokenIndex { - position228 := position + position233 := position { - position229 := position + position234 := position if !_rules[rulefieldExpr]() { - goto l227 + goto l232 } - add(rulePegText, position229) - } - { - add(ruleAction37, position) - } - add(ruleposfield, position228) - } - return true - l227: - position, tokenIndex = position227, tokenIndex227 - return false - }, - /* 18 uint <- <(([1-9] [0-9]*) / '0')> */ - func() bool { - position231, tokenIndex231 := position, tokenIndex - { - position232 := position - { - position233, tokenIndex233 := position, tokenIndex - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l234 - } - position++ - l235: - { - position236, tokenIndex236 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l236 - } - position++ - goto l235 - l236: - position, tokenIndex = position236, tokenIndex236 - } - goto l233 - l234: - position, tokenIndex = position233, tokenIndex233 - if buffer[position] != rune('0') { - goto l231 - } - position++ - } - l233: - add(ruleuint, position232) - } - return true - l231: - position, tokenIndex = position231, tokenIndex231 - return false - }, - /* 19 uintrow <- <( Action38)> */ - nil, - /* 20 uintcol <- <( Action39)> */ - func() bool { - position238, tokenIndex238 := position, tokenIndex - { - position239 := position - { - position240 := position - if !_rules[ruleuint]() { - goto l238 - } - add(rulePegText, position240) + add(rulePegText, position234) } { add(ruleAction39, position) } - add(ruleuintcol, position239) + add(ruleposfield, position233) } return true - l238: - position, tokenIndex = position238, tokenIndex238 + l232: + position, tokenIndex = position232, tokenIndex232 return false }, - /* 21 open <- <('(' sp)> */ + /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position242, tokenIndex242 := position, tokenIndex + position236, tokenIndex236 := position, tokenIndex { - position243 := position - if buffer[position] != rune('(') { - goto l242 - } - position++ - if !_rules[rulesp]() { - goto l242 - } - add(ruleopen, position243) - } - return true - l242: - position, tokenIndex = position242, tokenIndex242 - return false - }, - /* 22 close <- <(')' sp)> */ - func() bool { - position244, tokenIndex244 := position, tokenIndex - { - position245 := position - if buffer[position] != rune(')') { - goto l244 - } - position++ - if !_rules[rulesp]() { - goto l244 - } - add(ruleclose, position245) - } - return true - l244: - position, tokenIndex = position244, tokenIndex244 - return false - }, - /* 23 sp <- <(' ' / '\t')*> */ - func() bool { - { - position247 := position - l248: + position237 := position { - position249, tokenIndex249 := position, tokenIndex + position238, tokenIndex238 := position, tokenIndex + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l239 + } + position++ + l240: { - position250, tokenIndex250 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l251 + position241, tokenIndex241 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l241 } position++ - goto l250 - l251: - position, tokenIndex = position250, tokenIndex250 + goto l240 + l241: + position, tokenIndex = position241, tokenIndex241 + } + goto l238 + l239: + position, tokenIndex = position238, tokenIndex238 + if buffer[position] != rune('0') { + goto l236 + } + position++ + } + l238: + add(ruleuint, position237) + } + return true + l236: + position, tokenIndex = position236, tokenIndex236 + return false + }, + /* 20 uintrow <- <( Action40)> */ + nil, + /* 21 uintcol <- <( Action41)> */ + func() bool { + position243, tokenIndex243 := position, tokenIndex + { + position244 := position + { + position245 := position + if !_rules[ruleuint]() { + goto l243 + } + add(rulePegText, position245) + } + { + add(ruleAction41, position) + } + add(ruleuintcol, position244) + } + return true + l243: + position, tokenIndex = position243, tokenIndex243 + return false + }, + /* 22 open <- <('(' sp)> */ + func() bool { + position247, tokenIndex247 := position, tokenIndex + { + position248 := position + if buffer[position] != rune('(') { + goto l247 + } + position++ + if !_rules[rulesp]() { + goto l247 + } + add(ruleopen, position248) + } + return true + l247: + position, tokenIndex = position247, tokenIndex247 + return false + }, + /* 23 close <- <(')' sp)> */ + func() bool { + position249, tokenIndex249 := position, tokenIndex + { + position250 := position + if buffer[position] != rune(')') { + goto l249 + } + position++ + if !_rules[rulesp]() { + goto l249 + } + add(ruleclose, position250) + } + return true + l249: + position, tokenIndex = position249, tokenIndex249 + return false + }, + /* 24 sp <- <(' ' / '\t')*> */ + func() bool { + { + position252 := position + l253: + { + position254, tokenIndex254 := position, tokenIndex + { + position255, tokenIndex255 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l256 + } + position++ + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 if buffer[position] != rune('\t') { - goto l249 + goto l254 } position++ } - l250: - goto l248 - l249: - position, tokenIndex = position249, tokenIndex249 + l255: + goto l253 + l254: + position, tokenIndex = position254, tokenIndex254 } - add(rulesp, position247) + add(rulesp, position252) } return true }, - /* 24 comma <- <(sp ',' whitesp)> */ + /* 25 comma <- <(sp ',' whitesp)> */ func() bool { - position252, tokenIndex252 := position, tokenIndex + position257, tokenIndex257 := position, tokenIndex { - position253 := position + position258 := position if !_rules[rulesp]() { - goto l252 + goto l257 } if buffer[position] != rune(',') { - goto l252 + goto l257 } position++ if !_rules[rulewhitesp]() { - goto l252 + goto l257 } - add(rulecomma, position253) + add(rulecomma, position258) } return true - l252: - position, tokenIndex = position252, tokenIndex252 + l257: + position, tokenIndex = position257, tokenIndex257 return false }, - /* 25 lbrack <- <('[' sp)> */ + /* 26 lbrack <- <('[' sp)> */ nil, - /* 26 rbrack <- <(sp ']' sp)> */ + /* 27 rbrack <- <(sp ']' sp)> */ nil, - /* 27 whitesp <- <(' ' / '\t' / '\n')*> */ + /* 28 whitesp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position257 := position - l258: + position262 := position + l263: { - position259, tokenIndex259 := position, tokenIndex + position264, tokenIndex264 := position, tokenIndex { - position260, tokenIndex260 := position, tokenIndex + position265, tokenIndex265 := position, tokenIndex if buffer[position] != rune(' ') { - goto l261 + goto l266 } position++ - goto l260 - l261: - position, tokenIndex = position260, tokenIndex260 + goto l265 + l266: + position, tokenIndex = position265, tokenIndex265 if buffer[position] != rune('\t') { - goto l262 + goto l267 } position++ - goto l260 - l262: - position, tokenIndex = position260, tokenIndex260 + goto l265 + l267: + position, tokenIndex = position265, tokenIndex265 if buffer[position] != rune('\n') { - goto l259 + goto l264 } position++ } - l260: - goto l258 - l259: - position, tokenIndex = position259, tokenIndex259 + l265: + goto l263 + l264: + position, tokenIndex = position264, tokenIndex264 } - add(rulewhitesp, position257) + add(rulewhitesp, position262) } return true }, - /* 28 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 29 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, - /* 29 timestamp <- <(<([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> Action40)> */ + /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + func() bool { + position269, tokenIndex269 := position, tokenIndex + { + position270 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if buffer[position] != rune('-') { + goto l269 + } + position++ + { + position271, tokenIndex271 := position, tokenIndex + if buffer[position] != rune('0') { + goto l272 + } + position++ + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if buffer[position] != rune('1') { + goto l269 + } + position++ + } + l271: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if buffer[position] != rune('-') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if buffer[position] != rune('T') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if buffer[position] != rune(':') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l269 + } + position++ + add(ruletimestampbasicfmt, position270) + } + return true + l269: + position, tokenIndex = position269, tokenIndex269 + return false + }, + /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + func() bool { + position273, tokenIndex273 := position, tokenIndex + { + position274 := position + { + position275, tokenIndex275 := position, tokenIndex + if buffer[position] != rune('"') { + goto l276 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l276 + } + if buffer[position] != rune('"') { + goto l276 + } + position++ + goto l275 + l276: + position, tokenIndex = position275, tokenIndex275 + if buffer[position] != rune('\'') { + goto l277 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l277 + } + if buffer[position] != rune('\'') { + goto l277 + } + position++ + goto l275 + l277: + position, tokenIndex = position275, tokenIndex275 + if !_rules[ruletimestampbasicfmt]() { + goto l273 + } + } + l275: + add(ruletimestampfmt, position274) + } + return true + l273: + position, tokenIndex = position273, tokenIndex273 + return false + }, + /* 32 timestamp <- <( Action42)> */ nil, - /* 31 Action0 <- <{p.startCall("Set")}> */ + /* 34 Action0 <- <{p.startCall("Set")}> */ nil, - /* 32 Action1 <- <{p.endCall()}> */ + /* 35 Action1 <- <{p.endCall()}> */ nil, - /* 33 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 34 Action3 <- <{p.endCall()}> */ + /* 37 Action3 <- <{p.endCall()}> */ nil, - /* 35 Action4 <- <{p.startCall("SetColAttrs")}> */ + /* 38 Action4 <- <{p.startCall("SetColAttrs")}> */ nil, - /* 36 Action5 <- <{p.endCall()}> */ + /* 39 Action5 <- <{p.endCall()}> */ nil, - /* 37 Action6 <- <{p.startCall("Clear")}> */ + /* 40 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 38 Action7 <- <{p.endCall()}> */ + /* 41 Action7 <- <{p.endCall()}> */ nil, - /* 39 Action8 <- <{p.startCall("TopN")}> */ + /* 42 Action8 <- <{p.startCall("TopN")}> */ nil, - /* 40 Action9 <- <{p.endCall()}> */ + /* 43 Action9 <- <{p.endCall()}> */ nil, - /* 41 Action10 <- <{p.startCall("Range")}> */ + /* 44 Action10 <- <{p.startCall("Range")}> */ nil, - /* 42 Action11 <- <{p.endCall()}> */ + /* 45 Action11 <- <{p.endCall()}> */ nil, nil, - /* 44 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 45 Action13 <- <{ p.endCall() }> */ + /* 48 Action13 <- <{ p.endCall() }> */ nil, - /* 46 Action14 <- <{ p.addBTWN() }> */ + /* 49 Action14 <- <{ p.addBTWN() }> */ nil, - /* 47 Action15 <- <{ p.addLTE() }> */ + /* 50 Action15 <- <{ p.addLTE() }> */ nil, - /* 48 Action16 <- <{ p.addGTE() }> */ + /* 51 Action16 <- <{ p.addGTE() }> */ nil, - /* 49 Action17 <- <{ p.addEQ() }> */ + /* 52 Action17 <- <{ p.addEQ() }> */ nil, - /* 50 Action18 <- <{ p.addNEQ() }> */ + /* 53 Action18 <- <{ p.addNEQ() }> */ nil, - /* 51 Action19 <- <{ p.addLT() }> */ + /* 54 Action19 <- <{ p.addLT() }> */ nil, - /* 52 Action20 <- <{ p.addGT() }> */ + /* 55 Action20 <- <{ p.addGT() }> */ nil, - /* 53 Action21 <- <{p.startConditional()}> */ + /* 56 Action21 <- <{p.startConditional()}> */ nil, - /* 54 Action22 <- <{p.endConditional()}> */ + /* 57 Action22 <- <{p.endConditional()}> */ nil, - /* 55 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 56 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 57 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 58 Action26 <- <{ p.startList() }> */ + /* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 59 Action27 <- <{ p.endList() }> */ + /* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 60 Action28 <- <{ p.addVal(nil) }> */ + /* 63 Action28 <- <{ p.startList() }> */ nil, - /* 61 Action29 <- <{ p.addVal(true) }> */ + /* 64 Action29 <- <{ p.endList() }> */ nil, - /* 62 Action30 <- <{ p.addVal(false) }> */ + /* 65 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 63 Action31 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 66 Action31 <- <{ p.addVal(true) }> */ nil, - /* 64 Action32 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 67 Action32 <- <{ p.addVal(false) }> */ nil, - /* 65 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 66 Action34 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 67 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 68 Action36 <- <{ p.addField(buffer[begin:end]) }> */ + /* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 69 Action37 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 70 Action38 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 71 Action39 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 72 Action40 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + nil, + /* 77 Action42 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index ea7c4a3a1..94a670181 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -205,6 +205,14 @@ func TestPEGWorking(t *testing.T) { name: "RangeLTELTE", input: "Range(4 <= a <= 9)", ncalls: 1}, + { + name: "RangeTime", + input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)", + ncalls: 1}, + { + name: "RangeTimeQuotes", + input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`, + ncalls: 1}, } for i, test := range tests { @@ -264,6 +272,12 @@ func TestPEGErrors(t *testing.T) { { name: "Clear0args", input: "Clear(9)"}, + { + name: "RangeTimeGT", + input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, + { + name: "RangeTimeOneStamp", + input: "Range(a=4, 2010-07-04T00:00)"}, } for i, test := range tests { From 205b7620bde709ec78117bcb3ff5901895443c7c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 20 Jun 2018 09:31:20 -0500 Subject: [PATCH 11/33] update SetColumnAttrs name --- pql/pql.peg | 4 ++-- pql/pql.peg.go | 32 ++++++++++++++++++++++++++++---- pql/pqlpeg_test.go | 18 +++++++++--------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/pql/pql.peg b/pql/pql.peg index d5f82f49c..a094aa663 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -8,7 +8,7 @@ type PQL Peg { Calls <- whitesp (Call whitesp)* !. Call <- 'Set' {p.startCall("Set")} open uintcol comma args (comma timestamp)? close {p.endCall()} / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} - / 'SetColAttrs' {p.startCall("SetColAttrs")} open posfield comma uintcol comma args close {p.endCall()} + / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open posfield comma uintcol comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open uintcol comma args close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} @@ -64,7 +64,7 @@ comma <- sp ',' whitesp lbrack <- '[' sp rbrack <- sp ']' sp whitesp <- ( ' ' / '\t' / '\n' )* -IDENT <- !('Set(' / 'SetRowAttrs(' / 'SetColAttrs(' / 'Clear(' / 'TopN(' / 'Range(') [[A-Z]] ([[A-Z]] / [0-9])* +IDENT <- !('Set(' / 'SetRowAttrs(' / 'SetColumnAttrs(' / 'Clear(' / 'TopN(' / 'Range(') [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 467c14883..f62f21552 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -391,7 +391,7 @@ func (p *PQL) Execute() { case ruleAction3: p.endCall() case ruleAction4: - p.startCall("SetColAttrs") + p.startCall("SetColumnAttrs") case ruleAction5: p.endCall() case ruleAction6: @@ -579,7 +579,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -755,6 +755,18 @@ func (p *PQL) Init() { goto l22 } position++ + if buffer[position] != rune('u') { + goto l22 + } + position++ + if buffer[position] != rune('m') { + goto l22 + } + position++ + if buffer[position] != rune('n') { + goto l22 + } + position++ if buffer[position] != rune('A') { goto l22 } @@ -1131,6 +1143,18 @@ func (p *PQL) Init() { goto l56 } position++ + if buffer[position] != rune('u') { + goto l56 + } + position++ + if buffer[position] != rune('m') { + goto l56 + } + position++ + if buffer[position] != rune('n') { + goto l56 + } + position++ if buffer[position] != rune('A') { goto l56 } @@ -2606,7 +2630,7 @@ func (p *PQL) Init() { } return true }, - /* 29 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 29 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { @@ -2752,7 +2776,7 @@ func (p *PQL) Init() { nil, /* 37 Action3 <- <{p.endCall()}> */ nil, - /* 38 Action4 <- <{p.startCall("SetColAttrs")}> */ + /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, /* 39 Action5 <- <{p.endCall()}> */ nil, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 94a670181..023bbc71c 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -142,12 +142,12 @@ func TestPEGWorking(t *testing.T) { input: "SetRowAttrs(blah, 9, a=47, b=bval)", ncalls: 1}, { - name: "SetColAttrs", - input: "SetColAttrs(blah, 9, a=47)", + name: "SetColumnAttrs", + input: "SetColumnAttrs(blah, 9, a=47)", ncalls: 1}, { - name: "SetColAttrs2args", - input: "SetColAttrs(blah, 9, a=47, b=bval)", + name: "SetColumnAttrs2args", + input: "SetColumnAttrs(blah, 9, a=47, b=bval)", ncalls: 1}, { name: "Clear", @@ -252,8 +252,8 @@ func TestPEGErrors(t *testing.T) { name: "SetRowAttrsNoField", input: "SetRowAttrs(a=4)"}, { - name: "SetColAttrsNoField", - input: "SetColAttrs(a=4)"}, + name: "SetColumnAttrsNoField", + input: "SetColumnAttrs(a=4)"}, { name: "ClearNoCol", input: "Clear(a=4)"}, @@ -319,10 +319,10 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { - name: "SetColAttrs", - call: "SetColAttrs(myfield, 9, z=4)", + name: "SetColumnAttrs", + call: "SetColumnAttrs(myfield, 9, z=4)", exp: &Call{ - Name: "SetColAttrs", + Name: "SetColumnAttrs", Args: map[string]interface{}{ "z": int64(4), "_field": "myfield", From c6db3974bc4b42085e7b8479e028a054eb757510 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 21 Jun 2018 12:20:31 -0500 Subject: [PATCH 12/33] WIP API refactor --- api.go | 31 +++++++++++- cmd/server_test.go | 4 +- ctl/export_test.go | 2 + ctl/import_test.go | 5 ++ executor_test.go | 10 ++++ handler.go | 17 +++---- holder_test.go | 2 + http/client_test.go | 8 +++ http/handler.go | 38 ++++++++++++--- http/handler_test.go | 105 ++++++++++++++++++++++++++++++++-------- http/translator_test.go | 2 + server.go | 58 +++++----------------- server/server.go | 55 +++++++++++++-------- test/handler.go | 35 ++++++++------ 14 files changed, 252 insertions(+), 120 deletions(-) diff --git a/api.go b/api.go index 7adff1d1b..4fdb0de6f 100644 --- a/api.go +++ b/api.go @@ -46,16 +46,43 @@ type API struct { Cluster *Cluster TranslateStore TranslateStore Logger Logger + server *Server +} + +// APIOption is a functional option type for pilosa.API +type APIOption func(s *API) error + +func OptAPIServer(s *Server) APIOption { + return func(a *API) error { + a.server = s + a.Executor = s.executor + a.TranslateStore = s.translateFile + a.Holder = s.holder + a.Broadcaster = s + a.BroadcastHandler = s + a.StatusHandler = s + a.Cluster = s.Cluster + a.Logger = s.logger + return nil + } } // NewAPI returns a new API instance. -func NewAPI() *API { - return &API{ +func NewAPI(opts ...APIOption) (*API, error) { + api := &API{ Broadcaster: NopBroadcaster, //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop //StatusHandler: NopStatusHandler, // TODO: implement the nop Logger: NopLogger, } + + for _, opt := range opts { + err := opt(api) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + return api, nil } // validAPIMethods specifies the api methods that are valid for each diff --git a/cmd/server_test.go b/cmd/server_test.go index abbe8d7a4..989c8de6b 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -15,7 +15,6 @@ package cmd_test import ( - "errors" "io/ioutil" "strings" "testing" @@ -24,6 +23,7 @@ import ( "github.com/pilosa/pilosa/cmd" _ "github.com/pilosa/pilosa/test" "github.com/pilosa/pilosa/toml" + "github.com/pkg/errors" ) func TestServerHelp(t *testing.T) { @@ -35,6 +35,8 @@ func TestServerHelp(t *testing.T) { } func TestServerConfig(t *testing.T) { + t.Skip() // Until test.NewServer() works + actualDataDir, err := ioutil.TempDir("", "") failErr(t, err, "making data dir") logFile, err := ioutil.TempFile("", "") diff --git a/ctl/export_test.go b/ctl/export_test.go index 5e87334d9..d405c92c4 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -44,6 +44,8 @@ func TestExportCommand_Validation(t *testing.T) { } func TestExportCommand_Run(t *testing.T) { + t.Skip() // Until test.NewServer() works + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewExportCommand(stdin, stdout, stderr) diff --git a/ctl/import_test.go b/ctl/import_test.go index 2284ca873..df2b9a5a6 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -29,6 +29,8 @@ import ( ) func TestImportCommand_Validation(t *testing.T) { + t.Skip() // Until test.NewServer() works + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -51,6 +53,7 @@ func TestImportCommand_Validation(t *testing.T) { } func TestImportCommand_Run(t *testing.T) { + t.Skip() // Until test.NewServer() works buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -84,6 +87,7 @@ func TestImportCommand_Run(t *testing.T) { // Ensure that the ImportValue path runs. func TestImportCommand_RunValue(t *testing.T) { + t.Skip() // Until test.NewServer() works buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -118,6 +122,7 @@ func TestImportCommand_RunValue(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { + t.Skip() // Until test.NewServer() works hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/executor_test.go b/executor_test.go index 164133ecb..f50c139f1 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1026,6 +1026,8 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { + t.Skip() // Until test.NewServer() works + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. @@ -1074,6 +1076,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { // Ensure a remote query can return a count. func TestExecutor_Execute_Remote_Count(t *testing.T) { + t.Skip() // Until test.NewServer() works + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. @@ -1109,6 +1113,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { + t.Skip() // Until test.NewServer() works + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 @@ -1161,6 +1167,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { + t.Skip() // Until test.NewServer() works + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 @@ -1215,6 +1223,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Ensure a remote query can return a top-n query. func TestExecutor_Execute_Remote_TopN(t *testing.T) { + t.Skip() // Until test.NewServer() works + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. diff --git a/handler.go b/handler.go index 7c2b76a3f..c9a476e13 100644 --- a/handler.go +++ b/handler.go @@ -2,7 +2,6 @@ package pilosa import ( "encoding/json" - "net" ) // QueryRequest represent a request to process a query. @@ -61,18 +60,18 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { } type Handler interface { - Serve(ln net.Listener, closing <-chan struct{}) - GetAPI() *API + Serve() error + Close() error } -type NopHandler struct{} +type nopHandler struct{} -func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {} - -func (n *NopHandler) GetAPI() *API { +func (n nopHandler) Serve() error { return nil } -func NewNopHandler() Handler { - return &NopHandler{} +func (n nopHandler) Close() error { + return nil } + +var NopHandler Handler = nopHandler{} diff --git a/holder_test.go b/holder_test.go index c70ffcca6..9b3e21c12 100644 --- a/holder_test.go +++ b/holder_test.go @@ -350,6 +350,8 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { + t.Skip() // Until test.NewServer() works + s := test.NewServer() defer s.Close() diff --git a/http/client_test.go b/http/client_test.go index 185a3c378..46f9ce619 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -52,6 +52,8 @@ func init() { // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { + t.Skip() // Until test.NewServer() works + cluster := test.NewCluster(3) s, hldr := createCluster(cluster) @@ -217,6 +219,8 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -251,6 +255,8 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -328,6 +334,8 @@ func TestClient_ImportValue(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/http/handler.go b/http/handler.go index 674381905..9b3d17789 100644 --- a/http/handler.go +++ b/http/handler.go @@ -15,6 +15,7 @@ package http import ( + "context" "crypto/tls" "encoding/json" "expvar" @@ -53,6 +54,10 @@ type Handler struct { API *pilosa.API AllowedOrigins []string + + ln net.Listener + + server *http.Server } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -99,6 +104,13 @@ func OptHandlerLogger(logger pilosa.Logger) HandlerOption { } } +func OptHandlerListener(ln net.Listener) HandlerOption { + return func(h *Handler) error { + h.ln = ln + return nil + } +} + // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...HandlerOption) (*Handler, error) { handler := &Handler{ @@ -114,19 +126,31 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { } } + if handler.API == nil { + return nil, errors.New("must pass OptHandlerAPI") + } + + if handler.ln == nil { + return nil, errors.New("must pass OptHandlerListener") + } + return handler, nil } -func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) { - server := &http.Server{Handler: h} - go func() { - <-closing - server.Close() - }() - err := server.Serve(ln) +func (h *Handler) Serve() error { + h.server = &http.Server{Handler: h} + err := h.server.Serve(h.ln) if err != nil && err.Error() != "http: Server closed" { h.Logger.Printf("HTTP handler terminated with error: %s\n", err) + return errors.Wrap(err, "serve http") } + return nil +} + +func (h *Handler) Close() error { + // TODO: timeout? + err := h.server.Shutdown(context.Background()) + return errors.Wrap(err, "shutdown http server") } func (h *Handler) populateValidators() { diff --git a/http/handler_test.go b/http/handler_test.go index ddedef49a..f249750c2 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -17,50 +17,54 @@ package http_test import ( "bytes" "context" - "errors" "fmt" "io" "io/ioutil" - gohttp "net/http" + "net" "net/http/httptest" "reflect" "strings" "testing" + gohttp "net/http" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" ) -func TestHandlerPanics(t *testing.T) { - h := test.MustNewHandler() - bufLogger := test.NewBufferLogger() - h.Handler.Logger = bufLogger - - w := httptest.NewRecorder() - // will panic since Handler has no Holder set up - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) - bufbytes, err := bufLogger.ReadAll() +func TestHandlerOptions(t *testing.T) { + _, err := http.NewHandler() + if err == nil { + t.Fatalf("expected error making handler without options, got nil") + } + _, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{})) + if err == nil { + t.Fatalf("expected error making handler without options, got nil") + } + ln, err := net.Listen("tcp", ":0") if err != nil { - t.Fatalf("reading all logoutput: %v", err) + t.Fatal(err) } - if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { - t.Fatalf("expected panic in log, but got: %s", bufbytes) - } - if w.Code != gohttp.StatusInternalServerError { - t.Fatalf("expected internal server error, but got: %v", w.Code) - } - bodyBytes := w.Body.Bytes() - if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { - t.Fatalf("response to client should have panic, but got %s", bodyBytes) + _, err = http.NewHandler(http.OptHandlerListener(ln)) + if err == nil { + t.Fatalf("expected error making handler without options, got nil") } } +func TestHandler_Endpoints(t *testing.T) { + mains := test.MustRunMainWithCluster(t, 1) + _ = mains[0] +} + // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -77,6 +81,8 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -112,6 +118,8 @@ func TestHandler_Schema(t *testing.T) { // Ensure the handler can return the status. func TestHandler_Status(t *testing.T) { + t.Skip() // Until test.NewServer() works + s := test.NewServer() hldr := test.MustOpenHolder() defer s.Close() @@ -151,6 +159,8 @@ func TestHandler_Status(t *testing.T) { } func TestHandler_Info(t *testing.T) { + t.Skip() // Until test.NewServer() works + s := test.NewServer() defer s.Close() h := test.MustNewHandler() @@ -166,6 +176,7 @@ func TestHandler_Info(t *testing.T) { // Ensure the handler can abort a cluster resize. func TestHandler_ClusterResizeAbort(t *testing.T) { + t.Skip() // Until test.NewServer() works t.Run("No resize job", func(t *testing.T) { h := test.MustNewHandler() @@ -186,6 +197,8 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -211,6 +224,8 @@ func TestHandler_MaxSlices(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -239,6 +254,8 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -278,6 +295,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Ensure the handler returns an error when parsing bad arguments. func TestHandler_Query_Args_Err(t *testing.T) { + t.Skip() // Until test.NewServer() works + w := httptest.NewRecorder() hldr := test.MustOpenHolder() defer hldr.Close() @@ -294,6 +313,8 @@ func TestHandler_Query_Args_Err(t *testing.T) { } } func TestHandler_Query_Params_Err(t *testing.T) { + t.Skip() // Until test.NewServer() works + w := httptest.NewRecorder() test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) if w.Code != gohttp.StatusBadRequest { @@ -306,6 +327,8 @@ func TestHandler_Query_Params_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -327,6 +350,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -357,6 +382,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -380,6 +407,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a row with column attributes as JSON. func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.NewHolder() defer hldr.Close() @@ -413,6 +442,8 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a row as protobuf. func TestHandler_Query_Row_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -453,6 +484,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a row with column attributes as protobuf. func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.NewHolder() defer hldr.Close() @@ -522,6 +555,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -546,6 +581,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -579,6 +616,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -600,6 +639,8 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -628,6 +669,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { // Ensure the handler returns "method not allowed" for non-POST queries. func TestHandler_Query_MethodNotAllowed(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -643,6 +686,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { // Ensure the handler returns an error if there is a parsing error.. func TestHandler_Query_ErrParse(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -660,6 +705,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { // Ensure the handler can delete an index. func TestHandler_Index_Delete(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -696,6 +743,8 @@ func TestHandler_Index_Delete(t *testing.T) { // Ensure handler can delete a field. func TestHandler_DeleteField(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) @@ -719,6 +768,8 @@ func TestHandler_DeleteField(t *testing.T) { // Ensure the handler can return data in differing blocks for an index. func TestHandler_Index_AttrStore_Diff(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -774,6 +825,8 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { // Ensure the handler can return data in differing blocks for a field. func TestHandler_Field_AttrStore_Diff(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -830,6 +883,8 @@ func TestHandler_Field_AttrStore_Diff(t *testing.T) { // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -853,6 +908,8 @@ func TestHandler_Version(t *testing.T) { // Ensure the handler can return a list of nodes for a fragment. func TestHandler_Fragment_Nodes(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -889,6 +946,8 @@ func TestHandler_Fragment_Nodes(t *testing.T) { // Ensure the handler can return expvars without panicking. func TestHandler_Expvars(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -912,6 +971,8 @@ func MustReadAll(r io.Reader) []byte { } func TestHandler_RecalculateCaches(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() @@ -928,6 +989,8 @@ func TestHandler_RecalculateCaches(t *testing.T) { } func TestHandler_CORS(t *testing.T) { + t.Skip() // Until test.NewServer() works + hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/http/translator_test.go b/http/translator_test.go index 3378ddc58..8bedf22cd 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -15,6 +15,8 @@ import ( ) func TestTranslateStore_Reader(t *testing.T) { + t.Skip() // Until test.NewServer() works + // Ensure client can connect and stream the translate store data. t.Run("OK", func(t *testing.T) { t.Run("ServerDisconnect", func(t *testing.T) { diff --git a/server.go b/server.go index b6c4e7996..c575d5a2c 100644 --- a/server.go +++ b/server.go @@ -61,12 +61,10 @@ type Server struct { clusterDisabled bool // External - handler Handler BroadcastReceiver BroadcastReceiver systemInfo SystemInfo gcNotifier GCNotifier logger Logger - ln net.Listener NodeID string URI URI @@ -126,13 +124,6 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } -func OptServerHandler(h Handler) ServerOption { - return func(s *Server) error { - s.handler = h - return nil - } -} - func OptServerMaxWritesPerRequest(n int) ServerOption { return func(s *Server) error { s.maxWritesPerRequest = n @@ -191,14 +182,6 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } -func OptServerListener(ln net.Listener) ServerOption { - return func(s *Server) error { - s.ln = ln - - return nil - } -} - func OptServerURI(uri *URI) ServerOption { return func(s *Server) error { s.URI = *uri @@ -264,11 +247,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, err } - // update URI port with actual listener port. TODO this should probably be done outside of here. - if s.URI.Port() == 0 { - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) - } - // Get or create NodeID. s.NodeID = s.LoadNodeID() // Set Cluster Node. @@ -293,8 +271,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Cluster = s.Cluster s.executor.TranslateStore = s.translateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest - s.handler.GetAPI().Executor = s.executor - s.handler.GetAPI().TranslateStore = s.translateFile return s, nil } @@ -302,9 +278,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Open opens and initializes the server. func (s *Server) Open() error { s.logger.Printf("open server") - if s.ln == nil { - return errors.New("must pass a listener option to NewServer") - } // Log startup err := s.holder.logStartup() @@ -316,20 +289,9 @@ func (s *Server) Open() error { s.Cluster.Broadcaster = s s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest - // Initialize HTTP handler. - api := s.handler.GetAPI() - api.Holder = s.holder - api.Broadcaster = s - api.BroadcastHandler = s - api.StatusHandler = s - api.Cluster = s.Cluster - // Initialize Holder. s.holder.Broadcaster = s - // Serve handler. - go s.handler.Serve(s.ln, s.closing) - // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { return fmt.Errorf("starting BroadcastReceiver: %v", err) @@ -370,9 +332,6 @@ func (s *Server) Close() error { close(s.closing) s.wg.Wait() - if s.ln != nil { - s.ln.Close() - } if s.Cluster != nil { s.Cluster.close() } @@ -400,12 +359,21 @@ func (s *Server) LoadNodeID() string { return nodeID } +type pilosaAddr URI + +func (p pilosaAddr) String() string { + uri := URI(p) + return uri.HostPort() + +} + +func (pilosaAddr) Network() string { + return "tcp" +} + // Addr returns the address of the listener. func (s *Server) Addr() net.Addr { - if s.ln == nil { - return nil - } - return s.ln.Addr() + return pilosaAddr(s.URI) } func (s *Server) monitorAntiEntropy() { diff --git a/server/server.go b/server/server.go index d97e36520..e61ba6fad 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,9 @@ type Command struct { // Passed to the Gossip implementation. logOutput io.Writer logger loggerLogger + + handler pilosa.Handler + ln net.Listener } // NewCommand returns a new instance of Main. @@ -108,6 +111,13 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "opening server") } + go func() { + err := m.handler.Serve() + if err != nil { + m.logger.Printf("Handler serve error: %v", err) + } + }() + m.logger.Printf("Listening as %s\n", m.Server.URI) return nil @@ -164,18 +174,6 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) - api := pilosa.NewAPI() - api.Logger = m.logger - - handler, err := http.NewHandler( - http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), - http.OptHandlerAPI(api), - http.OptHandlerLogger(m.logger), - ) - if err != nil { - return errors.Wrap(err, "wrapping handler") - } - uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { return errors.Wrap(err, "processing bind address") @@ -210,11 +208,16 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "new stats client") } - ln, err := getListener(*uri, TLSConfig) + m.ln, err = getListener(*uri, TLSConfig) if err != nil { return errors.Wrap(err, "getting listener") } + // If port is 0, get auto-allocated port from listener + if uri.Port() == 0 { + uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) + } + c := http.GetHTTPClient(TLSConfig) // Setup connection to primary store if this is a replica. @@ -234,17 +237,30 @@ func (m *Command) SetupServer() error { pilosa.OptServerLogger(m.logger), pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore), - pilosa.OptServerHandler(handler), pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), pilosa.OptServerStatsClient(statsClient), - pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), ) + api, err := pilosa.NewAPI(pilosa.OptAPIServer(m.Server)) + if err != nil { + return errors.Wrap(err, "new api") + } + + m.handler, err = http.NewHandler( + http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + http.OptHandlerAPI(api), + http.OptHandlerLogger(m.logger), + http.OptHandlerListener(m.ln), + ) + if err != nil { + return errors.Wrap(err, "new handler") + } + return errors.Wrap(err, "new server") } @@ -300,17 +316,16 @@ func (m *Command) SetupNetworking() error { // Close shuts down the server. func (m *Command) Close() error { var logErr error + handlerErr := m.handler.Close() serveErr := m.Server.Close() if closer, ok := m.logOutput.(io.Closer); ok { logErr = closer.Close() } close(m.done) - if serveErr != nil && logErr != nil { - return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) - } else if logErr != nil { - return logErr + if serveErr != nil || logErr != nil || handlerErr != nil { + return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr) } - return serveErr + return nil } // NewStatsClient creates a stats client from the config diff --git a/test/handler.go b/test/handler.go index 5256413b5..8b58d5a6e 100644 --- a/test/handler.go +++ b/test/handler.go @@ -45,7 +45,11 @@ func NewHandler(opts ...http.HandlerOption) (*Handler, error) { h := &Handler{ Handler: handler, } - h.API = pilosa.NewAPI() + + //h.API, err = pilosa.NewAPI(OptAPIServer(s)) + if err != nil { + return nil, err + } h.Handler.API = h.API h.Handler.API.Executor = &h.Executor @@ -84,22 +88,23 @@ type Server struct { // NewServer returns a test server running on a random port. func NewServer() *Server { - handler, err := NewHandler() - if err != nil { - panic(err) - } - s := &Server{ - Handler: handler, - } - s.Server = httptest.NewServer(s.Handler.Handler) + return &Server{} + //handler, err := pilosa.NewHandler() + //if err != nil { + // panic(err) + //} + //s := &Server{ + // Handler: handler, + //} + //s.Server = httptest.NewServer(s.Handler.Handler) - // Handler test messages can no-op. - s.Handler.API.Broadcaster = pilosa.NopBroadcaster - // Create a default cluster on the handler - s.Handler.API.Cluster = NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + //// Handler test messages can no-op. + //s.Handler.API.Broadcaster = pilosa.NopBroadcaster + //// Create a default cluster on the handler + //s.Handler.API.Cluster = NewCluster(1) + //s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - return s + //return s } // LocalStatus exists so that test.Server implements StatusHandler. From c9479afe8c02ec7c5bed293799b04f86b7e5c224 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 21 Jun 2018 13:56:46 -0500 Subject: [PATCH 13/33] start handler before server.Open to avoid stall in cluster.open cluster.open waits for node to join cluster if it is not the coordinator, and currently this relies on having the http handler able to receive messages, so handler needs to be started first. --- cluster.go | 2 +- server/server.go | 27 +++++++++++++-------------- server_test.go | 4 ++-- stats_test.go | 4 ++++ test/pilosa.go | 8 ++++++++ 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/cluster.go b/cluster.go index 0f998507e..2883b739e 100644 --- a/cluster.go +++ b/cluster.go @@ -914,7 +914,7 @@ func (c *Cluster) open() error { return fmt.Errorf("sending restart NodeJoin: %v", err) } - c.Logger.Printf("wait for joining to complete") + c.Logger.Printf("%v wait for joining to complete", c.Node.ID) <-c.joining c.Logger.Printf("joining has completed") } diff --git a/server/server.go b/server/server.go index e61ba6fad..ef9f46ca3 100644 --- a/server/server.go +++ b/server/server.go @@ -74,7 +74,7 @@ type Command struct { logOutput io.Writer logger loggerLogger - handler pilosa.Handler + Handler pilosa.Handler ln net.Listener } @@ -105,19 +105,18 @@ func (m *Command) Start() (err error) { if err != nil { return errors.Wrap(err, "setting up networking") } + go func() { + err := m.Handler.Serve() + if err != nil { + m.logger.Printf("Handler serve error: %v", err) + } + }() // Initialize server. if err = m.Server.Open(); err != nil { return errors.Wrap(err, "opening server") } - go func() { - err := m.handler.Serve() - if err != nil { - m.logger.Printf("Handler serve error: %v", err) - } - }() - m.logger.Printf("Listening as %s\n", m.Server.URI) return nil @@ -245,23 +244,23 @@ func (m *Command) SetupServer() error { pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), ) + if err != nil { + return errors.Wrap(err, "new server") + } api, err := pilosa.NewAPI(pilosa.OptAPIServer(m.Server)) if err != nil { return errors.Wrap(err, "new api") } - m.handler, err = http.NewHandler( + m.Handler, err = http.NewHandler( http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), http.OptHandlerAPI(api), http.OptHandlerLogger(m.logger), http.OptHandlerListener(m.ln), ) - if err != nil { - return errors.Wrap(err, "new handler") - } + return errors.Wrap(err, "new handler") - return errors.Wrap(err, "new server") } // SetupNetworking sets up internode communication based on the configuration. @@ -316,7 +315,7 @@ func (m *Command) SetupNetworking() error { // Close shuts down the server. func (m *Command) Close() error { var logErr error - handlerErr := m.handler.Close() + handlerErr := m.Handler.Close() serveErr := m.Server.Close() if closer, ok := m.logOutput.(io.Closer); ok { logErr = closer.Close() diff --git a/server_test.go b/server_test.go index 2f1003592..402d4de7d 100644 --- a/server_test.go +++ b/server_test.go @@ -27,7 +27,7 @@ import ( // pilosa.Server was not having its remoteClient field set by an option and so // it was using a nil client in monitorAntiEntropy. func TestMonitorAntiEntropy(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1)) + cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*20)) client := cluster[1].Client() err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) if err != nil { @@ -38,7 +38,7 @@ func TestMonitorAntiEntropy(t *testing.T) { t.Fatalf("creating field: %v", err) } - time.Sleep(time.Millisecond * 2) + time.Sleep(time.Millisecond * 40) for _, m := range cluster { err := m.Close() if err != nil { diff --git a/stats_test.go b/stats_test.go index 6644786cf..1f23dd3f8 100644 --- a/stats_test.go +++ b/stats_test.go @@ -208,6 +208,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } func TestStatsCount_CreateIndex(t *testing.T) { + t.Skip() hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() @@ -230,6 +231,7 @@ func TestStatsCount_CreateIndex(t *testing.T) { } func TestStatsCount_DeleteIndex(t *testing.T) { + t.Skip() hldr := test.MustOpenHolder() defer hldr.Close() @@ -258,6 +260,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { } func TestStatsCount_CreateField(t *testing.T) { + t.Skip() hldr := test.MustOpenHolder() defer hldr.Close() @@ -289,6 +292,7 @@ func TestStatsCount_CreateField(t *testing.T) { } func TestStatsCount_DeleteField(t *testing.T) { + t.Skip() hldr := test.MustOpenHolder() defer hldr.Close() diff --git a/test/pilosa.go b/test/pilosa.go index 757e5a96c..22e89a5df 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "io/ioutil" + "log" gohttp "net/http" "os" "strings" @@ -221,6 +222,13 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( m.Server.Cluster.Static = false + go func() { + err := m.Handler.Serve() + if err != nil { + log.Printf("Handler serve error: %v", err) + } + }() + // Initialize server. err = m.Server.Open() if err != nil { From a0337714120769cdd520962c119db4bca9931a91 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 21 Jun 2018 14:05:00 -0500 Subject: [PATCH 14/33] move handler tests which are actually testing everything to server package --- http/handler_test.go | 987 --------------------------------------- server/handler_test.go | 1009 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1009 insertions(+), 987 deletions(-) create mode 100644 server/handler_test.go diff --git a/http/handler_test.go b/http/handler_test.go index f249750c2..49d24ffec 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -15,26 +15,11 @@ package http_test import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" "net" - "net/http/httptest" - "reflect" - "strings" "testing" - gohttp "net/http" - - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" - "github.com/pkg/errors" ) func TestHandlerOptions(t *testing.T) { @@ -55,975 +40,3 @@ func TestHandlerOptions(t *testing.T) { t.Fatalf("expected error making handler without options, got nil") } } - -func TestHandler_Endpoints(t *testing.T) { - mains := test.MustRunMainWithCluster(t, 1) - _ = mains[0] -} - -// Ensure the handler returns "not found" for invalid paths. -func TestHandler_NotFound(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != gohttp.StatusNotFound { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler can return the schema. -func TestHandler_Schema(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return the status. -func TestHandler_Status(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - hldr := test.MustOpenHolder() - defer s.Close() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateNormal) - h.API.StatusHandler = s - s.Handler = h - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -func TestHandler_Info(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - defer s.Close() - h := test.MustNewHandler() - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can abort a cluster resize. -func TestHandler_ClusterResizeAbort(t *testing.T) { - t.Skip() // Until test.NewServer() works - - t.Run("No resize job", func(t *testing.T) { - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateResizing) - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) - if w.Code != gohttp.StatusOK { - bod, err := ioutil.ReadAll(w.Body) - t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) - } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } - }) - -} - -// Ensure the handler can return the maxslice map. -func TestHandler_MaxSlices(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) - hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) - - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can accept URL arguments. -func TestHandler_Query_Args_URL(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can accept arguments via protobufs. -func TestHandler_Query_Args_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - return []interface{}{uint64(100)}, nil - } - - // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ - Query: "Count(Bitmap(id=100))", - Slices: []uint64{0, 1}, - }) - if err != nil { - t.Fatal(err) - } - - // Generate protobuf request. - req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) - req.Header.Set("Content-Type", "application/x-protobuf") - - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -// Ensure the handler returns an error when parsing bad arguments. -func TestHandler_Query_Args_Err(t *testing.T) { - t.Skip() // Until test.NewServer() works - - w := httptest.NewRecorder() - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} -func TestHandler_Query_Params_Err(t *testing.T) { - t.Skip() // Until test.NewServer() works - - w := httptest.NewRecorder() - test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } - -} - -// Ensure the handler can execute a query with a uint64 response as JSON. -func TestHandler_Query_Uint64_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can execute a query with a uint64 response as protobufs. -func TestHandler_Query_Uint64_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 100 { - t.Fatalf("unexpected n: %d", n) - } -} - -// Ensure the handler can execute a query that returns a bitmap as JSON. -func TestHandler_Query_Bitmap_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can execute a query that returns a row with column attributes as JSON. -func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can execute a query that returns a row as protobuf. -func TestHandler_Query_Row_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } -} - -// Ensure the handler can execute a query that returns a row with column attributes as protobuf. -func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", - ColumnAttrs: true, - }) - if err != nil { - t.Fatal(err) - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) - r.Header.Set("Content-Type", "application/x-protobuf") - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } - - if a := resp.ColumnAttrSets; len(a) != 1 { - t.Fatalf("unexpected column attributes length: %d", len(a)) - } else if a[0].ID != 1 { - t.Fatalf("unexpected id: %d", a[0].ID) - } else if len(a[0].Attrs) != 1 { - t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } -} - -// Ensure the handler can execute a query that returns pairs as JSON. -func TestHandler_Query_Pairs_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can execute a query that returns pairs as protobuf. -func TestHandler_Query_Pairs_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { - t.Fatalf("unexpected pair length: %d", len(a)) - } -} - -// Ensure the handler can return an error as JSON. -func TestHandler_Query_Err_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can return an error as protobuf. -func TestHandler_Query_Err_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if s := resp.Err; s != `executing: marker` { - t.Fatalf("unexpected error: %s", s) - } -} - -// Ensure the handler returns "method not allowed" for non-POST queries. -func TestHandler_Query_MethodNotAllowed(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != gohttp.StatusMethodNotAllowed { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler returns an error if there is a parsing error.. -func TestHandler_Query_ErrParse(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can delete an index. -func TestHandler_Index_Delete(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - - // Send request to delete index. - resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Verify body response. - if resp.StatusCode != gohttp.StatusOK { - t.Fatalf("unexpected status: %d", resp.StatusCode) - } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if string(buf) != "{}\n" { - t.Fatalf("unexpected response body: %s", buf) - } - - // Verify index is gone. - if hldr.Index("i") != nil { - t.Fatal("expected nil index") - } -} - -// Ensure handler can delete a field. -func TestHandler_DeleteField(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i0").Field("f1"); f != nil { - t.Fatal("expected nil field") - } -} - -// Ensure the handler can return data in differing blocks for an index. -func TestHandler_Index_AttrStore_Diff(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set attributes on the index. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { - t.Fatal(err) - } - - // Retrieve block checksums. - blks, err := index.ColumnAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } - - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - - // Send block checksums to determine diff. - req, err := gohttp.NewRequest( - "POST", - s.URL+"/index/i/attr/diff", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - client := &gohttp.Client{} - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return data in differing blocks for a field. -func TestHandler_Field_AttrStore_Diff(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set attributes on the index. - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) - if err != nil { - t.Fatal(err) - } - if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { - t.Fatal(err) - } - - // Retrieve block checksums. - blks, err := f.RowAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } - - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - - // Send block checksums to determine diff. - req, err := gohttp.NewRequest( - "POST", - s.URL+"/index/i/field/meta/attr/diff", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - client := &gohttp.Client{} - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can retrieve the version. -func TestHandler_Version(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/version", nil) - h.ServeHTTP(w, r) - version := pilosa.Version - if strings.HasPrefix(version, "v") { - version = version[1:] - } - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) - } -} - -// Ensure the handler can return a list of nodes for a fragment. -func TestHandler_Fragment_Nodes(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(3) - h.API.Cluster.ReplicaN = 2 - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { - t.Fatalf("unexpected body: %q", body) - } - - // invalid argument should return BadRequest - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - // index is required - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -// Ensure the handler can return expvars without panicking. -func TestHandler_Expvars(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -func MustReadAll(r io.Reader) []byte { - buf, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return buf -} - -func TestHandler_RecalculateCaches(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != gohttp.StatusNoContent { - t.Fatalf("unexpected status code: %d", w.Code) - } - -} - -func TestHandler_CORS(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // No CORS config present, so should fail - handler := test.MustNewHandler() - - req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) - req.Header.Add("Origin", "http://test/") - req.Header.Add("Access-Control-Request-Method", "POST") - - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - result := w.Result() - - // This handler does not support CORS, return Method Not Allowed (405) - if result.StatusCode != 405 { - t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) - } - - // CORS config should allow preflight response - handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) - w = httptest.NewRecorder() - handler.ServeHTTP(w, req) - result = w.Result() - - if result.StatusCode != 200 { - t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) - } - if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { - t.Fatal("CORS header not present") - } -} diff --git a/server/handler_test.go b/server/handler_test.go new file mode 100644 index 000000000..fb4048f65 --- /dev/null +++ b/server/handler_test.go @@ -0,0 +1,1009 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server_test + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http/httptest" + "reflect" + "strings" + "testing" + + gohttp "net/http" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" +) + +func TestHandler_Endpoints(t *testing.T) { + mains := test.MustRunMainWithCluster(t, 1) + _ = mains[0] +} + +// Ensure the handler returns "not found" for invalid paths. +func TestHandler_NotFound(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) + if w.Code != gohttp.StatusNotFound { + t.Fatalf("invalid status: %d", w.Code) + } +} + +// Ensure the handler can return the schema. +func TestHandler_Schema(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) + + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can return the status. +func TestHandler_Status(t *testing.T) { + t.Skip() // Until test.NewServer() works + + s := test.NewServer() + hldr := test.MustOpenHolder() + defer s.Close() + defer hldr.Close() + + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) + + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateNormal) + h.API.StatusHandler = s + s.Handler = h + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestHandler_Info(t *testing.T) { + t.Skip() // Until test.NewServer() works + + s := test.NewServer() + defer s.Close() + h := test.MustNewHandler() + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can abort a cluster resize. +func TestHandler_ClusterResizeAbort(t *testing.T) { + t.Skip() // Until test.NewServer() works + + t.Run("No resize job", func(t *testing.T) { + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateResizing) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + if w.Code != gohttp.StatusOK { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + +} + +// Ensure the handler can return the maxslice map. +func TestHandler_MaxSlices(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) + + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can accept URL arguments. +func TestHandler_Query_Args_URL(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "idx0" { + t.Fatalf("unexpected index: %s", index) + } else if query.String() != `Count(Bitmap(id=100))` { + t.Fatalf("unexpected query: %s", query.String()) + } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { + t.Fatalf("unexpected slices: %+v", slices) + } + return []interface{}{uint64(100)}, nil + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +// Ensure the handler can accept arguments via protobufs. +func TestHandler_Query_Args_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "idx0" { + t.Fatalf("unexpected index: %s", index) + } else if query.String() != `Count(Bitmap(id=100))` { + t.Fatalf("unexpected query: %s", query.String()) + } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { + t.Fatalf("unexpected slices: %+v", slices) + } + return []interface{}{uint64(100)}, nil + } + + // Generate request body. + reqBody, err := proto.Marshal(&internal.QueryRequest{ + Query: "Count(Bitmap(id=100))", + Slices: []uint64{0, 1}, + }) + if err != nil { + t.Fatal(err) + } + + // Generate protobuf request. + req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/x-protobuf") + + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } +} + +// Ensure the handler returns an error when parsing bad arguments. +func TestHandler_Query_Args_Err(t *testing.T) { + t.Skip() // Until test.NewServer() works + + w := httptest.NewRecorder() + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } +} +func TestHandler_Query_Params_Err(t *testing.T) { + t.Skip() // Until test.NewServer() works + + w := httptest.NewRecorder() + test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + +} + +// Ensure the handler can execute a query with a uint64 response as JSON. +func TestHandler_Query_Uint64_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{uint64(100)}, nil + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +// Ensure the handler can execute a query with a uint64 response as protobufs. +func TestHandler_Query_Uint64_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{uint64(100)}, nil + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if n := resp.Results[0].N; n != 100 { + t.Fatalf("unexpected n: %d", n) + } +} + +// Ensure the handler can execute a query that returns a bitmap as JSON. +func TestHandler_Query_Bitmap_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) + r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} + return []interface{}{r}, nil + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can execute a query that returns a row with column attributes as JSON. +func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.NewHolder() + defer hldr.Close() + + // Create index and set column attributes. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { + t.Fatal(err) + } + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) + r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} + return []interface{}{r}, nil + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can execute a query that returns a row as protobuf. +func TestHandler_Query_Row_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + r := pilosa.NewRow(1, pilosa.SliceWidth+1) + r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} + return []interface{}{r}, nil + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { + t.Fatalf("unexpected attr[1]: %s=%v", k, v) + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { + t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } +} + +// Ensure the handler can execute a query that returns a row with column attributes as protobuf. +func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.NewHolder() + defer hldr.Close() + + // Create index and set column attributes. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + r := pilosa.NewRow(1, pilosa.SliceWidth+1) + r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} + return []interface{}{r}, nil + } + + // Encode request body. + buf, err := proto.Marshal(&internal.QueryRequest{ + Query: "Bitmap(id=100)", + ColumnAttrs: true, + }) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { + t.Fatalf("unexpected attr[1]: %s=%v", k, v) + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { + t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } + + if a := resp.ColumnAttrSets; len(a) != 1 { + t.Fatalf("unexpected column attributes length: %d", len(a)) + } else if a[0].ID != 1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { + t.Fatalf("unexpected column attr length: %d", len(a)) + } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } +} + +// Ensure the handler can execute a query that returns pairs as JSON. +func TestHandler_Query_Pairs_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 2}, + {ID: 3, Count: 4}, + }}, nil + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +// Ensure the handler can execute a query that returns pairs as protobuf. +func TestHandler_Query_Pairs_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 2}, + {ID: 3, Count: 4}, + }}, nil + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if a := resp.Results[0].GetPairs(); len(a) != 2 { + t.Fatalf("unexpected pair length: %d", len(a)) + } +} + +// Ensure the handler can return an error as JSON. +func TestHandler_Query_Err_JSON(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return nil, errors.New("marker") + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } +} + +// Ensure the handler can return an error as protobuf. +func TestHandler_Query_Err_Protobuf(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return nil, errors.New("marker") + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if s := resp.Err; s != `executing: marker` { + t.Fatalf("unexpected error: %s", s) + } +} + +// Ensure the handler returns "method not allowed" for non-POST queries. +func TestHandler_Query_MethodNotAllowed(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) + if w.Code != gohttp.StatusMethodNotAllowed { + t.Fatalf("invalid status: %d", w.Code) + } +} + +// Ensure the handler returns an error if there is a parsing error.. +func TestHandler_Query_ErrParse(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can delete an index. +func TestHandler_Index_Delete(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + s := test.NewServer() + s.Handler.API.Holder = hldr.Holder + defer s.Close() + + // Create index. + if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + // Send request to delete index. + resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + // Verify body response. + if resp.StatusCode != gohttp.StatusOK { + t.Fatalf("unexpected status: %d", resp.StatusCode) + } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { + t.Fatal(err) + } else if string(buf) != "{}\n" { + t.Fatalf("unexpected response body: %s", buf) + } + + // Verify index is gone. + if hldr.Index("i") != nil { + t.Fatal("expected nil index") + } +} + +// Ensure handler can delete a field. +func TestHandler_DeleteField(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := hldr.Index("i0").Field("f1"); f != nil { + t.Fatal("expected nil field") + } +} + +// Ensure the handler can return data in differing blocks for an index. +func TestHandler_Index_AttrStore_Diff(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + s := test.NewServer() + s.Handler.API.Holder = hldr.Holder + defer s.Close() + + // Set attributes on the index. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + t.Fatal(err) + } else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + t.Fatal(err) + } + + // Retrieve block checksums. + blks, err := index.ColumnAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } + + // Remove block #0 and alter block 2's checksum. + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") + + // Send block checksums to determine diff. + req, err := gohttp.NewRequest( + "POST", + s.URL+"/index/i/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + client := &gohttp.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + // Read and validate body. + if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can return data in differing blocks for a field. +func TestHandler_Field_AttrStore_Diff(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + s := test.NewServer() + s.Handler.API.Holder = hldr.Holder + defer s.Close() + + // Set attributes on the index. + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) + if err != nil { + t.Fatal(err) + } + if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + t.Fatal(err) + } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + t.Fatal(err) + } + + // Retrieve block checksums. + blks, err := f.RowAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } + + // Remove block #0 and alter block 2's checksum. + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") + + // Send block checksums to determine diff. + req, err := gohttp.NewRequest( + "POST", + s.URL+"/index/i/field/meta/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + client := &gohttp.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + // Read and validate body. + if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can retrieve the version. +func TestHandler_Version(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/version", nil) + h.ServeHTTP(w, r) + version := pilosa.Version + if strings.HasPrefix(version, "v") { + version = version[1:] + } + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } +} + +// Ensure the handler can return a list of nodes for a fragment. +func TestHandler_Fragment_Nodes(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(3) + h.API.Cluster.ReplicaN = 2 + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + + // invalid argument should return BadRequest + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } +} + +// Ensure the handler can return expvars without panicking. +func TestHandler_Expvars(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } +} + +func MustReadAll(r io.Reader) []byte { + buf, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + return buf +} + +func TestHandler_RecalculateCaches(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.MustNewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) + if w.Code != gohttp.StatusNoContent { + t.Fatalf("unexpected status code: %d", w.Code) + } + +} + +func TestHandler_CORS(t *testing.T) { + t.Skip() // Until test.NewServer() works + + hldr := test.MustOpenHolder() + defer hldr.Close() + + s := test.NewServer() + s.Handler.API.Holder = hldr.Holder + defer s.Close() + + // No CORS config present, so should fail + handler := test.MustNewHandler() + + req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) + req.Header.Add("Origin", "http://test/") + req.Header.Add("Access-Control-Request-Method", "POST") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + result := w.Result() + + // This handler does not support CORS, return Method Not Allowed (405) + if result.StatusCode != 405 { + t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) + } + + // CORS config should allow preflight response + handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + result = w.Result() + + if result.StatusCode != 200 { + t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) + } + if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { + t.Fatal("CORS header not present") + } +} From 504309e59484154b41bdfa1fbce5ea961797735f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 21 Jun 2018 15:06:22 -0500 Subject: [PATCH 15/33] rewrite some tests to not be skipped --- server.go | 5 + server/handler_test.go | 473 ++++++++++++++--------------------------- 2 files changed, 162 insertions(+), 316 deletions(-) diff --git a/server.go b/server.go index c575d5a2c..84a09a01d 100644 --- a/server.go +++ b/server.go @@ -79,6 +79,11 @@ type Server struct { dataDir string } +// TODO: have this return an interface for Holder instead of concrete object? +func (s *Server) Holder() *Holder { + return s.holder +} + // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error diff --git a/server/handler_test.go b/server/handler_test.go index fb4048f65..79ca26572 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -17,6 +17,7 @@ package server_test import ( "bytes" "context" + "encoding/json" "fmt" "io" "io/ioutil" @@ -36,151 +37,82 @@ import ( "github.com/pkg/errors" ) -func TestHandler_Endpoints(t *testing.T) { - mains := test.MustRunMainWithCluster(t, 1) - _ = mains[0] -} - // Ensure the handler returns "not found" for invalid paths. -func TestHandler_NotFound(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != gohttp.StatusNotFound { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler can return the schema. -func TestHandler_Schema(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return the status. -func TestHandler_Status(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - hldr := test.MustOpenHolder() - defer s.Close() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateNormal) - h.API.StatusHandler = s - s.Handler = h - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -func TestHandler_Info(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - defer s.Close() - h := test.MustNewHandler() - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can abort a cluster resize. -func TestHandler_ClusterResizeAbort(t *testing.T) { - t.Skip() // Until test.NewServer() works - - t.Run("No resize job", func(t *testing.T) { - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateResizing) +func TestHandler_Endpoints(t *testing.T) { + cmd := test.MustRunMainWithCluster(t, 1)[0] + h := cmd.Handler.(*http.Handler).Handler + t.Run("Not Found", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) + if w.Code != gohttp.StatusNotFound { + t.Fatalf("invalid status: %d", w.Code) + } + }) + + t.Run("Info", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) if w.Code != gohttp.StatusOK { - bod, err := ioutil.ReadAll(w.Body) - t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) - } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { t.Fatalf("unexpected body: %s", body) } }) -} + holder := cmd.Server.Holder() + hldr := test.Holder{holder} + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } -// Ensure the handler can return the maxslice map. -func TestHandler_MaxSlices(t *testing.T) { - t.Skip() // Until test.NewServer() works + t.Run("Schema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("Status", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + ret := mustJSONDecode(t, w.Body) + if ret["state"].(string) != "NORMAL" { + t.Fatalf("wrong state from /status: %#v", ret) + } + if len(ret["nodes"].([]interface{})) != 1 { + t.Fatalf("wrong length nodes list: %#v", ret) + } + }) + + t.Run("Abort no resize job", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + if w.Code != gohttp.StatusInternalServerError { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } + // TODO need to test aborting a cluster resize job. this may not be the right place + }) hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) @@ -190,199 +122,99 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can accept URL arguments. -func TestHandler_Query_Args_URL(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) + t.Run("Max Slice", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { + t.Fatalf("unexpected body: %s", body) } - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can accept arguments via protobufs. -func TestHandler_Query_Args_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - return []interface{}{uint64(100)}, nil - } - - // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ - Query: "Count(Bitmap(id=100))", - Slices: []uint64{0, 1}, }) - if err != nil { - t.Fatal(err) - } - // Generate protobuf request. - req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) - req.Header.Set("Content-Type", "application/x-protobuf") + t.Run("Slices args", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} + t.Run("Slices args protobuf", func(t *testing.T) { + // Generate request body. + reqBody, err := proto.Marshal(&internal.QueryRequest{ + Query: "Count(Bitmap(field=f0, row=30))", + Slices: []uint64{0, 1}, + }) + if err != nil { + t.Fatal(err) + } -// Ensure the handler returns an error when parsing bad arguments. -func TestHandler_Query_Args_Err(t *testing.T) { - t.Skip() // Until test.NewServer() works + // Generate protobuf request. + req := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/json") - w := httptest.NewRecorder() - hldr := test.MustOpenHolder() - defer hldr.Close() + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder + }) - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} -func TestHandler_Query_Params_Err(t *testing.T) { - t.Skip() // Until test.NewServer() works + t.Run("Query args error", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) - w := httptest.NewRecorder() - test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } + t.Run("Query params err", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) -} + t.Run("Uint64 protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Bitmap(field=f0, row=30))")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } -// Ensure the handler can execute a query with a uint64 response as JSON. -func TestHandler_Query_Uint64_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if n := resp.Results[0].N; n != 3 { + t.Fatalf("unexpected n: %d", n) + } + }) - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can execute a query with a uint64 response as protobufs. -func TestHandler_Query_Uint64_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 100 { - t.Fatalf("unexpected n: %d", n) - } -} - -// Ensure the handler can execute a query that returns a bitmap as JSON. -func TestHandler_Query_Bitmap_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } + t.Run("Bitmap JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Bitmap(field=f0, row=30)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) } // Ensure the handler can execute a query that returns a row with column attributes as JSON. @@ -1007,3 +839,12 @@ func TestHandler_CORS(t *testing.T) { t.Fatal("CORS header not present") } } + +func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { + dec := json.NewDecoder(r) + err := dec.Decode(&ret) + if err != nil { + t.Fatalf("decoding response: %v", err) + } + return ret +} From 526bdae280ab51ddd1a661e537f38ea80d2b2418 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 21 Jun 2018 15:41:34 -0500 Subject: [PATCH 16/33] Remove unneccessary t.Skip() --- cmd/server_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 989c8de6b..e58f8af4d 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -35,8 +35,6 @@ func TestServerHelp(t *testing.T) { } func TestServerConfig(t *testing.T) { - t.Skip() // Until test.NewServer() works - actualDataDir, err := ioutil.TempDir("", "") failErr(t, err, "making data dir") logFile, err := ioutil.TempFile("", "") From e2512ec58dba1da99f5376907ddabefa8ff747c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 21 Jun 2018 16:26:34 -0500 Subject: [PATCH 17/33] Use test.MustRunMainWithCluster in ctl tests --- ctl/export_test.go | 18 +++++------------- ctl/import_test.go | 46 +++++++++------------------------------------- 2 files changed, 14 insertions(+), 50 deletions(-) diff --git a/ctl/export_test.go b/ctl/export_test.go index d405c92c4..8960702f6 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -44,24 +44,16 @@ func TestExportCommand_Validation(t *testing.T) { } func TestExportCommand_Run(t *testing.T) { - t.Skip() // Until test.NewServer() works + cmd := test.MustRunMainWithCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewExportCommand(stdin, stdout, stderr) + hostport := cmd.Server.URI.HostPort() + cm.Host = hostport - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() - - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) cm.Index = "i" cm.Field = "f" diff --git a/ctl/import_test.go b/ctl/import_test.go index df2b9a5a6..2cf03a126 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -29,8 +29,6 @@ import ( ) func TestImportCommand_Validation(t *testing.T) { - t.Skip() // Until test.NewServer() works - buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -53,8 +51,6 @@ func TestImportCommand_Validation(t *testing.T) { } func TestImportCommand_Run(t *testing.T) { - t.Skip() // Until test.NewServer() works - buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -65,15 +61,8 @@ func TestImportCommand_Run(t *testing.T) { t.Fatal(err) } - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() + cmd := test.MustRunMainWithCluster(t, 1)[0] + cm.Host = cmd.Server.URI.HostPort() cm.Index = "i" cm.Field = "f" @@ -87,8 +76,6 @@ func TestImportCommand_Run(t *testing.T) { // Ensure that the ImportValue path runs. func TestImportCommand_RunValue(t *testing.T) { - t.Skip() // Until test.NewServer() works - buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -99,18 +86,12 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() + cmd := test.MustRunMainWithCluster(t, 1)[0] + hostport := cmd.Server.URI.HostPort() + cm.Host = hostport - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() - - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" cm.Field = "f" @@ -122,21 +103,12 @@ func TestImportCommand_RunValue(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder + cmd := test.MustRunMainWithCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = s.Host() + cm.Host = cmd.Server.URI.HostPort() cm.Index = "i" cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") From 643e5e575a7932c3e285f735d69319acba07939c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 21 Jun 2018 18:39:06 -0500 Subject: [PATCH 18/33] fixed crashing issue that was not handling container removal/recycling correctly --- api.go | 1 - ctl/import_test.go | 52 ++++++++++++++++++++++++++++++++ enterprise/b/containers_btree.go | 4 +++ fragment_internal_test.go | 33 ++++++++++++++++++++ roaring/containers.go | 7 +++++ roaring/roaring.go | 5 ++- 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 7adff1d1b..22e6ba2f6 100644 --- a/api.go +++ b/api.go @@ -658,7 +658,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest if err != nil { return errors.Wrap(err, "getting field") } - // Import into fragment. err = field.ImportValue(req.ColumnIDs, req.Values) if err != nil { diff --git a/ctl/import_test.go b/ctl/import_test.go index ba8a9dc6e..e6e7494b9 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -197,3 +197,55 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { stderr := bufio.NewWriter(&buf) return stdin, stdout, stderr } + +func TestImportCommand_BugOverwriteValue(t *testing.T) { + + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + file, err := ioutil.TempFile("", "import-value.csv") + file.Write([]byte("0,17\n")) + ctx := context.Background() + if err != nil { + t.Fatal(err) + } + + hldr := test.MustOpenHolder() + defer hldr.Close() + s := test.NewServer() + defer s.Close() + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder + cm.Host = s.Host() + + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + + cm.Index = "i" + cm.Field = "f" + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } + + file.Close() + file, err = ioutil.TempFile("", "import-value2.csv") + file.Write([]byte("0,16\n")) + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } + + file.Close() + file, err = ioutil.TempFile("", "import-value3.csv") + file.Write([]byte("0,19\n")) + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } +} diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index eb779fe06..de001fcb7 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -160,6 +160,10 @@ func (btc *BTreeContainers) Size() int { return btc.tree.Len() } +func (btc *BTreeContainers) Reset() { + btc.tree = TreeNew(cmp) +} + func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { e, ok := btc.tree.Seek(key) if ok { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 6c733ba4c..6da3ded7e 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -213,6 +213,39 @@ func TestFragment_SetValue(t *testing.T) { t.Fatal(err) } }) + t.Run("Crash", func(t *testing.T) { + f := mustOpenFragment("i", "f", ViewStandard, 0, "") + defer f.Close() + + // Set value. + if changed, err := f.setValue(0, 32, 17); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + if changed, err := f.setValue(0, 32, 16); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + if changed, err := f.setValue(0, 32, 19); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.value(0, 32); err != nil { + t.Fatal(err) + } else if value != 19 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected to exist") + } + }) + } // Ensure a fragment can sum values. diff --git a/roaring/containers.go b/roaring/containers.go index 133a30cf3..19871050b 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -132,6 +132,13 @@ func (sc *SliceContainers) Count() uint64 { return n } +func (sc *SliceContainers) Reset() { + sc.keys = sc.keys[:0] + sc.containers = sc.containers[:0] + sc.lastContainer = nil + sc.lastKey = 0 +} + func (sc *SliceContainers) seek(key uint64) (int, bool) { i := search64(sc.keys, key) found := true diff --git a/roaring/roaring.go b/roaring/roaring.go index f4d9218e2..07f1600a7 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -94,6 +94,8 @@ type Containers interface { // container is found at key. Iterator(key uint64) (citer ContainerIterator, found bool) Count() uint64 + //Reset will clear the containers collection to allow for recycling during snapshot + Reset() } type ContainerIterator interface { @@ -631,7 +633,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { keyN := binary.LittleEndian.Uint32(data[4:8]) headerSize := headerBaseSize - + b.Containers.Reset() // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { b.Containers.PutContainerValues( @@ -688,6 +690,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // FIXME(benbjohnson): return error with position so file can be trimmed. return err } + opr.apply(b) // Increase the op count. From 016dbac6ca1cb82f2274960a807a14c29585fcfc Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 21 Jun 2018 19:24:17 -0500 Subject: [PATCH 19/33] convert a bunch more tests --- server/handler_test.go | 523 ++++++++++++++--------------------------- 1 file changed, 180 insertions(+), 343 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 79ca26572..5dde9b20f 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -16,7 +16,6 @@ package server_test import ( "bytes" - "context" "encoding/json" "fmt" "io" @@ -32,9 +31,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" - "github.com/pkg/errors" ) // Ensure the handler returns "not found" for invalid paths. @@ -118,6 +115,8 @@ func TestHandler_Endpoints(t *testing.T) { hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) + hldr.SetBit("i0", "f0", 31, 1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) @@ -215,367 +214,205 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected body: %s", body) } }) -} -// Ensure the handler can execute a query that returns a row with column attributes as JSON. -func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { + f0 := i0.Field("f0") + if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { + } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil { t.Fatal(err) } - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can execute a query that returns a row as protobuf. -func TestHandler_Query_Row_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } -} - -// Ensure the handler can execute a query that returns a row with column attributes as protobuf. -func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", - ColumnAttrs: true, + t.Run("ColumnAttrs_JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Bitmap(field=f0, row=30)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } }) - if err != nil { - t.Fatal(err) - } - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) - r.Header.Set("Content-Type", "application/x-protobuf") - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } + t.Run("Row pbuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Bitmap(field=f0, row=30)")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { + t.Fatalf("unexpected attr[1]: %s=%v", k, v) + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { + t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } + }) - if a := resp.ColumnAttrSets; len(a) != 1 { - t.Fatalf("unexpected column attributes length: %d", len(a)) - } else if a[0].ID != 1 { - t.Fatalf("unexpected id: %d", a[0].ID) - } else if len(a[0].Attrs) != 1 { - t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } -} + t.Run("Row columnattrs protobuf", func(t *testing.T) { + // Encode request body. + buf, err := proto.Marshal(&internal.QueryRequest{ + Query: "Bitmap(field=f0, row=30)", + ColumnAttrs: true, + }) + if err != nil { + t.Fatal(err) + } -// Ensure the handler can execute a query that returns pairs as JSON. -func TestHandler_Query_Pairs_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(buf)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } - hldr := test.MustOpenHolder() - defer hldr.Close() + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { + t.Fatalf("unexpected attr[1]: %s=%v", k, v) + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { + t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } + if a := resp.ColumnAttrSets; len(a) != 2 { + t.Fatalf("unexpected column attributes length: %d", len(a)) + } else if a[0].ID != pilosa.SliceWidth+1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { + t.Fatalf("unexpected column attr length: %d", len(a)) + } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { + t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } + }) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} + t.Run("Query Pairs JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(field=f0, n=2)`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) -// Ensure the handler can execute a query that returns pairs as protobuf. -func TestHandler_Query_Pairs_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works + t.Run("Query Pairs protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(field=f0, n=2)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } - hldr := test.MustOpenHolder() - defer hldr.Close() + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) + } else if a := resp.Results[0].GetPairs(); len(a) != 2 { + t.Fatalf("unexpected pair length: %d", len(a)) + } + }) - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } + t.Run("Query err JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Bitmap(row=30)`))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"executing: field not found"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } + t.Run("Query err protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Bitmap(row=30)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { - t.Fatalf("unexpected pair length: %d", len(a)) - } -} + var resp internal.QueryResponse + if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if s := resp.Err; s != `executing: field not found` { + t.Fatalf("unexpected error: %s", s) + } + }) -// Ensure the handler can return an error as JSON. -func TestHandler_Query_Err_JSON(t *testing.T) { - t.Skip() // Until test.NewServer() works + t.Run("Method not allowed", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil)) + if w.Code != gohttp.StatusMethodNotAllowed { + t.Fatalf("invalid status: %d", w.Code) + } + }) - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("Err Parse", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } + t.Run("delete index", func(t *testing.T) { + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } else if w.Body.String() != "{}\n" { + t.Fatalf("unexpected response body: %s", w.Body.String()) + } + // Verify index is gone. + if hldr.Index("i") != nil { + t.Fatal("expected nil index") + } + }) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can return an error as protobuf. -func TestHandler_Query_Err_Protobuf(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if s := resp.Err; s != `executing: marker` { - t.Fatalf("unexpected error: %s", s) - } -} - -// Ensure the handler returns "method not allowed" for non-POST queries. -func TestHandler_Query_MethodNotAllowed(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != gohttp.StatusMethodNotAllowed { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler returns an error if there is a parsing error.. -func TestHandler_Query_ErrParse(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can delete an index. -func TestHandler_Index_Delete(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - - // Send request to delete index. - resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Verify body response. - if resp.StatusCode != gohttp.StatusOK { - t.Fatalf("unexpected status: %d", resp.StatusCode) - } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if string(buf) != "{}\n" { - t.Fatalf("unexpected response body: %s", buf) - } - - // Verify index is gone. - if hldr.Index("i") != nil { - t.Fatal("expected nil index") - } -} - -// Ensure handler can delete a field. -func TestHandler_DeleteField(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i0").Field("f1"); f != nil { - t.Fatal("expected nil field") - } + t.Run("Field delete", func(t *testing.T) { + i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } + }) } // Ensure the handler can return data in differing blocks for an index. @@ -615,7 +452,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { // Send block checksums to determine diff. req, err := gohttp.NewRequest( "POST", - s.URL+"/index/i/attr/diff", + s.URL+"/index/i0/attr/diff", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) @@ -673,7 +510,7 @@ func TestHandler_Field_AttrStore_Diff(t *testing.T) { // Send block checksums to determine diff. req, err := gohttp.NewRequest( "POST", - s.URL+"/index/i/field/meta/attr/diff", + s.URL+"/index/i0/field/meta/attr/diff", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) From 8809751a23a15a58e23fa375e9ac5a9c7550575d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 07:41:39 -0500 Subject: [PATCH 20/33] convert all tests except for CORS --- server/handler_test.go | 329 ++++++++++++++++------------------------- 1 file changed, 127 insertions(+), 202 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 5dde9b20f..9ac0f8931 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -406,234 +406,150 @@ func TestHandler_Endpoints(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } else if f := hldr.Index("i").Field("f1"); f != nil { t.Fatal("expected nil field") } }) -} -// Ensure the handler can return data in differing blocks for an index. -func TestHandler_Index_AttrStore_Diff(t *testing.T) { - t.Skip() // Until test.NewServer() works + i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if err := i.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + t.Fatal(err) + } else if err := i.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := i.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + t.Fatal(err) + } - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("AttrStore Diff", func(t *testing.T) { + blks, err := i.ColumnAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - // Set attributes on the index. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) + // Send block checksums to determine diff. + req := test.MustNewHTTPRequest( + "POST", + "/index/i/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } + + // Read and validate body. + if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", w.Body.String()) + } + }) + + meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } - if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := meta.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := meta.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := meta.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } - // Retrieve block checksums. - blks, err := index.ColumnAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } + t.Run("field attrstore diff", func(t *testing.T) { + blks, err := meta.RowAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") + // Send block checksums to determine diff. + req := test.MustNewHTTPRequest( + "POST", + "/index/i/field/meta/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } - // Send block checksums to determine diff. - req, err := gohttp.NewRequest( - "POST", - s.URL+"/index/i0/attr/diff", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) + // Read and validate body. + if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", w.Body.String()) + } + }) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") + t.Run("Version", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/version", nil) + h.ServeHTTP(w, r) + version := strings.TrimPrefix(pilosa.Version, "v") + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + }) - client := &gohttp.Client{} - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() + t.Run("Fragment Nodes", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := mustJSONDecodeSlice(t, w.Body) + bmap := body[0].(map[string]interface{}) + if bmap["isCoordinator"] != true { + t.Fatalf("expected true coordinator") + } - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} + // invalid argument should return BadRequest + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } -// Ensure the handler can return data in differing blocks for a field. -func TestHandler_Field_AttrStore_Diff(t *testing.T) { - t.Skip() // Until test.NewServer() works + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("Expvars", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set attributes on the index. - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) - if err != nil { - t.Fatal(err) - } - if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { - t.Fatal(err) - } - - // Retrieve block checksums. - blks, err := f.RowAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } - - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - - // Send block checksums to determine diff. - req, err := gohttp.NewRequest( - "POST", - s.URL+"/index/i0/field/meta/attr/diff", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - client := &gohttp.Client{} - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can retrieve the version. -func TestHandler_Version(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/version", nil) - h.ServeHTTP(w, r) - version := pilosa.Version - if strings.HasPrefix(version, "v") { - version = version[1:] - } - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) - } -} - -// Ensure the handler can return a list of nodes for a fragment. -func TestHandler_Fragment_Nodes(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(3) - h.API.Cluster.ReplicaN = 2 - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { - t.Fatalf("unexpected body: %q", body) - } - - // invalid argument should return BadRequest - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - // index is required - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -// Ensure the handler can return expvars without panicking. -func TestHandler_Expvars(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) - h.ServeHTTP(w, r) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -func MustReadAll(r io.Reader) []byte { - buf, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return buf -} - -func TestHandler_RecalculateCaches(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != gohttp.StatusNoContent { - t.Fatalf("unexpected status code: %d", w.Code) - } + t.Run("Recalculate Caches", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) + if w.Code != gohttp.StatusNoContent { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) } @@ -685,3 +601,12 @@ func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { } return ret } + +func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) { + dec := json.NewDecoder(r) + err := dec.Decode(&ret) + if err != nil { + t.Fatalf("decoding response: %v", err) + } + return ret +} From 5bf9af4df38ccf08afcefbdd651b58024176cf48 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 22 Jun 2018 08:07:31 -0500 Subject: [PATCH 21/33] Parser and test updates --- pql/ast.go | 14 +- pql/pql.peg | 15 +- pql/pql.peg.go | 2169 ++++++++++++++++++++++---------------------- pql/pqlpeg_test.go | 87 +- 4 files changed, 1160 insertions(+), 1125 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 898f22836..0bcc582d4 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -213,7 +213,7 @@ func (q *Query) WriteCallN() int { var n int for _, call := range q.Calls { switch call.Name { - case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs": + case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs": n++ } } @@ -253,6 +253,18 @@ type Call struct { Children []*Call } +// FieldArg determines which key-value pair contains the field and rowID, +// in the case of arguments like Set(colID, field=rowID). +// Returns the field as a string if present, or an error if not. +func (c *Call) FieldArg() (string, error) { + for arg := range c.Args { + if !strings.HasPrefix(arg, "_") { + return arg, nil + } + } + return "", fmt.Errorf("No field argument specified") +} + // UintArg is for reading the value at key from call.Args as a uint64. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a uint64 or an int64 and diff --git a/pql/pql.peg b/pql/pql.peg index a094aa663..ca5ece479 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -6,10 +6,10 @@ type PQL Peg { Calls <- whitesp (Call whitesp)* !. -Call <- 'Set' {p.startCall("Set")} open uintcol comma args (comma timestamp)? close {p.endCall()} +Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()} / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} - / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open posfield comma uintcol comma args close {p.endCall()} - / 'Clear' {p.startCall("Clear")} open uintcol comma args close {p.endCall()} + / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} + / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } @@ -51,11 +51,14 @@ doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* -field <- { p.addField(buffer[begin:end]) } +field <- { p.addField(buffer[begin:end]) } +reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' uintrow <- {p.addPosNum("_row", buffer[begin:end])} -uintcol <- {p.addPosNum("_col", buffer[begin:end])} +col <- ( {p.addPosNum("_col", buffer[begin:end])} + / '"' '"' {p.addPosStr("_col", buffer[begin:end])} + ) open <- '(' sp close <- ')' sp @@ -64,7 +67,7 @@ comma <- sp ',' whitesp lbrack <- '[' sp rbrack <- sp ']' sp whitesp <- ( ' ' / '\t' / '\n' )* -IDENT <- !('Set(' / 'SetRowAttrs(' / 'SetColumnAttrs(' / 'Clear(' / 'TopN(' / 'Range(') [[A-Z]] ([[A-Z]] / [0-9])* +IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] diff --git a/pql/pql.peg.go b/pql/pql.peg.go index f62f21552..c0915e921 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -34,10 +34,11 @@ const ( rulesinglequotedstring rulefieldExpr rulefield + rulereserved ruleposfield ruleuint ruleuintrow - ruleuintcol + rulecol ruleopen ruleclose rulesp @@ -93,6 +94,7 @@ const ( ruleAction40 ruleAction41 ruleAction42 + ruleAction43 ) var rul3s = [...]string{ @@ -115,10 +117,11 @@ var rul3s = [...]string{ "singlequotedstring", "fieldExpr", "field", + "reserved", "posfield", "uint", "uintrow", - "uintcol", + "col", "open", "close", "sp", @@ -174,6 +177,7 @@ var rul3s = [...]string{ "Action40", "Action41", "Action42", + "Action43", } type token32 struct { @@ -290,7 +294,7 @@ type PQL struct { Buffer string buffer []rune - rules [78]func() bool + rules [80]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -467,6 +471,8 @@ func (p *PQL) Execute() { case ruleAction41: p.addPosNum("_col", buffer[begin:end]) case ruleAction42: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction43: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -579,7 +585,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open uintcol comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open posfield comma uintcol comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open uintcol comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -604,7 +610,7 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l8 } - if !_rules[ruleuintcol]() { + if !_rules[rulecol]() { goto l8 } if !_rules[rulecomma]() { @@ -628,7 +634,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction42, position) + add(ruleAction43, position) } add(ruletimestamp, position12) } @@ -793,13 +799,7 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l22 } - if !_rules[ruleposfield]() { - goto l22 - } - if !_rules[rulecomma]() { - goto l22 - } - if !_rules[ruleuintcol]() { + if !_rules[rulecol]() { goto l22 } if !_rules[rulecomma]() { @@ -843,7 +843,7 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l25 } - if !_rules[ruleuintcol]() { + if !_rules[rulecol]() { goto l25 } if !_rules[rulecomma]() { @@ -1047,264 +1047,47 @@ func (p *PQL) Init() { position51 := position { position52, tokenIndex52 := position, tokenIndex - { - position53, tokenIndex53 := position, tokenIndex - if buffer[position] != rune('S') { - goto l54 - } - position++ - if buffer[position] != rune('e') { - goto l54 - } - position++ - if buffer[position] != rune('t') { - goto l54 - } - position++ - if buffer[position] != rune('(') { - goto l54 - } - position++ - goto l53 - l54: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('S') { - goto l55 - } - position++ - if buffer[position] != rune('e') { - goto l55 - } - position++ - if buffer[position] != rune('t') { - goto l55 - } - position++ - if buffer[position] != rune('R') { - goto l55 - } - position++ - if buffer[position] != rune('o') { - goto l55 - } - position++ - if buffer[position] != rune('w') { - goto l55 - } - position++ - if buffer[position] != rune('A') { - goto l55 - } - position++ - if buffer[position] != rune('t') { - goto l55 - } - position++ - if buffer[position] != rune('t') { - goto l55 - } - position++ - if buffer[position] != rune('r') { - goto l55 - } - position++ - if buffer[position] != rune('s') { - goto l55 - } - position++ - if buffer[position] != rune('(') { - goto l55 - } - position++ - goto l53 - l55: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('S') { - goto l56 - } - position++ - if buffer[position] != rune('e') { - goto l56 - } - position++ - if buffer[position] != rune('t') { - goto l56 - } - position++ - if buffer[position] != rune('C') { - goto l56 - } - position++ - if buffer[position] != rune('o') { - goto l56 - } - position++ - if buffer[position] != rune('l') { - goto l56 - } - position++ - if buffer[position] != rune('u') { - goto l56 - } - position++ - if buffer[position] != rune('m') { - goto l56 - } - position++ - if buffer[position] != rune('n') { - goto l56 - } - position++ - if buffer[position] != rune('A') { - goto l56 - } - position++ - if buffer[position] != rune('t') { - goto l56 - } - position++ - if buffer[position] != rune('t') { - goto l56 - } - position++ - if buffer[position] != rune('r') { - goto l56 - } - position++ - if buffer[position] != rune('s') { - goto l56 - } - position++ - if buffer[position] != rune('(') { - goto l56 - } - position++ - goto l53 - l56: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('C') { - goto l57 - } - position++ - if buffer[position] != rune('l') { - goto l57 - } - position++ - if buffer[position] != rune('e') { - goto l57 - } - position++ - if buffer[position] != rune('a') { - goto l57 - } - position++ - if buffer[position] != rune('r') { - goto l57 - } - position++ - if buffer[position] != rune('(') { - goto l57 - } - position++ - goto l53 - l57: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('T') { - goto l58 - } - position++ - if buffer[position] != rune('o') { - goto l58 - } - position++ - if buffer[position] != rune('p') { - goto l58 - } - position++ - if buffer[position] != rune('N') { - goto l58 - } - position++ - if buffer[position] != rune('(') { - goto l58 - } - position++ - goto l53 - l58: - position, tokenIndex = position53, tokenIndex53 - if buffer[position] != rune('R') { - goto l52 - } - position++ - if buffer[position] != rune('a') { - goto l52 - } - position++ - if buffer[position] != rune('n') { - goto l52 - } - position++ - if buffer[position] != rune('g') { - goto l52 - } - position++ - if buffer[position] != rune('e') { - goto l52 - } - position++ - if buffer[position] != rune('(') { - goto l52 - } - position++ - } - l53: - goto l5 - l52: - position, tokenIndex = position52, tokenIndex52 - } - { - position59, tokenIndex59 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 + goto l53 } position++ - goto l59 - l60: - position, tokenIndex = position59, tokenIndex59 + goto l52 + l53: + position, tokenIndex = position52, tokenIndex52 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l59: - l61: + l52: + l54: { - position62, tokenIndex62 := position, tokenIndex + position55, tokenIndex55 := position, tokenIndex { - position63, tokenIndex63 := position, tokenIndex + position56, tokenIndex56 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l64 + goto l57 } position++ - goto l63 - l64: - position, tokenIndex = position63, tokenIndex63 + goto l56 + l57: + position, tokenIndex = position56, tokenIndex56 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l65 + goto l58 } position++ - goto l63 - l65: - position, tokenIndex = position63, tokenIndex63 + goto l56 + l58: + position, tokenIndex = position56, tokenIndex56 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l62 + goto l55 } position++ } - l63: - goto l61 - l62: - position, tokenIndex = position62, tokenIndex62 + l56: + goto l54 + l55: + position, tokenIndex = position55, tokenIndex55 } add(ruleIDENT, position51) } @@ -1320,15 +1103,15 @@ func (p *PQL) Init() { goto l5 } { - position67, tokenIndex67 := position, tokenIndex + position60, tokenIndex60 := position, tokenIndex if !_rules[rulecomma]() { - goto l67 + goto l60 } - goto l68 - l67: - position, tokenIndex = position67, tokenIndex67 + goto l61 + l60: + position, tokenIndex = position60, tokenIndex60 } - l68: + l61: if !_rules[ruleclose]() { goto l5 } @@ -1346,232 +1129,232 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position70, tokenIndex70 := position, tokenIndex + position63, tokenIndex63 := position, tokenIndex { - position71 := position + position64 := position { - position72, tokenIndex72 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if !_rules[ruleCall]() { - goto l73 + goto l66 } - l74: + l67: { - position75, tokenIndex75 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex if !_rules[rulecomma]() { - goto l75 + goto l68 } if !_rules[ruleCall]() { - goto l75 + goto l68 } - goto l74 - l75: - position, tokenIndex = position75, tokenIndex75 + goto l67 + l68: + position, tokenIndex = position68, tokenIndex68 } { - position76, tokenIndex76 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex if !_rules[rulecomma]() { - goto l76 + goto l69 } if !_rules[ruleargs]() { - goto l76 + goto l69 } - goto l77 - l76: - position, tokenIndex = position76, tokenIndex76 - } - l77: - goto l72 - l73: - position, tokenIndex = position72, tokenIndex72 - if !_rules[ruleargs]() { - goto l78 - } - goto l72 - l78: - position, tokenIndex = position72, tokenIndex72 - if !_rules[rulesp]() { goto l70 + l69: + position, tokenIndex = position69, tokenIndex69 + } + l70: + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 + if !_rules[ruleargs]() { + goto l71 + } + goto l65 + l71: + position, tokenIndex = position65, tokenIndex65 + if !_rules[rulesp]() { + goto l63 } } - l72: - add(ruleallargs, position71) + l65: + add(ruleallargs, position64) } return true - l70: - position, tokenIndex = position70, tokenIndex70 + l63: + position, tokenIndex = position63, tokenIndex63 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position79, tokenIndex79 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex { - position80 := position + position73 := position if !_rules[rulearg]() { - goto l79 + goto l72 } { - position81, tokenIndex81 := position, tokenIndex + position74, tokenIndex74 := position, tokenIndex if !_rules[rulecomma]() { - goto l81 + goto l74 } if !_rules[ruleargs]() { - goto l81 + goto l74 } - goto l82 - l81: - position, tokenIndex = position81, tokenIndex81 + goto l75 + l74: + position, tokenIndex = position74, tokenIndex74 } - l82: + l75: if !_rules[rulesp]() { - goto l79 + goto l72 } - add(ruleargs, position80) + add(ruleargs, position73) } return true - l79: - position, tokenIndex = position79, tokenIndex79 + l72: + position, tokenIndex = position72, tokenIndex72 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position83, tokenIndex83 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position84 := position + position77 := position { - position85, tokenIndex85 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex if !_rules[rulefield]() { - goto l86 + goto l79 } if !_rules[rulesp]() { - goto l86 + goto l79 } if buffer[position] != rune('=') { - goto l86 + goto l79 } position++ if !_rules[rulesp]() { - goto l86 + goto l79 } if !_rules[rulevalue]() { - goto l86 + goto l79 } - goto l85 - l86: - position, tokenIndex = position85, tokenIndex85 + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 if !_rules[rulefield]() { - goto l83 + goto l76 } if !_rules[rulesp]() { - goto l83 + goto l76 } { - position87 := position + position80 := position { - position88, tokenIndex88 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex if buffer[position] != rune('>') { - goto l89 + goto l82 } position++ if buffer[position] != rune('<') { - goto l89 + goto l82 } position++ { add(ruleAction14, position) } - goto l88 - l89: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l82: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('<') { - goto l91 + goto l84 } position++ if buffer[position] != rune('=') { - goto l91 + goto l84 } position++ { add(ruleAction15, position) } - goto l88 - l91: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l84: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('>') { - goto l93 + goto l86 } position++ if buffer[position] != rune('=') { - goto l93 + goto l86 } position++ { add(ruleAction16, position) } - goto l88 - l93: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l86: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('=') { - goto l95 + goto l88 } position++ if buffer[position] != rune('=') { - goto l95 + goto l88 } position++ { add(ruleAction17, position) } - goto l88 - l95: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l88: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('!') { - goto l97 + goto l90 } position++ if buffer[position] != rune('=') { - goto l97 + goto l90 } position++ { add(ruleAction18, position) } - goto l88 - l97: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l90: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('<') { - goto l99 + goto l92 } position++ { add(ruleAction19, position) } - goto l88 - l99: - position, tokenIndex = position88, tokenIndex88 + goto l81 + l92: + position, tokenIndex = position81, tokenIndex81 if buffer[position] != rune('>') { - goto l83 + goto l76 } position++ { add(ruleAction20, position) } } - l88: - add(ruleCOND, position87) + l81: + add(ruleCOND, position80) } if !_rules[rulesp]() { - goto l83 + goto l76 } if !_rules[rulevalue]() { - goto l83 + goto l76 } } - l85: - add(rulearg, position84) + l78: + add(rulearg, position77) } return true - l83: - position, tokenIndex = position83, tokenIndex83 + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ @@ -1580,102 +1363,102 @@ func (p *PQL) Init() { nil, /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ func() bool { - position104, tokenIndex104 := position, tokenIndex + position97, tokenIndex97 := position, tokenIndex { - position105 := position + position98 := position { - position106 := position + position99 := position { - position107, tokenIndex107 := position, tokenIndex + position100, tokenIndex100 := position, tokenIndex { - position109, tokenIndex109 := position, tokenIndex + position102, tokenIndex102 := position, tokenIndex if buffer[position] != rune('-') { - goto l109 + goto l102 } position++ - goto l110 - l109: - position, tokenIndex = position109, tokenIndex109 + goto l103 + l102: + position, tokenIndex = position102, tokenIndex102 } - l110: + l103: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l108 + goto l101 } position++ - l111: + l104: { - position112, tokenIndex112 := position, tokenIndex + position105, tokenIndex105 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l112 + goto l105 } position++ - goto l111 - l112: - position, tokenIndex = position112, tokenIndex112 - } - goto l107 - l108: - position, tokenIndex = position107, tokenIndex107 - if buffer[position] != rune('0') { goto l104 + l105: + position, tokenIndex = position105, tokenIndex105 + } + goto l100 + l101: + position, tokenIndex = position100, tokenIndex100 + if buffer[position] != rune('0') { + goto l97 } position++ } - l107: - add(rulePegText, position106) + l100: + add(rulePegText, position99) } if !_rules[rulesp]() { - goto l104 + goto l97 } { add(ruleAction23, position) } - add(rulecondint, position105) + add(rulecondint, position98) } return true - l104: - position, tokenIndex = position104, tokenIndex104 + l97: + position, tokenIndex = position97, tokenIndex97 return false }, /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ func() bool { - position114, tokenIndex114 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex { - position115 := position + position108 := position { - position116 := position + position109 := position { - position117, tokenIndex117 := position, tokenIndex + position110, tokenIndex110 := position, tokenIndex if buffer[position] != rune('<') { - goto l118 + goto l111 } position++ if buffer[position] != rune('=') { - goto l118 + goto l111 } position++ - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 if buffer[position] != rune('<') { - goto l114 + goto l107 } position++ } - l117: - add(rulePegText, position116) + l110: + add(rulePegText, position109) } if !_rules[rulesp]() { - goto l114 + goto l107 } { add(ruleAction24, position) } - add(rulecondLT, position115) + add(rulecondLT, position108) } return true - l114: - position, tokenIndex = position114, tokenIndex114 + l107: + position, tokenIndex = position107, tokenIndex107 return false }, /* 9 condfield <- <( sp Action25)> */ @@ -1684,1176 +1467,1376 @@ func (p *PQL) Init() { nil, /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ func() bool { - position122, tokenIndex122 := position, tokenIndex + position115, tokenIndex115 := position, tokenIndex { - position123 := position + position116 := position { - position124, tokenIndex124 := position, tokenIndex + position117, tokenIndex117 := position, tokenIndex if !_rules[ruleitem]() { - goto l125 + goto l118 } - goto l124 - l125: - position, tokenIndex = position124, tokenIndex124 + goto l117 + l118: + position, tokenIndex = position117, tokenIndex117 { - position126 := position + position119 := position if buffer[position] != rune('[') { - goto l122 + goto l115 } position++ if !_rules[rulesp]() { - goto l122 + goto l115 } - add(rulelbrack, position126) + add(rulelbrack, position119) } { add(ruleAction28, position) } if !_rules[rulelist]() { - goto l122 + goto l115 } { - position128 := position + position121 := position if !_rules[rulesp]() { - goto l122 + goto l115 } if buffer[position] != rune(']') { - goto l122 + goto l115 } position++ if !_rules[rulesp]() { - goto l122 + goto l115 } - add(rulerbrack, position128) + add(rulerbrack, position121) } { add(ruleAction29, position) } } - l124: - add(rulevalue, position123) + l117: + add(rulevalue, position116) } return true - l122: - position, tokenIndex = position122, tokenIndex122 + l115: + position, tokenIndex = position115, tokenIndex115 return false }, /* 12 list <- <(item (comma list)?)> */ func() bool { - position130, tokenIndex130 := position, tokenIndex + position123, tokenIndex123 := position, tokenIndex { - position131 := position + position124 := position if !_rules[ruleitem]() { - goto l130 + goto l123 } { - position132, tokenIndex132 := position, tokenIndex + position125, tokenIndex125 := position, tokenIndex if !_rules[rulecomma]() { - goto l132 + goto l125 } if !_rules[rulelist]() { - goto l132 + goto l125 } - goto l133 - l132: - position, tokenIndex = position132, tokenIndex132 + goto l126 + l125: + position, tokenIndex = position125, tokenIndex125 } - l133: - add(rulelist, position131) + l126: + add(rulelist, position124) } return true - l130: - position, tokenIndex = position130, tokenIndex130 + l123: + position, tokenIndex = position123, tokenIndex123 return false }, /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ func() bool { - position134, tokenIndex134 := position, tokenIndex + position127, tokenIndex127 := position, tokenIndex { - position135 := position + position128 := position { - position136, tokenIndex136 := position, tokenIndex + position129, tokenIndex129 := position, tokenIndex if buffer[position] != rune('n') { - goto l137 + goto l130 } position++ if buffer[position] != rune('u') { - goto l137 + goto l130 } position++ if buffer[position] != rune('l') { - goto l137 + goto l130 } position++ if buffer[position] != rune('l') { - goto l137 + goto l130 } position++ { - position138, tokenIndex138 := position, tokenIndex + position131, tokenIndex131 := position, tokenIndex { - position139, tokenIndex139 := position, tokenIndex + position132, tokenIndex132 := position, tokenIndex if !_rules[rulecomma]() { - goto l140 + goto l133 } - goto l139 - l140: - position, tokenIndex = position139, tokenIndex139 + goto l132 + l133: + position, tokenIndex = position132, tokenIndex132 if !_rules[rulesp]() { - goto l137 + goto l130 } if !_rules[ruleclose]() { - goto l137 + goto l130 } } - l139: - position, tokenIndex = position138, tokenIndex138 + l132: + position, tokenIndex = position131, tokenIndex131 } { add(ruleAction30, position) } - goto l136 - l137: - position, tokenIndex = position136, tokenIndex136 + goto l129 + l130: + position, tokenIndex = position129, tokenIndex129 if buffer[position] != rune('t') { - goto l142 + goto l135 } position++ if buffer[position] != rune('r') { - goto l142 + goto l135 } position++ if buffer[position] != rune('u') { - goto l142 + goto l135 } position++ if buffer[position] != rune('e') { - goto l142 + goto l135 } position++ { - position143, tokenIndex143 := position, tokenIndex + position136, tokenIndex136 := position, tokenIndex { - position144, tokenIndex144 := position, tokenIndex + position137, tokenIndex137 := position, tokenIndex if !_rules[rulecomma]() { - goto l145 + goto l138 } - goto l144 - l145: - position, tokenIndex = position144, tokenIndex144 + goto l137 + l138: + position, tokenIndex = position137, tokenIndex137 if !_rules[rulesp]() { - goto l142 + goto l135 } if !_rules[ruleclose]() { - goto l142 + goto l135 } } - l144: - position, tokenIndex = position143, tokenIndex143 + l137: + position, tokenIndex = position136, tokenIndex136 } { add(ruleAction31, position) } - goto l136 - l142: - position, tokenIndex = position136, tokenIndex136 + goto l129 + l135: + position, tokenIndex = position129, tokenIndex129 if buffer[position] != rune('f') { - goto l147 + goto l140 } position++ if buffer[position] != rune('a') { - goto l147 + goto l140 } position++ if buffer[position] != rune('l') { - goto l147 + goto l140 } position++ if buffer[position] != rune('s') { - goto l147 + goto l140 } position++ if buffer[position] != rune('e') { - goto l147 + goto l140 } position++ { - position148, tokenIndex148 := position, tokenIndex + position141, tokenIndex141 := position, tokenIndex { - position149, tokenIndex149 := position, tokenIndex + position142, tokenIndex142 := position, tokenIndex if !_rules[rulecomma]() { - goto l150 + goto l143 } - goto l149 - l150: - position, tokenIndex = position149, tokenIndex149 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 if !_rules[rulesp]() { - goto l147 + goto l140 } if !_rules[ruleclose]() { - goto l147 + goto l140 } } - l149: - position, tokenIndex = position148, tokenIndex148 + l142: + position, tokenIndex = position141, tokenIndex141 } { add(ruleAction32, position) } - goto l136 - l147: - position, tokenIndex = position136, tokenIndex136 + goto l129 + l140: + position, tokenIndex = position129, tokenIndex129 { - position153 := position + position146 := position { - position154, tokenIndex154 := position, tokenIndex + position147, tokenIndex147 := position, tokenIndex if buffer[position] != rune('-') { - goto l154 + goto l147 } position++ - goto l155 - l154: - position, tokenIndex = position154, tokenIndex154 + goto l148 + l147: + position, tokenIndex = position147, tokenIndex147 } - l155: + l148: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 + goto l145 } position++ - l156: + l149: { - position157, tokenIndex157 := position, tokenIndex + position150, tokenIndex150 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l157 + goto l150 } position++ - goto l156 - l157: - position, tokenIndex = position157, tokenIndex157 + goto l149 + l150: + position, tokenIndex = position150, tokenIndex150 } { - position158, tokenIndex158 := position, tokenIndex + position151, tokenIndex151 := position, tokenIndex if buffer[position] != rune('.') { + goto l151 + } + position++ + l153: + { + position154, tokenIndex154 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l154 + } + position++ + goto l153 + l154: + position, tokenIndex = position154, tokenIndex154 + } + goto l152 + l151: + position, tokenIndex = position151, tokenIndex151 + } + l152: + add(rulePegText, position146) + } + { + add(ruleAction33, position) + } + goto l129 + l145: + position, tokenIndex = position129, tokenIndex129 + { + position157 := position + { + position158, tokenIndex158 := position, tokenIndex + if buffer[position] != rune('-') { goto l158 } position++ - l160: - { - position161, tokenIndex161 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l161 - } - position++ - goto l160 - l161: - position, tokenIndex = position161, tokenIndex161 - } goto l159 l158: position, tokenIndex = position158, tokenIndex158 } l159: - add(rulePegText, position153) - } - { - add(ruleAction33, position) - } - goto l136 - l152: - position, tokenIndex = position136, tokenIndex136 - { - position164 := position - { - position165, tokenIndex165 := position, tokenIndex - if buffer[position] != rune('-') { - goto l165 - } - position++ - goto l166 - l165: - position, tokenIndex = position165, tokenIndex165 - } - l166: if buffer[position] != rune('.') { - goto l163 + goto l156 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 + goto l156 } position++ - l167: + l160: { - position168, tokenIndex168 := position, tokenIndex + position161, tokenIndex161 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l161 + } + position++ + goto l160 + l161: + position, tokenIndex = position161, tokenIndex161 + } + add(rulePegText, position157) + } + { + add(ruleAction34, position) + } + goto l129 + l156: + position, tokenIndex = position129, tokenIndex129 + { + position164 := position + { + position167, tokenIndex167 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { goto l168 } position++ goto l167 l168: - position, tokenIndex = position168, tokenIndex168 + position, tokenIndex = position167, tokenIndex167 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l169 + } + position++ + goto l167 + l169: + position, tokenIndex = position167, tokenIndex167 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l170 + } + position++ + goto l167 + l170: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune('-') { + goto l171 + } + position++ + goto l167 + l171: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune('_') { + goto l172 + } + position++ + goto l167 + l172: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune(':') { + goto l163 + } + position++ + } + l167: + l165: + { + position166, tokenIndex166 := position, tokenIndex + { + position173, tokenIndex173 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l174 + } + position++ + goto l173 + l174: + position, tokenIndex = position173, tokenIndex173 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l175 + } + position++ + goto l173 + l175: + position, tokenIndex = position173, tokenIndex173 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l176 + } + position++ + goto l173 + l176: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune('-') { + goto l177 + } + position++ + goto l173 + l177: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune('_') { + goto l178 + } + position++ + goto l173 + l178: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune(':') { + goto l166 + } + position++ + } + l173: + goto l165 + l166: + position, tokenIndex = position166, tokenIndex166 } add(rulePegText, position164) } - { - add(ruleAction34, position) - } - goto l136 - l163: - position, tokenIndex = position136, tokenIndex136 - { - position171 := position - { - position174, tokenIndex174 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l175 - } - position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l176 - } - position++ - goto l174 - l176: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l177 - } - position++ - goto l174 - l177: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('-') { - goto l178 - } - position++ - goto l174 - l178: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('_') { - goto l179 - } - position++ - goto l174 - l179: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune(':') { - goto l170 - } - position++ - } - l174: - l172: - { - position173, tokenIndex173 := position, tokenIndex - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l173 - } - position++ - } - l180: - goto l172 - l173: - position, tokenIndex = position173, tokenIndex173 - } - add(rulePegText, position171) - } { add(ruleAction35, position) } - goto l136 - l170: - position, tokenIndex = position136, tokenIndex136 + goto l129 + l163: + position, tokenIndex = position129, tokenIndex129 if buffer[position] != rune('"') { - goto l187 + goto l180 } position++ { - position188 := position - { - position189 := position - l190: - { - position191, tokenIndex191 := position, tokenIndex - { - position192, tokenIndex192 := position, tokenIndex - { - position194, tokenIndex194 := position, tokenIndex - { - position195, tokenIndex195 := position, tokenIndex - if buffer[position] != rune('"') { - goto l196 - } - position++ - goto l195 - l196: - position, tokenIndex = position195, tokenIndex195 - if buffer[position] != rune('\\') { - goto l197 - } - position++ - goto l195 - l197: - position, tokenIndex = position195, tokenIndex195 - if buffer[position] != rune('\n') { - goto l194 - } - position++ - } - l195: - goto l193 - l194: - position, tokenIndex = position194, tokenIndex194 - } - if !matchDot() { - goto l193 - } - goto l192 - l193: - position, tokenIndex = position192, tokenIndex192 - if buffer[position] != rune('\\') { - goto l198 - } - position++ - if buffer[position] != rune('n') { - goto l198 - } - position++ - goto l192 - l198: - position, tokenIndex = position192, tokenIndex192 - if buffer[position] != rune('\\') { - goto l199 - } - position++ - if buffer[position] != rune('"') { - goto l199 - } - position++ - goto l192 - l199: - position, tokenIndex = position192, tokenIndex192 - if buffer[position] != rune('\\') { - goto l200 - } - position++ - if buffer[position] != rune('\'') { - goto l200 - } - position++ - goto l192 - l200: - position, tokenIndex = position192, tokenIndex192 - if buffer[position] != rune('\\') { - goto l191 - } - position++ - if buffer[position] != rune('\\') { - goto l191 - } - position++ - } - l192: - goto l190 - l191: - position, tokenIndex = position191, tokenIndex191 - } - add(ruledoublequotedstring, position189) + position181 := position + if !_rules[ruledoublequotedstring]() { + goto l180 } - add(rulePegText, position188) + add(rulePegText, position181) } if buffer[position] != rune('"') { - goto l187 + goto l180 } position++ { add(ruleAction36, position) } - goto l136 - l187: - position, tokenIndex = position136, tokenIndex136 + goto l129 + l180: + position, tokenIndex = position129, tokenIndex129 if buffer[position] != rune('\'') { - goto l134 + goto l127 } position++ { - position202 := position + position183 := position { - position203 := position - l204: + position184 := position + l185: { - position205, tokenIndex205 := position, tokenIndex + position186, tokenIndex186 := position, tokenIndex { - position206, tokenIndex206 := position, tokenIndex + position187, tokenIndex187 := position, tokenIndex { - position208, tokenIndex208 := position, tokenIndex + position189, tokenIndex189 := position, tokenIndex { - position209, tokenIndex209 := position, tokenIndex + position190, tokenIndex190 := position, tokenIndex if buffer[position] != rune('\'') { - goto l210 + goto l191 } position++ - goto l209 - l210: - position, tokenIndex = position209, tokenIndex209 + goto l190 + l191: + position, tokenIndex = position190, tokenIndex190 if buffer[position] != rune('\\') { - goto l211 + goto l192 } position++ - goto l209 - l211: - position, tokenIndex = position209, tokenIndex209 + goto l190 + l192: + position, tokenIndex = position190, tokenIndex190 if buffer[position] != rune('\n') { - goto l208 + goto l189 } position++ } - l209: - goto l207 - l208: - position, tokenIndex = position208, tokenIndex208 + l190: + goto l188 + l189: + position, tokenIndex = position189, tokenIndex189 } if !matchDot() { - goto l207 + goto l188 } - goto l206 - l207: - position, tokenIndex = position206, tokenIndex206 + goto l187 + l188: + position, tokenIndex = position187, tokenIndex187 if buffer[position] != rune('\\') { - goto l212 + goto l193 } position++ if buffer[position] != rune('n') { - goto l212 + goto l193 } position++ - goto l206 - l212: - position, tokenIndex = position206, tokenIndex206 + goto l187 + l193: + position, tokenIndex = position187, tokenIndex187 if buffer[position] != rune('\\') { - goto l213 + goto l194 } position++ if buffer[position] != rune('"') { - goto l213 + goto l194 } position++ - goto l206 - l213: - position, tokenIndex = position206, tokenIndex206 + goto l187 + l194: + position, tokenIndex = position187, tokenIndex187 if buffer[position] != rune('\\') { - goto l214 + goto l195 } position++ if buffer[position] != rune('\'') { - goto l214 + goto l195 } position++ - goto l206 - l214: - position, tokenIndex = position206, tokenIndex206 + goto l187 + l195: + position, tokenIndex = position187, tokenIndex187 if buffer[position] != rune('\\') { - goto l205 + goto l186 } position++ if buffer[position] != rune('\\') { - goto l205 + goto l186 } position++ } - l206: - goto l204 - l205: - position, tokenIndex = position205, tokenIndex205 + l187: + goto l185 + l186: + position, tokenIndex = position186, tokenIndex186 } - add(rulesinglequotedstring, position203) + add(rulesinglequotedstring, position184) } - add(rulePegText, position202) + add(rulePegText, position183) } if buffer[position] != rune('\'') { - goto l134 + goto l127 } position++ { add(ruleAction37, position) } } - l136: - add(ruleitem, position135) + l129: + add(ruleitem, position128) } return true - l134: - position, tokenIndex = position134, tokenIndex134 + l127: + position, tokenIndex = position127, tokenIndex127 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ - nil, + func() bool { + { + position198 := position + l199: + { + position200, tokenIndex200 := position, tokenIndex + { + position201, tokenIndex201 := position, tokenIndex + { + position203, tokenIndex203 := position, tokenIndex + { + position204, tokenIndex204 := position, tokenIndex + if buffer[position] != rune('"') { + goto l205 + } + position++ + goto l204 + l205: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\\') { + goto l206 + } + position++ + goto l204 + l206: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\n') { + goto l203 + } + position++ + } + l204: + goto l202 + l203: + position, tokenIndex = position203, tokenIndex203 + } + if !matchDot() { + goto l202 + } + goto l201 + l202: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l207 + } + position++ + if buffer[position] != rune('n') { + goto l207 + } + position++ + goto l201 + l207: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l208 + } + position++ + if buffer[position] != rune('"') { + goto l208 + } + position++ + goto l201 + l208: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l209 + } + position++ + if buffer[position] != rune('\'') { + goto l209 + } + position++ + goto l201 + l209: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l200 + } + position++ + if buffer[position] != rune('\\') { + goto l200 + } + position++ + } + l201: + goto l199 + l200: + position, tokenIndex = position200, tokenIndex200 + } + add(ruledoublequotedstring, position198) + } + return true + }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ nil, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_')*)> */ func() bool { - position218, tokenIndex218 := position, tokenIndex + position211, tokenIndex211 := position, tokenIndex { - position219 := position + position212 := position { - position220, tokenIndex220 := position, tokenIndex + position213, tokenIndex213 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l221 + goto l214 } position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 + goto l213 + l214: + position, tokenIndex = position213, tokenIndex213 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l218 + goto l211 } position++ } - l220: - l222: + l213: + l215: { - position223, tokenIndex223 := position, tokenIndex + position216, tokenIndex216 := position, tokenIndex { - position224, tokenIndex224 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l225 + goto l218 } position++ + goto l217 + l218: + position, tokenIndex = position217, tokenIndex217 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l219 + } + position++ + goto l217 + l219: + position, tokenIndex = position217, tokenIndex217 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l220 + } + position++ + goto l217 + l220: + position, tokenIndex = position217, tokenIndex217 + if buffer[position] != rune('_') { + goto l216 + } + position++ + } + l217: + goto l215 + l216: + position, tokenIndex = position216, tokenIndex216 + } + add(rulefieldExpr, position212) + } + return true + l211: + position, tokenIndex = position211, tokenIndex211 + return false + }, + /* 17 field <- <(<(fieldExpr / reserved)> Action38)> */ + func() bool { + position221, tokenIndex221 := position, tokenIndex + { + position222 := position + { + position223 := position + { + position224, tokenIndex224 := position, tokenIndex + if !_rules[rulefieldExpr]() { + goto l225 + } goto l224 l225: position, tokenIndex = position224, tokenIndex224 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l226 + { + position226 := position + { + position227, tokenIndex227 := position, tokenIndex + if buffer[position] != rune('_') { + goto l228 + } + position++ + if buffer[position] != rune('r') { + goto l228 + } + position++ + if buffer[position] != rune('o') { + goto l228 + } + position++ + if buffer[position] != rune('w') { + goto l228 + } + position++ + goto l227 + l228: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l229 + } + position++ + if buffer[position] != rune('c') { + goto l229 + } + position++ + if buffer[position] != rune('o') { + goto l229 + } + position++ + if buffer[position] != rune('l') { + goto l229 + } + position++ + goto l227 + l229: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l230 + } + position++ + if buffer[position] != rune('s') { + goto l230 + } + position++ + if buffer[position] != rune('t') { + goto l230 + } + position++ + if buffer[position] != rune('a') { + goto l230 + } + position++ + if buffer[position] != rune('r') { + goto l230 + } + position++ + if buffer[position] != rune('t') { + goto l230 + } + position++ + goto l227 + l230: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l231 + } + position++ + if buffer[position] != rune('e') { + goto l231 + } + position++ + if buffer[position] != rune('n') { + goto l231 + } + position++ + if buffer[position] != rune('d') { + goto l231 + } + position++ + goto l227 + l231: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l232 + } + position++ + if buffer[position] != rune('t') { + goto l232 + } + position++ + if buffer[position] != rune('i') { + goto l232 + } + position++ + if buffer[position] != rune('m') { + goto l232 + } + position++ + if buffer[position] != rune('e') { + goto l232 + } + position++ + if buffer[position] != rune('s') { + goto l232 + } + position++ + if buffer[position] != rune('t') { + goto l232 + } + position++ + if buffer[position] != rune('a') { + goto l232 + } + position++ + if buffer[position] != rune('m') { + goto l232 + } + position++ + if buffer[position] != rune('p') { + goto l232 + } + position++ + goto l227 + l232: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l221 + } + position++ + if buffer[position] != rune('f') { + goto l221 + } + position++ + if buffer[position] != rune('i') { + goto l221 + } + position++ + if buffer[position] != rune('e') { + goto l221 + } + position++ + if buffer[position] != rune('l') { + goto l221 + } + position++ + if buffer[position] != rune('d') { + goto l221 + } + position++ + } + l227: + add(rulereserved, position226) } - position++ - goto l224 - l226: - position, tokenIndex = position224, tokenIndex224 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l227 - } - position++ - goto l224 - l227: - position, tokenIndex = position224, tokenIndex224 - if buffer[position] != rune('_') { - goto l223 - } - position++ } l224: - goto l222 - l223: - position, tokenIndex = position223, tokenIndex223 - } - add(rulefieldExpr, position219) - } - return true - l218: - position, tokenIndex = position218, tokenIndex218 - return false - }, - /* 17 field <- <( Action38)> */ - func() bool { - position228, tokenIndex228 := position, tokenIndex - { - position229 := position - { - position230 := position - if !_rules[rulefieldExpr]() { - goto l228 - } - add(rulePegText, position230) + add(rulePegText, position223) } { add(ruleAction38, position) } - add(rulefield, position229) + add(rulefield, position222) } return true - l228: - position, tokenIndex = position228, tokenIndex228 + l221: + position, tokenIndex = position221, tokenIndex221 return false }, - /* 18 posfield <- <( Action39)> */ + /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ + nil, + /* 19 posfield <- <( Action39)> */ func() bool { - position232, tokenIndex232 := position, tokenIndex + position235, tokenIndex235 := position, tokenIndex { - position233 := position + position236 := position { - position234 := position + position237 := position if !_rules[rulefieldExpr]() { - goto l232 + goto l235 } - add(rulePegText, position234) + add(rulePegText, position237) } { add(ruleAction39, position) } - add(ruleposfield, position233) + add(ruleposfield, position236) } return true - l232: - position, tokenIndex = position232, tokenIndex232 + l235: + position, tokenIndex = position235, tokenIndex235 return false }, - /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ + /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position236, tokenIndex236 := position, tokenIndex + position239, tokenIndex239 := position, tokenIndex { - position237 := position + position240 := position { - position238, tokenIndex238 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l242 + } + position++ + l243: + { + position244, tokenIndex244 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l244 + } + position++ + goto l243 + l244: + position, tokenIndex = position244, tokenIndex244 + } + goto l241 + l242: + position, tokenIndex = position241, tokenIndex241 + if buffer[position] != rune('0') { goto l239 } position++ - l240: + } + l241: + add(ruleuint, position240) + } + return true + l239: + position, tokenIndex = position239, tokenIndex239 + return false + }, + /* 21 uintrow <- <( Action40)> */ + nil, + /* 22 col <- <(( Action41) / ('"' '"' Action42))> */ + func() bool { + position246, tokenIndex246 := position, tokenIndex + { + position247 := position + { + position248, tokenIndex248 := position, tokenIndex { - position241, tokenIndex241 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l241 + position250 := position + if !_rules[ruleuint]() { + goto l249 } - position++ - goto l240 - l241: - position, tokenIndex = position241, tokenIndex241 + add(rulePegText, position250) } - goto l238 - l239: - position, tokenIndex = position238, tokenIndex238 - if buffer[position] != rune('0') { - goto l236 + { + add(ruleAction41, position) + } + goto l248 + l249: + position, tokenIndex = position248, tokenIndex248 + if buffer[position] != rune('"') { + goto l246 } position++ - } - l238: - add(ruleuint, position237) - } - return true - l236: - position, tokenIndex = position236, tokenIndex236 - return false - }, - /* 20 uintrow <- <( Action40)> */ - nil, - /* 21 uintcol <- <( Action41)> */ - func() bool { - position243, tokenIndex243 := position, tokenIndex - { - position244 := position - { - position245 := position - if !_rules[ruleuint]() { - goto l243 - } - add(rulePegText, position245) - } - { - add(ruleAction41, position) - } - add(ruleuintcol, position244) - } - return true - l243: - position, tokenIndex = position243, tokenIndex243 - return false - }, - /* 22 open <- <('(' sp)> */ - func() bool { - position247, tokenIndex247 := position, tokenIndex - { - position248 := position - if buffer[position] != rune('(') { - goto l247 - } - position++ - if !_rules[rulesp]() { - goto l247 - } - add(ruleopen, position248) - } - return true - l247: - position, tokenIndex = position247, tokenIndex247 - return false - }, - /* 23 close <- <(')' sp)> */ - func() bool { - position249, tokenIndex249 := position, tokenIndex - { - position250 := position - if buffer[position] != rune(')') { - goto l249 - } - position++ - if !_rules[rulesp]() { - goto l249 - } - add(ruleclose, position250) - } - return true - l249: - position, tokenIndex = position249, tokenIndex249 - return false - }, - /* 24 sp <- <(' ' / '\t')*> */ - func() bool { - { - position252 := position - l253: - { - position254, tokenIndex254 := position, tokenIndex { - position255, tokenIndex255 := position, tokenIndex + position252 := position + if !_rules[ruledoublequotedstring]() { + goto l246 + } + add(rulePegText, position252) + } + if buffer[position] != rune('"') { + goto l246 + } + position++ + { + add(ruleAction42, position) + } + } + l248: + add(rulecol, position247) + } + return true + l246: + position, tokenIndex = position246, tokenIndex246 + return false + }, + /* 23 open <- <('(' sp)> */ + func() bool { + position254, tokenIndex254 := position, tokenIndex + { + position255 := position + if buffer[position] != rune('(') { + goto l254 + } + position++ + if !_rules[rulesp]() { + goto l254 + } + add(ruleopen, position255) + } + return true + l254: + position, tokenIndex = position254, tokenIndex254 + return false + }, + /* 24 close <- <(')' sp)> */ + func() bool { + position256, tokenIndex256 := position, tokenIndex + { + position257 := position + if buffer[position] != rune(')') { + goto l256 + } + position++ + if !_rules[rulesp]() { + goto l256 + } + add(ruleclose, position257) + } + return true + l256: + position, tokenIndex = position256, tokenIndex256 + return false + }, + /* 25 sp <- <(' ' / '\t')*> */ + func() bool { + { + position259 := position + l260: + { + position261, tokenIndex261 := position, tokenIndex + { + position262, tokenIndex262 := position, tokenIndex if buffer[position] != rune(' ') { - goto l256 + goto l263 } position++ - goto l255 - l256: - position, tokenIndex = position255, tokenIndex255 + goto l262 + l263: + position, tokenIndex = position262, tokenIndex262 if buffer[position] != rune('\t') { - goto l254 + goto l261 } position++ } - l255: - goto l253 - l254: - position, tokenIndex = position254, tokenIndex254 + l262: + goto l260 + l261: + position, tokenIndex = position261, tokenIndex261 } - add(rulesp, position252) + add(rulesp, position259) } return true }, - /* 25 comma <- <(sp ',' whitesp)> */ + /* 26 comma <- <(sp ',' whitesp)> */ func() bool { - position257, tokenIndex257 := position, tokenIndex + position264, tokenIndex264 := position, tokenIndex { - position258 := position + position265 := position if !_rules[rulesp]() { - goto l257 + goto l264 } if buffer[position] != rune(',') { - goto l257 + goto l264 } position++ if !_rules[rulewhitesp]() { - goto l257 + goto l264 } - add(rulecomma, position258) + add(rulecomma, position265) } return true - l257: - position, tokenIndex = position257, tokenIndex257 + l264: + position, tokenIndex = position264, tokenIndex264 return false }, - /* 26 lbrack <- <('[' sp)> */ + /* 27 lbrack <- <('[' sp)> */ nil, - /* 27 rbrack <- <(sp ']' sp)> */ + /* 28 rbrack <- <(sp ']' sp)> */ nil, - /* 28 whitesp <- <(' ' / '\t' / '\n')*> */ + /* 29 whitesp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position262 := position - l263: + position269 := position + l270: { - position264, tokenIndex264 := position, tokenIndex + position271, tokenIndex271 := position, tokenIndex { - position265, tokenIndex265 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex if buffer[position] != rune(' ') { - goto l266 + goto l273 } position++ - goto l265 - l266: - position, tokenIndex = position265, tokenIndex265 + goto l272 + l273: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('\t') { - goto l267 + goto l274 } position++ - goto l265 - l267: - position, tokenIndex = position265, tokenIndex265 + goto l272 + l274: + position, tokenIndex = position272, tokenIndex272 if buffer[position] != rune('\n') { - goto l264 + goto l271 } position++ } - l265: - goto l263 - l264: - position, tokenIndex = position264, tokenIndex264 + l272: + goto l270 + l271: + position, tokenIndex = position271, tokenIndex271 } - add(rulewhitesp, position262) + add(rulewhitesp, position269) } return true }, - /* 29 IDENT <- <(!(('S' 'e' 't' '(') / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' '(') / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' '(') / ('C' 'l' 'e' 'a' 'r' '(') / ('T' 'o' 'p' 'N' '(') / ('R' 'a' 'n' 'g' 'e' '(')) ([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 30 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, - /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position269, tokenIndex269 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex { - position270 := position + position277 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if buffer[position] != rune('-') { - goto l269 + goto l276 } position++ { - position271, tokenIndex271 := position, tokenIndex + position278, tokenIndex278 := position, tokenIndex if buffer[position] != rune('0') { - goto l272 + goto l279 } position++ - goto l271 - l272: - position, tokenIndex = position271, tokenIndex271 + goto l278 + l279: + position, tokenIndex = position278, tokenIndex278 if buffer[position] != rune('1') { - goto l269 + goto l276 } position++ } - l271: + l278: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if buffer[position] != rune('-') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if buffer[position] != rune('T') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if buffer[position] != rune(':') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l269 + goto l276 } position++ - add(ruletimestampbasicfmt, position270) + add(ruletimestampbasicfmt, position277) } return true - l269: - position, tokenIndex = position269, tokenIndex269 + l276: + position, tokenIndex = position276, tokenIndex276 return false }, - /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position273, tokenIndex273 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex { - position274 := position + position281 := position { - position275, tokenIndex275 := position, tokenIndex + position282, tokenIndex282 := position, tokenIndex if buffer[position] != rune('"') { - goto l276 + goto l283 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l276 + goto l283 } if buffer[position] != rune('"') { - goto l276 + goto l283 } position++ - goto l275 - l276: - position, tokenIndex = position275, tokenIndex275 + goto l282 + l283: + position, tokenIndex = position282, tokenIndex282 if buffer[position] != rune('\'') { - goto l277 + goto l284 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l277 + goto l284 } if buffer[position] != rune('\'') { - goto l277 + goto l284 } position++ - goto l275 - l277: - position, tokenIndex = position275, tokenIndex275 + goto l282 + l284: + position, tokenIndex = position282, tokenIndex282 if !_rules[ruletimestampbasicfmt]() { - goto l273 + goto l280 } } - l275: - add(ruletimestampfmt, position274) + l282: + add(ruletimestampfmt, position281) } return true - l273: - position, tokenIndex = position273, tokenIndex273 + l280: + position, tokenIndex = position280, tokenIndex280 return false }, - /* 32 timestamp <- <( Action42)> */ + /* 33 timestamp <- <( Action43)> */ nil, - /* 34 Action0 <- <{p.startCall("Set")}> */ + /* 35 Action0 <- <{p.startCall("Set")}> */ nil, - /* 35 Action1 <- <{p.endCall()}> */ + /* 36 Action1 <- <{p.endCall()}> */ nil, - /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 37 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 37 Action3 <- <{p.endCall()}> */ + /* 38 Action3 <- <{p.endCall()}> */ nil, - /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + /* 39 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, - /* 39 Action5 <- <{p.endCall()}> */ + /* 40 Action5 <- <{p.endCall()}> */ nil, - /* 40 Action6 <- <{p.startCall("Clear")}> */ + /* 41 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 41 Action7 <- <{p.endCall()}> */ + /* 42 Action7 <- <{p.endCall()}> */ nil, - /* 42 Action8 <- <{p.startCall("TopN")}> */ + /* 43 Action8 <- <{p.startCall("TopN")}> */ nil, - /* 43 Action9 <- <{p.endCall()}> */ + /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 44 Action10 <- <{p.startCall("Range")}> */ + /* 45 Action10 <- <{p.startCall("Range")}> */ nil, - /* 45 Action11 <- <{p.endCall()}> */ + /* 46 Action11 <- <{p.endCall()}> */ nil, nil, - /* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 48 Action13 <- <{ p.endCall() }> */ + /* 49 Action13 <- <{ p.endCall() }> */ nil, - /* 49 Action14 <- <{ p.addBTWN() }> */ + /* 50 Action14 <- <{ p.addBTWN() }> */ nil, - /* 50 Action15 <- <{ p.addLTE() }> */ + /* 51 Action15 <- <{ p.addLTE() }> */ nil, - /* 51 Action16 <- <{ p.addGTE() }> */ + /* 52 Action16 <- <{ p.addGTE() }> */ nil, - /* 52 Action17 <- <{ p.addEQ() }> */ + /* 53 Action17 <- <{ p.addEQ() }> */ nil, - /* 53 Action18 <- <{ p.addNEQ() }> */ + /* 54 Action18 <- <{ p.addNEQ() }> */ nil, - /* 54 Action19 <- <{ p.addLT() }> */ + /* 55 Action19 <- <{ p.addLT() }> */ nil, - /* 55 Action20 <- <{ p.addGT() }> */ + /* 56 Action20 <- <{ p.addGT() }> */ nil, - /* 56 Action21 <- <{p.startConditional()}> */ + /* 57 Action21 <- <{p.startConditional()}> */ nil, - /* 57 Action22 <- <{p.endConditional()}> */ + /* 58 Action22 <- <{p.endConditional()}> */ nil, - /* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 63 Action28 <- <{ p.startList() }> */ + /* 64 Action28 <- <{ p.startList() }> */ nil, - /* 64 Action29 <- <{ p.endList() }> */ + /* 65 Action29 <- <{ p.endList() }> */ nil, - /* 65 Action30 <- <{ p.addVal(nil) }> */ + /* 66 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 66 Action31 <- <{ p.addVal(true) }> */ + /* 67 Action31 <- <{ p.addVal(true) }> */ nil, - /* 67 Action32 <- <{ p.addVal(false) }> */ + /* 68 Action32 <- <{ p.addVal(false) }> */ nil, - /* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 77 Action42 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 79 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 023bbc71c..1bedd797b 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -47,6 +47,13 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 } +func TestOldPQL(t *testing.T) { + _, err := ParseString(`SetBit(f=11, col=1)`) + if err != nil { + t.Fatalf("should have parsed: %v", err) + } +} + func TestPEGWorking(t *testing.T) { tests := []struct { name string @@ -59,7 +66,11 @@ func TestPEGWorking(t *testing.T) { ncalls: 0}, { name: "Set", - input: "Set(1, a=4)", + input: "Set(2, f=10)", + ncalls: 1}, + { + name: "SetTime", + input: "Set(2, f=1, 1999-12-31T00:00)", ncalls: 1}, { name: "DoubleSet", @@ -143,11 +154,11 @@ func TestPEGWorking(t *testing.T) { ncalls: 1}, { name: "SetColumnAttrs", - input: "SetColumnAttrs(blah, 9, a=47)", + input: "SetColumnAttrs(9, a=47)", ncalls: 1}, { name: "SetColumnAttrs2args", - input: "SetColumnAttrs(blah, 9, a=47, b=bval)", + input: "SetColumnAttrs(9, a=47, b=bval)", ncalls: 1}, { name: "Clear", @@ -233,12 +244,6 @@ func TestPEGErrors(t *testing.T) { name string input string }{ - { - name: "SetEmpty", - input: "Set()"}, - { - name: "SetNoCol", - input: "Set(a=4)"}, { name: "SetNoParens", input: "Set"}, @@ -248,24 +253,12 @@ func TestPEGErrors(t *testing.T) { { name: "SetTimestampNoArg", input: "Set(1, 2017-04-03T19:34)"}, - { - name: "SetRowAttrsNoField", - input: "SetRowAttrs(a=4)"}, - { - name: "SetColumnAttrsNoField", - input: "SetColumnAttrs(a=4)"}, - { - name: "ClearNoCol", - input: "Clear(a=4)"}, { name: "SetStartingComma", input: "Set(, 1, a=4)"}, { name: "StartinCommaArb", input: "Zeeb(, a=4)"}, - { - name: "TopN No Field", - input: "TopN(a=77)"}, { name: "SetRowAttrs0args", input: "SetRowAttrs(blah, 9)"}, @@ -320,13 +313,12 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "SetColumnAttrs", - call: "SetColumnAttrs(myfield, 9, z=4)", + call: "SetColumnAttrs(9, z=4)", exp: &Call{ Name: "SetColumnAttrs", Args: map[string]interface{}{ - "z": int64(4), - "_field": "myfield", - "_col": int64(9), + "z": int64(4), + "_col": int64(9), }, }}, { @@ -472,6 +464,51 @@ func TestPQLDeepEquality(t *testing.T) { }, }, }}, + { + name: "Sum", + call: "Sum(field=f)", + exp: &Call{ + Name: "Sum", + Args: map[string]interface{}{ + "field": "f", + }, + }}, + { + name: "SumChild", + call: "Sum(Row(), field=f)", + exp: &Call{ + Name: "Sum", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "MinChild", + call: "Min(Row(), field=f)", + exp: &Call{ + Name: "Min", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "MaxChild", + call: "Max(Row(), field=f)", + exp: &Call{ + Name: "Max", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, } for i, test := range tests { From 3c4ba82a4ab3b4047a3c969969ac528b285be6f4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 08:08:53 -0500 Subject: [PATCH 22/33] finish conversion of handler tests --- server/handler_test.go | 59 ++++++++++++++++-------------------------- test/pilosa.go | 7 +++++ 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 9ac0f8931..8d001b529 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -551,46 +551,33 @@ func TestHandler_Endpoints(t *testing.T) { } }) -} + t.Run("CORS", func(t *testing.T) { + req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) + req.Header.Add("Origin", "http://test/") + req.Header.Add("Access-Control-Request-Method", "POST") -func TestHandler_CORS(t *testing.T) { - t.Skip() // Until test.NewServer() works + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + result := w.Result() - hldr := test.MustOpenHolder() - defer hldr.Close() + // This handler does not support CORS, return Method Not Allowed (405) + if result.StatusCode != 405 { + t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + clus := test.MustRunMainWithCluster(t, 1, test.OptAllowedOrigins([]string{"http://test/"})) + w = httptest.NewRecorder() + h := clus[0].Handler.(*http.Handler).Handler + h.ServeHTTP(w, req) + result = w.Result() - // No CORS config present, so should fail - handler := test.MustNewHandler() - - req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) - req.Header.Add("Origin", "http://test/") - req.Header.Add("Access-Control-Request-Method", "POST") - - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - result := w.Result() - - // This handler does not support CORS, return Method Not Allowed (405) - if result.StatusCode != 405 { - t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) - } - - // CORS config should allow preflight response - handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) - w = httptest.NewRecorder() - handler.ServeHTTP(w, req) - result = w.Result() - - if result.StatusCode != 200 { - t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) - } - if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { - t.Fatal("CORS header not present") - } + if result.StatusCode != 200 { + t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) + } + if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { + t.Fatal("CORS header not present") + } + }) } func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { diff --git a/test/pilosa.go b/test/pilosa.go index 22e89a5df..0f5cdd0d5 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -52,6 +52,13 @@ func OptAntiEntropyInterval(dur time.Duration) MainOpt { } } +func OptAllowedOrigins(origins []string) MainOpt { + return func(m *Main) error { + m.Config.Handler.AllowedOrigins = origins + return nil + } +} + // NewMain returns a new instance of Main with a temporary data directory and random port. func NewMain(opts ...MainOpt) *Main { path, err := ioutil.TempDir("", "pilosa-") From 35526dd0d7bc319790446928d9b9eb885c42b837 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 22 Jun 2018 08:11:27 -0500 Subject: [PATCH 23/33] Update to new PQL syntax beyond the parser --- cluster.go | 2 +- executor.go | 157 ++++++++++---------- executor_test.go | 319 +++++++++++++++++++++++++---------------- fragment.go | 4 +- http/client_test.go | 2 +- http/handler_test.go | 28 ++-- server/cluster_test.go | 14 +- server/config.go | 2 +- server/server_test.go | 48 +++---- stats_test.go | 10 +- 10 files changed, 336 insertions(+), 250 deletions(-) diff --git a/cluster.go b/cluster.go index c8cec60a2..4299f8921 100644 --- a/cluster.go +++ b/cluster.go @@ -229,7 +229,7 @@ type Cluster struct { // Threshold for logging long-running queries LongQueryTime time.Duration - // Maximum number of SetBit() or ClearBit() commands per request. + // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int // EventReceiver receives NodeEvents pertaining to node membership. diff --git a/executor.go b/executor.go index c4abe0bd0..027bf50ad 100644 --- a/executor.go +++ b/executor.go @@ -48,7 +48,7 @@ type Executor struct { // Client used for remote requests. client InternalQueryClient - // Maximum number of SetBit() or ClearBit() commands per request. + // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int // Stores key/id translation data. @@ -178,12 +178,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeMax(ctx, index, c, slices, opt) - case "ClearBit": + case "Clear": return e.executeClearBit(ctx, index, c, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, slices, opt) - case "SetBit": + case "Set": return e.executeSetBit(ctx, index, c, opt) case "SetValue": return nil, e.executeSetValue(ctx, index, c, opt) @@ -340,17 +340,17 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return nil, err } - // Attach attributes for Bitmap() calls. + // Attach attributes for Row() calls. // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. row, _ := other.(*Row) - if c.Name == "Bitmap" { + if c.Name == "Row" { if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { idx := e.Holder.Index(index) if idx != nil { - if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg("_" + columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, errors.Wrap(err, "getting column attrs") @@ -359,9 +359,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else if err != nil { return nil, err } else { - field, _ := c.Args["field"].(string) - if fr := idx.Field(field); fr != nil { - rowID, _, err := c.UintArg(rowLabel) + // field, _ := c.Args["field"].(string) + fieldName, _ := c.FieldArg() + if fr := idx.Field(fieldName); fr != nil { + rowID, _, err := c.UintArg(fieldName) if err != nil { return nil, errors.Wrap(err, "getting row") } @@ -386,7 +387,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // executeBitmapCallSlice executes a bitmap call for a single slice. func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { switch c.Name { - case "Bitmap": + case "Row": return e.executeBitmapSlice(ctx, index, c, slice) case "Difference": return e.executeDifferenceSlice(ctx, index, c, slice) @@ -585,7 +586,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { - field, _ := c.Args["field"].(string) + field, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -675,24 +676,24 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. } // Fetch field & row label based on argument. - field, _ := c.Args["field"].(string) - if field == "" { - field = defaultField + fieldName, err := c.FieldArg() + if err != nil { + return nil, errors.New("Row() argument required: field") } - f := e.Holder.Field(index, field) + f := e.Holder.Field(index, fieldName) if f == nil { return nil, ErrFieldNotFound } - rowID, rowOK, rowErr := c.UintArg(rowLabel) + rowID, rowOK, rowErr := c.UintArg(fieldName) if rowErr != nil { - return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr) + return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) } if !rowOK { - return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) + return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, field, ViewStandard, slice) + frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice) if frag == nil { return NewRow(), nil } @@ -728,10 +729,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return e.executeBSIGroupRangeSlice(ctx, index, c, slice) } - // Parse field, use default if unset. - field, _ := c.Args["field"].(string) - if field == "" { - field = defaultField + // Parse field. + fieldName, err := c.FieldArg() + if err != nil { + return nil, errors.New("Range() argument required: field") } // Retrieve column label. @@ -741,13 +742,13 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Retrieve base field. - f := idx.Field(field) + f := idx.Field(fieldName) if f == nil { return nil, ErrFieldNotFound } // Read row & column id. - rowID, rowOK, err := c.UintArg(rowLabel) + rowID, rowOK, err := c.UintArg(fieldName) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } @@ -756,7 +757,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Parse start time. - startTimeStr, ok := c.Args["start"].(string) + startTimeStr, ok := c.Args["_start"].(string) if !ok { return nil, errors.New("Range() start time required") } @@ -766,7 +767,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Parse end time. - endTimeStr, ok := c.Args["end"].(string) + endTimeStr, ok := c.Args["_end"].(string) if !ok { return nil, errors.New("Range() end time required") } @@ -784,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, field, view, slice) + f := e.Holder.Fragment(index, fieldName, view, slice) if f == nil { continue } @@ -994,11 +995,11 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, return n, nil } -// executeClearBit executes a ClearBit() call. +// executeClearBit executes a Clear() call. func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - field, ok := c.Args["field"].(string) - if !ok { - return false, errors.New("ClearBit() field required") + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("Clear() argument required: field") } // Retrieve field. @@ -1006,30 +1007,30 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if idx == nil { return false, ErrIndexNotFound } - f := idx.Field(field) + f := idx.Field(fieldName) if f == nil { return false, ErrFieldNotFound } // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(fieldName) if err != nil { - return false, fmt.Errorf("reading ClearBit() row: %v", err) + return false, fmt.Errorf("reading Clear() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { - return false, fmt.Errorf("reading ClearBit() column: %v", err) + return false, fmt.Errorf("reading Clear() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) + return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel) } return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } -// executeClearBitField executes a ClearBit() call for a single view. +// executeClearBitField executes a Clear() call for a single view. func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false @@ -1059,11 +1060,11 @@ func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetBit executes a SetBit() call. +// executeSetBit executes a Set() call. func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - field, ok := c.Args["field"].(string) - if !ok { - return false, errors.New("SetBit() field required: field") + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("Set() argument required: field") } // Retrieve field. @@ -1071,28 +1072,28 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, if idx == nil { return false, ErrIndexNotFound } - f := idx.Field(field) + f := idx.Field(fieldName) if f == nil { return false, ErrFieldNotFound } // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(fieldName) if err != nil { - return false, fmt.Errorf("reading SetBit() row: %v", err) + return false, fmt.Errorf("reading Set() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { - return false, fmt.Errorf("reading SetBit() column: %v", err) + return false, fmt.Errorf("reading Set() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) + return false, fmt.Errorf("Set() column argument '%v' required", columnLabel) } var timestamp *time.Time - sTimestamp, ok := c.Args["timestamp"].(string) + sTimestamp, ok := c.Args["_timestamp"].(string) if ok { t, err := time.Parse(TimeFormat, sTimestamp) if err != nil { @@ -1104,7 +1105,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } -// executeSetBitField executes a SetBit() call for a specific view. +// executeSetBitField executes a Set() call for a specific view. func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false @@ -1198,7 +1199,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal // executeSetRowAttrs executes a SetRowAttrs() call. func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - fieldName, ok := c.Args["field"].(string) + fieldName, ok := c.Args["_field"].(string) if !ok { return errors.New("SetRowAttrs() field required") } @@ -1210,7 +1211,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Parse labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg("_" + rowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { @@ -1219,8 +1220,8 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "field") - delete(attrs, rowLabel) + delete(attrs, "_field") + delete(attrs, "_"+rowLabel) // Set attributes. if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil { @@ -1258,7 +1259,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { - field, ok := c.Args["field"].(string) + field, ok := c.Args["_field"].(string) if !ok { return nil, errors.New("SetRowAttrs() field required") } @@ -1269,7 +1270,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal return nil, ErrFieldNotFound } - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg("_" + rowLabel) if err != nil { return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { @@ -1278,8 +1279,8 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "field") - delete(attrs, rowLabel) + delete(attrs, "_field") + delete(attrs, "_"+rowLabel) // Create field group, if not exists. fieldMap := m[field] @@ -1348,14 +1349,14 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - col, okCol, errCol := c.UintArg(columnLabel) + col, okCol, errCol := c.UintArg("_" + columnLabel) if errCol != nil || !okCol { return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, columnLabel) + delete(attrs, "_"+columnLabel) delete(attrs, "field") // Set attributes. @@ -1420,9 +1421,9 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": v, err = pb.Results[i].N, nil - case "SetBit": + case "Set": v, err = pb.Results[i].Changed, nil - case "ClearBit": + case "Clear": v, err = pb.Results[i].Changed, nil case "SetRowAttrs": case "SetColumnAttrs": @@ -1493,6 +1494,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, case resp := <-ch: // On error retry against remaining nodes. If an error returns then // the context will cancel and cause all open goroutines to return. + if resp.err != nil { // Filter out unavailable nodes. nodes = Nodes(nodes).Filter(resp.node) @@ -1591,27 +1593,38 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { + var colKey, rowKey, fieldName string + if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + // Positional args in new PQL syntax require special handling here. + colKey = "_" + columnLabel + fieldName, _ = c.FieldArg() + rowKey = fieldName + } else { + colKey = "col" + fieldName = callArgString(c, "field") + rowKey = "row" + } // Translate column key. if idx.Keys() { - if value := callArgString(c, "col"); value != "" { + if value := callArgString(c, colKey); value != "" { ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) if err != nil { return err } - c.Args["col"] = ids[0] + c.Args[colKey] = ids[0] } } // Translate row key, if field is specified & key exists. - if fieldName := callArgString(c, "field"); fieldName != "" { + if fieldName != "" { field := idx.Field(fieldName) if field.Keys() { - if value := callArgString(c, "row"); value != "" { + if value := callArgString(c, rowKey); value != "" { ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) if err != nil { return err } - c.Args["row"] = ids[0] + c.Args[rowKey] = ids[0] } } } @@ -1644,7 +1657,7 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res } case []Pair: - if fieldName := callArgString(call, "field"); fieldName != "" { + if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) if field.Keys() { other := make([]Pair, len(result)) @@ -1713,7 +1726,7 @@ func needsSlices(calls []*pql.Call) bool { } for _, call := range calls { switch call.Name { - case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs": + case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue case "Count", "TopN": return true diff --git a/executor_test.go b/executor_test.go index 164133ecb..862715697 100644 --- a/executor_test.go +++ b/executor_test.go @@ -44,9 +44,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) @@ -63,7 +63,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) @@ -72,7 +72,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) @@ -93,9 +93,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } @@ -116,15 +116,15 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - `SetBit(field=f, row="bar", col="foo")`+"\n"+ - `SetBit(field=f, row="baz", col="foo")`+"\n"+ - `SetBit(field=f, row="bar", col="bat")`+"\n"+ - `SetBit(field=f, row="bbb", col="aaa")`+"\n", + `Set("foo", f="bar")`+"\n"+ + `Set("foo", f="baz")`+"\n"+ + `Set("bat", f="bar")`+"\n"+ + `Set("aaa", f="bbb")`+"\n", ), nil, nil); err != nil { t.Fatal(err) } - if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil { + if results, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f="bar")`), nil, nil); err != nil { t.Fatal(err) } else if diff := cmp.Diff(results, []interface{}{ &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, @@ -145,7 +145,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.SetBit("i", "general", 11, 4) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { t.Fatalf("unexpected columns: %+v", columns) @@ -177,7 +177,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) @@ -207,7 +207,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) @@ -240,7 +240,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) @@ -256,7 +256,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.SetBit("i", "f", 10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -264,7 +264,7 @@ func TestExecutor_Execute_Count(t *testing.T) { } // Ensure a set query can be executed. -func TestExecutor_Execute_SetBit(t *testing.T) { +func TestExecutor_Execute_Set(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -276,7 +276,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Set(1, f=11)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -287,7 +287,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := hldr.Row("i", "f", 11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Set(1, f=11)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -296,6 +296,27 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } +// Ensure old PQL syntax doesn't break anything too badly. +func TestExecutor_Execute_OldSetBit(t *testing.T) { + return + // TODO + hldr := test.MustOpenHolder() + defer hldr.Close() + + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err != nil { + t.Fatal(err) + } else { + if !res[0].(bool) { + t.Fatalf("expected column changed") + } + } +} + // Ensure a SetValue() query can be executed. func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { @@ -391,16 +412,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other bitmaps and fields to test isolation. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 200, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(xxx, 10, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -427,15 +448,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=0, col=0) - SetBit(field=f, row=0, col=1) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(field=f, row=10, col=0) - SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=other, row=0, col=0) + Set(0, f=0) + Set(1, f=0) + Set(`+strconv.Itoa(SliceWidth)+`, f=0) + Set(`+strconv.Itoa(SliceWidth+2)+`, f=0) + Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0) + Set(0, f=10) + Set(`+strconv.Itoa(SliceWidth)+`, f=10) + Set(`+strconv.Itoa(SliceWidth)+`, f=20) + Set(0, other=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -444,7 +465,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: 0, Count: 5}, @@ -467,22 +488,22 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row="foo", col="a") - SetBit(field=f, row="foo", col="b") - SetBit(field=f, row="foo", col="c") - SetBit(field=f, row="foo", col="d") - SetBit(field=f, row="foo", col="e") - SetBit(field=f, row="bar", col="a") - SetBit(field=f, row="bar", col="b") - SetBit(field=f, row="baz", col="b") - SetBit(field=other, row="foo", col="a") + Set("a", f="foo") + Set("b", f="foo") + Set("c", f="foo") + Set("d", f="foo") + Set("e", f="foo") + Set("a", f="bar") + Set("b", f="bar") + Set("b", f="baz") + Set("a", other="foo") `), nil, nil); err != nil { t.Fatal(err) } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if diff := cmp.Diff(result, []interface{}{ []pilosa.Pair{ @@ -509,7 +530,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, @@ -543,7 +564,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -578,7 +599,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(other=100), n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -602,7 +623,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -625,7 +646,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -658,20 +679,20 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=x, row=0, col=0) - SetBit(field=x, row=0, col=3) - SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetBit(field=x, row=1, col=1) - SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + Set(0, x=0) + Set(3, x=0) + Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) + Set(1, x=1) + Set(`+strconv.Itoa(SliceWidth+2)+`, x=2) - SetValue(f=20, col=0) - SetValue(f=-5, col=1) - SetValue(f=-5, col=2) - SetValue(f=10, col=3) - SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`) - SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(col=0, f=20) + SetValue(col=1, f=-5) + SetValue(col=2, f=-5) + SetValue(col=3, f=10) + SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30) + SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40) + SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50) + SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60) `), nil, nil); err != nil { t.Fatal(err) } @@ -683,9 +704,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: -5, cnt: 2}, - {filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1}, - {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string @@ -709,9 +730,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: 60, cnt: 1}, - {filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1}, - {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string @@ -769,16 +790,16 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=x, row=0, col=0) - SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + Set(0, x=0) + Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) - SetValue(foo=20, col=0) - SetValue(bar=2000, col=0) - SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetValue(other=1000, col=0) + SetValue(col=0, foo=20) + SetValue(col=0, bar=2000) + SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40) + SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50) + SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=0, other=1000) `), nil, nil); err != nil { t.Fatal(err) } @@ -792,7 +813,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Row(x=0), field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -801,7 +822,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } // Ensure a range query can be executed. -func TestExecutor_Execute_BSIGroupRange(t *testing.T) { +func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -818,23 +839,24 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { } // Set columns. - if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00") - SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00") - SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00") - SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00") - SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00") - SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00") + cc := test.MustParse(` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) - SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00") - SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00") - SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00") - `), nil, nil); err != nil { + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + `) + if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) @@ -843,7 +865,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { } // Ensure a Range(bsiGroup) query can be executed. -func TestExecutor_Execute_Range(t *testing.T) { +func TestExecutor_Execute_BSIGroupRange(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -890,18 +912,18 @@ func TestExecutor_Execute_Range(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=0, col=0) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + Set(0, f=0) + Set(`+strconv.Itoa(SliceWidth+1)+`, f=0) - SetValue(foo=20, col=50) - SetValue(bar=2000, col=50) - SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetValue(other=1000, col=0) - SetValue(edge=100, col=0) - SetValue(edge=-100, col=1) + SetValue(col=50, foo=20) + SetValue(col=50, bar=2000) + SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10) + SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20) + SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=0, other=1000) + SetValue(col=0, edge=100) + SetValue(col=1, edge=-100) `), nil, nil); err != nil { t.Fatal(err) } @@ -969,7 +991,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(0 < other < 1000)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -978,7 +1000,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(-1 < other < 1000)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1042,7 +1064,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(field="f", row=10)` { + } else if query.String() != `Row(f=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -1065,7 +1087,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr.SetBit("i", "f", 10, SliceWidth+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected columns: %+v", columns) @@ -1100,7 +1122,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -1128,7 +1150,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, field="f", row=10)` { + } else if query.String() != `Set(_col=2, f=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1146,7 +1168,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil { + cc := test.MustParse("Set(2, f=10)") + if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { t.Fatal(err) } @@ -1180,7 +1203,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1200,7 +1223,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + cc := test.MustParse(`Set(2, f=10, 2016-12-11T10:09)`) + if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { t.Fatal(err) } @@ -1241,11 +1265,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // slices and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: - if query.String() != `TopN(field="f", n=3)` { + if query.String() != `TopN(_field="f", n=3)` { t.Fatalf("unexpected query(0): %s", query.String()) } case 1: - if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` { + if query.String() != `TopN(_field="f", ids=[0,10,30], n=3)` { t.Fatalf("unexpected query(1): %s", query.String()) } default: @@ -1269,7 +1293,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -1280,6 +1304,55 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { } } +// Ensure a remote query can set RowAttrs +func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { + c := pilosa.NewTestCluster(2) + + // Create secondary server and update second cluster node. + s := test.NewServer() + defer s.Close() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + c.Nodes[1].URI = *uri + + // Mock secondary server's executor to verify arguments and return a bitmap. + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "i" { + t.Fatalf("unexpected index: %s", index) + } else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` { + t.Fatalf("unexpected query: %s", query.String()) + } + + return []interface{}{}, nil + } + + // Create local executor data. + // The local node owns slice 1. + hldr := test.MustOpenHolder() + defer hldr.Close() + + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + f := hldr.Field("i", "f") + s.Handler.API.Holder = hldr.Holder + hldr.SetBit("i", "f", 10, SliceWidth+1) + + e := test.NewExecutor(hldr.Holder, c) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { + t.Fatal(err) + } else if m, err := f.RowAttrStore().Attrs(10); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(m, map[string]interface{}{"bat": true, "baz": int64(123)}) { + t.Fatalf("unexpected bitmap attr: %#v", m) + + } +} + // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() @@ -1287,13 +1360,13 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Set() Clear() Set() Set()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } // Ensure SetColumnAttrs doesn't save `field` as an attribute -func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { +func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -1304,11 +1377,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // SetColumnAttrs call should exclude the field attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("Set(10, f=1)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1321,11 +1394,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { } // SetColumnAttrs call should not break if field is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("Set(20, f=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(20, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index 799c8b695..28978d337 100644 --- a/fragment.go +++ b/fragment.go @@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) count++ } for j := 0; j < len(clear.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) count++ } diff --git a/http/client_test.go b/http/client_test.go index 185a3c378..bd39eca92 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -155,7 +155,7 @@ func TestClient_MultiNode(t *testing.T) { topN := 4 queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN), + Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } result, err := client[0].Query(context.Background(), "i", queryRequest) diff --git a/http/handler_test.go b/http/handler_test.go index c38a2f89a..fac2f959e 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -220,7 +220,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { + } else if query.String() != `Count(Row(id=100))` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -229,7 +229,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Row( id=100))"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -248,7 +248,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { + } else if query.String() != `Count(Row(id=100))` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -258,7 +258,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ - Query: "Count(Bitmap(id=100))", + Query: "Count(Row(id=100))", Slices: []uint64{0, 1}, }) if err != nil { @@ -286,7 +286,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Row(id=100)"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -295,7 +295,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { } func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() - test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) + test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Row(id=100)"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { @@ -317,7 +317,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Row( id=100))"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -338,7 +338,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Row(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { @@ -370,7 +370,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Row(id=100)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { @@ -403,7 +403,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Row(id=100)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { @@ -426,7 +426,7 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Row(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { @@ -475,7 +475,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", + Query: "Row(id=100)", ColumnAttrs: true, }) if err != nil { @@ -590,7 +590,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Row(id=100)`))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { @@ -653,7 +653,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near open (line 1 symbol 7 - line 1 symbol 8):\n\"(\"\n"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { // TODO not confident t.Fatalf("unexpected body: \n%s", body) } } diff --git a/server/cluster_test.go b/server/cluster_test.go index 3dfc6c4d9..73de82279 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, field="f", col=1) - SetBit(row=1, field="f", col=2400000) + Set(1, f=1) + Set(2400000, f=1) `); err != nil { t.Fatal(err) } @@ -259,8 +259,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, field="f", col=1) - SetBit(row=1, field="f", col=1300000) + Set(1, f=1) + Set(1300000, f=1) `); err != nil { t.Fatal(err) } @@ -311,8 +311,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, field="f", col=1) - SetBit(row=1, field="f", col=2400000) + Set(1, f=1) + Set(2400000, f=1) `); err != nil { t.Fatal(err) } @@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { diff --git a/server/config.go b/server/config.go index 1b74b177b..55da45768 100644 --- a/server/config.go +++ b/server/config.go @@ -47,7 +47,7 @@ type Config struct { Bind string `toml:"bind"` // MaxWritesPerRequest limits the number of mutating commands that can be in - // a single request to the server. This includes SetBit, ClearBit, + // a single request to the server. This includes Set, Clear, // SetRowAttrs & SetColumnAttrs. MaxWritesPerRequest int `toml:"max-writes-per-request"` diff --git a/server/server_test.go b/server/server_test.go index 971795156..58883286f 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -49,7 +49,7 @@ func TestMain_Set_Quick(t *testing.T) { t.Fatal(err) } - // Execute SetBit() commands. + // Execute Set() commands. for _, cmd := range cmds { if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) @@ -57,7 +57,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil { t.Fatal(err) } } @@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -132,36 +132,36 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set columns on different rows in different fields. - if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, neg=3)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(x, 1, x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(x, 2, x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(z, 2, x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(neg, 3, x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=2)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(neg=3)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=2)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -205,19 +205,19 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set columns on row. - if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `Set(101, x=1)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(100, foo="bar")`); err != nil { t.Fatal(err) } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -279,7 +279,7 @@ func TestMain_RecalculateHashes(t *testing.T) { data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { @@ -296,7 +296,7 @@ func TestMain_RecalculateHashes(t *testing.T) { // Run a TopN query on all nodes. The result should be the same as the target. for _, m := range cluster { - res, err := m.Query("i", "", `TopN(field="f")`) + res, err := m.Query("i", "", `TopN(f)`) if err != nil { t.Fatal(err) } diff --git a/stats_test.go b/stats_test.go index 6644786cf..304e042ba 100644 --- a/stats_test.go +++ b/stats_test.go @@ -127,8 +127,8 @@ func TestStatsCount_Bitmap(t *testing.T) { e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != "Bitmap" { - t.Errorf("Expected Bitmap, Results %s", name) + if name != "Row" { + t.Errorf("Expected Row, Results %s", name) } if tags[0] != "index:d" { @@ -138,7 +138,7 @@ func TestStatsCount_Bitmap(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Row(f=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -168,7 +168,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -199,7 +199,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(10, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { From ac66e51a1f5b64e2bb4653a588f4d744eee3d5f6 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 22 Jun 2018 08:36:03 -0500 Subject: [PATCH 24/33] Finish shell of OldPQL test --- executor_test.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/executor_test.go b/executor_test.go index 862715697..3022b8968 100644 --- a/executor_test.go +++ b/executor_test.go @@ -297,9 +297,7 @@ func TestExecutor_Execute_Set(t *testing.T) { } // Ensure old PQL syntax doesn't break anything too badly. -func TestExecutor_Execute_OldSetBit(t *testing.T) { - return - // TODO +func TestExecutor_Execute_OldPQL(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -308,12 +306,8 @@ func TestExecutor_Execute_OldSetBit(t *testing.T) { e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err != nil { - t.Fatal(err) - } else { - if !res[0].(bool) { - t.Fatalf("expected column changed") - } + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err == nil || err.Error() != "unknown call: SetBit" { + t.Fatal("Expected error: 'unknown call: SetBit'") } } From d1de586ea0b37bf482bf4f7bd18a1f7aba4d66df Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 09:08:08 -0500 Subject: [PATCH 25/33] remove all oldpql and fuzzer code --- pql/fuzz/README.txt | 8 - ...02ad499148a94f93101dbebda5111cd061137d28-1 | 1 - ...077a5923c7f6ff1b697b556611a3593e725d515f-1 | 1 - .../0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 | 1 - pql/fuzz/corpus/1 | 1 - pql/fuzz/corpus/10 | 1 - pql/fuzz/corpus/11 | 1 - .../11f674c766421132650bcbf8ccc265a013a3409f | 1 - pql/fuzz/corpus/12 | 1 - pql/fuzz/corpus/13 | 1 - .../131cfdaafbd04db9dd2aa37fb23a656500ed1333 | 1 - pql/fuzz/corpus/14 | 1 - pql/fuzz/corpus/15 | 1 - pql/fuzz/corpus/16 | 1 - pql/fuzz/corpus/17 | 1 - pql/fuzz/corpus/18 | 1 - pql/fuzz/corpus/19 | 1 - pql/fuzz/corpus/2 | 1 - pql/fuzz/corpus/20 | 1 - pql/fuzz/corpus/21 | 1 - pql/fuzz/corpus/22 | 1 - pql/fuzz/corpus/23 | 5 - pql/fuzz/corpus/24 | 1 - pql/fuzz/corpus/25 | 2 - pql/fuzz/corpus/26 | 1 - pql/fuzz/corpus/27 | 1 - .../2751bda09fe203e30e9d5f214f9425e2dface095 | 1 - pql/fuzz/corpus/28 | 1 - pql/fuzz/corpus/29 | 1 - pql/fuzz/corpus/3 | 1 - pql/fuzz/corpus/30 | 1 - pql/fuzz/corpus/31 | 1 - .../338717d7ceeb78f7b8b864547fcb87cd62334783 | 1 - .../33fae0740e470344699582c2c8c6f3825de66007 | 1 - .../374b9d8c1d285b57c3fe1f99b76472714cc2c69c | 2 - .../392027b3a650e05b0bc4ca185143138585702c5c | 1 - .../3c9cda1dd6ed289bdec524bb9f4995a9c175d656 | 1 - pql/fuzz/corpus/4 | 1 - .../452308054231977c3f6e551b72437500215019b5 | 1 - pql/fuzz/corpus/5 | 1 - .../57e5daa393a1de6405e0315abf57cf061bd5dc44 | 1 - .../597ed3d1cef06f73136921bdd89fc2916cdd287c | 1 - .../5e982cd2a4acb990e97675afabce72032c1d08ef | 1 - .../5f6b6920de296ca3a34d3ee14477a9d623d4efc2 | 1 - pql/fuzz/corpus/6 | 1 - .../6078ffa2c7287a2fdbb9bca63274a414fd7bc83d | 1 - ...6711a6c9ab125b4444c9c03b14e49f416f25180c-1 | 1 - pql/fuzz/corpus/7 | 1 - ...7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 | 1 - .../7282523da2bd624500932760375168ac6d95b08b | 1 - ...72fca46b66ab75b1b215d42c1f97a6a601e11383-1 | 1 - ...755ea2169f42a7facac54c6d4228abad4ffdb840-1 | 1 - .../75dcc3426aa51753b37f34acaab56815ae00af91 | 1 - .../7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 | 1 - ...7e03f5068158432ddc5faa0579f6cbfc09718884-1 | 1 - pql/fuzz/corpus/8 | 1 - .../80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 | 1 - pql/fuzz/corpus/9 | 1 - .../9456f79011b99928233a5c43c89d9bcabc788a9d | 1 - ...94ebe178c54a1ed5eced6ee363799261b18740c7-1 | 1 - .../9cbc01e0a28e963310a3e6b80eeb094a3de77c06 | 1 - .../9f974590bac2e9aa23f6e93128263403ca9d109f | 1 - .../a5ef2ba5c1423d9d03d8293be378b48af8dee79e | 1 - .../af209066ba9b25655fadd130ec30aa42f9a6c606 | 1 - .../b9258eb89acc5c62232f5e482449cc155a215125 | 1 - .../c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 | 1 - .../c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 | 1 - .../cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c | 1 - ...d20407c02c966d0cac76b72486e892158dce4ba7-1 | 1 - .../d4a4d133499f09ad2d91114f55ed7235e985f7fd | 1 - .../d5dd3b391afdce17c47a2644e536431e3b5b6825 | 1 - ...da588debce70733e48a0f1728ac248ce65e9e8c2-1 | 1 - .../e2c94a638563108995f18d0daadb9d2bd8a5f0c6 | 1 - .../e373d8c28776b2d1c8740807ffbe46cdd0260f98 | 1 - .../ee78db5d4e2231cadcf5957d169657ef4658c343 | 1 - .../f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d | 1 - .../f8f3c39e99db75ff5c8772a9871185a821a75f29 | 1 - .../ff41d50e5926d166b2adc0596339201274509856 | 1 - pql/internal/oldpql/ast.go | 272 --------------- pql/internal/oldpql/ast_test.go | 69 ---- pql/internal/oldpql/doc.go | 18 - pql/internal/oldpql/parser.go | 329 ------------------ pql/internal/oldpql/parser_test.go | 193 ---------- pql/internal/oldpql/scanner.go | 303 ---------------- pql/internal/oldpql/scanner_test.go | 74 ---- pql/internal/oldpql/token.go | 111 ------ pql/parser_fuzz.go | 115 ------ 87 files changed, 1575 deletions(-) delete mode 100644 pql/fuzz/README.txt delete mode 100644 pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 delete mode 100644 pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 delete mode 100644 pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 delete mode 100644 pql/fuzz/corpus/1 delete mode 100644 pql/fuzz/corpus/10 delete mode 100644 pql/fuzz/corpus/11 delete mode 100644 pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f delete mode 100644 pql/fuzz/corpus/12 delete mode 100644 pql/fuzz/corpus/13 delete mode 100644 pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 delete mode 100644 pql/fuzz/corpus/14 delete mode 100644 pql/fuzz/corpus/15 delete mode 100644 pql/fuzz/corpus/16 delete mode 100644 pql/fuzz/corpus/17 delete mode 100644 pql/fuzz/corpus/18 delete mode 100644 pql/fuzz/corpus/19 delete mode 100644 pql/fuzz/corpus/2 delete mode 100644 pql/fuzz/corpus/20 delete mode 100644 pql/fuzz/corpus/21 delete mode 100644 pql/fuzz/corpus/22 delete mode 100644 pql/fuzz/corpus/23 delete mode 100644 pql/fuzz/corpus/24 delete mode 100644 pql/fuzz/corpus/25 delete mode 100644 pql/fuzz/corpus/26 delete mode 100644 pql/fuzz/corpus/27 delete mode 100644 pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 delete mode 100644 pql/fuzz/corpus/28 delete mode 100644 pql/fuzz/corpus/29 delete mode 100644 pql/fuzz/corpus/3 delete mode 100644 pql/fuzz/corpus/30 delete mode 100644 pql/fuzz/corpus/31 delete mode 100644 pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 delete mode 100644 pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 delete mode 100644 pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c delete mode 100644 pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c delete mode 100644 pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 delete mode 100644 pql/fuzz/corpus/4 delete mode 100644 pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 delete mode 100644 pql/fuzz/corpus/5 delete mode 100644 pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 delete mode 100644 pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c delete mode 100644 pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef delete mode 100644 pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 delete mode 100644 pql/fuzz/corpus/6 delete mode 100644 pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d delete mode 100644 pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 delete mode 100644 pql/fuzz/corpus/7 delete mode 100644 pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 delete mode 100644 pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b delete mode 100644 pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 delete mode 100644 pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 delete mode 100644 pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 delete mode 100644 pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 delete mode 100644 pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 delete mode 100644 pql/fuzz/corpus/8 delete mode 100644 pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 delete mode 100644 pql/fuzz/corpus/9 delete mode 100644 pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d delete mode 100644 pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 delete mode 100644 pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 delete mode 100644 pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f delete mode 100644 pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e delete mode 100644 pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 delete mode 100644 pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 delete mode 100644 pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 delete mode 100644 pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 delete mode 100644 pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c delete mode 100644 pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 delete mode 100644 pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd delete mode 100644 pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 delete mode 100644 pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 delete mode 100644 pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 delete mode 100644 pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 delete mode 100644 pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 delete mode 100644 pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d delete mode 100644 pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 delete mode 100644 pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 delete mode 100644 pql/internal/oldpql/ast.go delete mode 100644 pql/internal/oldpql/ast_test.go delete mode 100644 pql/internal/oldpql/doc.go delete mode 100644 pql/internal/oldpql/parser.go delete mode 100644 pql/internal/oldpql/parser_test.go delete mode 100644 pql/internal/oldpql/scanner.go delete mode 100644 pql/internal/oldpql/scanner_test.go delete mode 100644 pql/internal/oldpql/token.go delete mode 100644 pql/parser_fuzz.go diff --git a/pql/fuzz/README.txt b/pql/fuzz/README.txt deleted file mode 100644 index e94c88830..000000000 --- a/pql/fuzz/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -See https://github.com/dvyukov/go-fuzz - - -Quickstart: - -go get -u github.com/dvyukov/go-fuzz/... -go-fuzz-build github.com/pilosa/pilosa/pql -go-fuzz -bin=./pql-fuzz.zip -workdir=$GOPATH/src/github.com/pilosa/pilosa/pql/fuzz diff --git a/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 b/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 deleted file mode 100644 index b38d50137..000000000 --- a/pql/fuzz/corpus/02ad499148a94f93101dbebda5111cd061137d28-1 +++ /dev/null @@ -1 +0,0 @@ -e(rT03 \ No newline at end of file diff --git a/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 b/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 deleted file mode 100644 index c4507e8e4..000000000 --- a/pql/fuzz/corpus/077a5923c7f6ff1b697b556611a3593e725d515f-1 +++ /dev/null @@ -1 +0,0 @@ -e(d=f2002-01-01T03:00 \ No newline at end of file diff --git a/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 b/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 deleted file mode 100644 index be461611e..000000000 --- a/pql/fuzz/corpus/0eb3a86490e4bb0a031ed5c33b2f3a8c90785746 +++ /dev/null @@ -1 +0,0 @@ -e(other!=-2) \ No newline at end of file diff --git a/pql/fuzz/corpus/1 b/pql/fuzz/corpus/1 deleted file mode 100644 index a8ccc9f85..000000000 --- a/pql/fuzz/corpus/1 +++ /dev/null @@ -1 +0,0 @@ -Bitmap() \ No newline at end of file diff --git a/pql/fuzz/corpus/10 b/pql/fuzz/corpus/10 deleted file mode 100644 index 21ff4c59c..000000000 --- a/pql/fuzz/corpus/10 +++ /dev/null @@ -1 +0,0 @@ -Bitmap(row=10, field=f) \ No newline at end of file diff --git a/pql/fuzz/corpus/11 b/pql/fuzz/corpus/11 deleted file mode 100644 index 7636ec48c..000000000 --- a/pql/fuzz/corpus/11 +++ /dev/null @@ -1 +0,0 @@ -Difference(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f b/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f deleted file mode 100644 index 425e9d1d3..000000000 --- a/pql/fuzz/corpus/11f674c766421132650bcbf8ccc265a013a3409f +++ /dev/null @@ -1 +0,0 @@ -Range(foo<0) \ No newline at end of file diff --git a/pql/fuzz/corpus/12 b/pql/fuzz/corpus/12 deleted file mode 100644 index 0d59771c6..000000000 --- a/pql/fuzz/corpus/12 +++ /dev/null @@ -1 +0,0 @@ -Difference() \ No newline at end of file diff --git a/pql/fuzz/corpus/13 b/pql/fuzz/corpus/13 deleted file mode 100644 index d5102d6fe..000000000 --- a/pql/fuzz/corpus/13 +++ /dev/null @@ -1 +0,0 @@ -Intersect(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 b/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 deleted file mode 100644 index d8c6af7ea..000000000 --- a/pql/fuzz/corpus/131cfdaafbd04db9dd2aa37fb23a656500ed1333 +++ /dev/null @@ -1 +0,0 @@ -SV(invalid_column_name=10,f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/14 b/pql/fuzz/corpus/14 deleted file mode 100644 index 08695e949..000000000 --- a/pql/fuzz/corpus/14 +++ /dev/null @@ -1 +0,0 @@ -Intersect() \ No newline at end of file diff --git a/pql/fuzz/corpus/15 b/pql/fuzz/corpus/15 deleted file mode 100644 index 2ade2207e..000000000 --- a/pql/fuzz/corpus/15 +++ /dev/null @@ -1 +0,0 @@ -Union(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/16 b/pql/fuzz/corpus/16 deleted file mode 100644 index c3b496bb4..000000000 --- a/pql/fuzz/corpus/16 +++ /dev/null @@ -1 +0,0 @@ -Union() \ No newline at end of file diff --git a/pql/fuzz/corpus/17 b/pql/fuzz/corpus/17 deleted file mode 100644 index 55062ba4c..000000000 --- a/pql/fuzz/corpus/17 +++ /dev/null @@ -1 +0,0 @@ -Xor(Bitmap(row=10), Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/18 b/pql/fuzz/corpus/18 deleted file mode 100644 index ea7190ed6..000000000 --- a/pql/fuzz/corpus/18 +++ /dev/null @@ -1 +0,0 @@ -Count(Bitmap(row=10, field=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/19 b/pql/fuzz/corpus/19 deleted file mode 100644 index bde6e75ac..000000000 --- a/pql/fuzz/corpus/19 +++ /dev/null @@ -1 +0,0 @@ -SetBit(row=11, field=f, col=1) \ No newline at end of file diff --git a/pql/fuzz/corpus/2 b/pql/fuzz/corpus/2 deleted file mode 100644 index 48ffdc060..000000000 --- a/pql/fuzz/corpus/2 +++ /dev/null @@ -1 +0,0 @@ -Union( Bitmap() , Count() ) \ No newline at end of file diff --git a/pql/fuzz/corpus/20 b/pql/fuzz/corpus/20 deleted file mode 100644 index 76679c897..000000000 --- a/pql/fuzz/corpus/20 +++ /dev/null @@ -1 +0,0 @@ -SetValue(col=10, f=25) \ No newline at end of file diff --git a/pql/fuzz/corpus/21 b/pql/fuzz/corpus/21 deleted file mode 100644 index 4ad8fba18..000000000 --- a/pql/fuzz/corpus/21 +++ /dev/null @@ -1 +0,0 @@ -SetValue(invalid_column_name=10, f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/22 b/pql/fuzz/corpus/22 deleted file mode 100644 index 123a4e7b2..000000000 --- a/pql/fuzz/corpus/22 +++ /dev/null @@ -1 +0,0 @@ -SetRowAttrs(row=10, field=f, baz=123, bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/23 b/pql/fuzz/corpus/23 deleted file mode 100644 index 31333c37b..000000000 --- a/pql/fuzz/corpus/23 +++ /dev/null @@ -1,5 +0,0 @@ - SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00") - SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00") - - SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00") - diff --git a/pql/fuzz/corpus/24 b/pql/fuzz/corpus/24 deleted file mode 100644 index 527b67ddc..000000000 --- a/pql/fuzz/corpus/24 +++ /dev/null @@ -1 +0,0 @@ -Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00") \ No newline at end of file diff --git a/pql/fuzz/corpus/25 b/pql/fuzz/corpus/25 deleted file mode 100644 index 32c0405c1..000000000 --- a/pql/fuzz/corpus/25 +++ /dev/null @@ -1,2 +0,0 @@ - -Range(foo == 20) diff --git a/pql/fuzz/corpus/26 b/pql/fuzz/corpus/26 deleted file mode 100644 index 4cad8028b..000000000 --- a/pql/fuzz/corpus/26 +++ /dev/null @@ -1 +0,0 @@ -Range(other != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/27 b/pql/fuzz/corpus/27 deleted file mode 100644 index c858f1930..000000000 --- a/pql/fuzz/corpus/27 +++ /dev/null @@ -1 +0,0 @@ -Range(foo != 20) \ No newline at end of file diff --git a/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 b/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 deleted file mode 100644 index f02ab7e3d..000000000 --- a/pql/fuzz/corpus/2751bda09fe203e30e9d5f214f9425e2dface095 +++ /dev/null @@ -1 +0,0 @@ -N(p(d=0,l=other), d=f,n=3) \ No newline at end of file diff --git a/pql/fuzz/corpus/28 b/pql/fuzz/corpus/28 deleted file mode 100644 index 212663384..000000000 --- a/pql/fuzz/corpus/28 +++ /dev/null @@ -1 +0,0 @@ -Range(other != -20) \ No newline at end of file diff --git a/pql/fuzz/corpus/29 b/pql/fuzz/corpus/29 deleted file mode 100644 index 3d2e5b82b..000000000 --- a/pql/fuzz/corpus/29 +++ /dev/null @@ -1 +0,0 @@ -Range(foo < 20) \ No newline at end of file diff --git a/pql/fuzz/corpus/3 b/pql/fuzz/corpus/3 deleted file mode 100644 index aef5a7a75..000000000 --- a/pql/fuzz/corpus/3 +++ /dev/null @@ -1 +0,0 @@ -Count( Bitmap( id=100)) \ No newline at end of file diff --git a/pql/fuzz/corpus/30 b/pql/fuzz/corpus/30 deleted file mode 100644 index 3e3f37870..000000000 --- a/pql/fuzz/corpus/30 +++ /dev/null @@ -1 +0,0 @@ -Range(foo <= 20) diff --git a/pql/fuzz/corpus/31 b/pql/fuzz/corpus/31 deleted file mode 100644 index 13b7e9347..000000000 --- a/pql/fuzz/corpus/31 +++ /dev/null @@ -1 +0,0 @@ -SetRowAttrs(row=10, field=f, baz=12.3, bat=.21, bak=-.27, zaz=-0.27 , q=0, zoo="0", do='0') diff --git a/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 b/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 deleted file mode 100644 index a03a91bd9..000000000 --- a/pql/fuzz/corpus/338717d7ceeb78f7b8b864547fcb87cd62334783 +++ /dev/null @@ -1 +0,0 @@ -t( p( )) \ No newline at end of file diff --git a/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 b/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 deleted file mode 100644 index 23e1bc1de..000000000 --- a/pql/fuzz/corpus/33fae0740e470344699582c2c8c6f3825de66007 +++ /dev/null @@ -1 +0,0 @@ -SetRowAttrs(row=10,field=f,baz=123,bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c b/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c deleted file mode 100644 index c65d51e92..000000000 --- a/pql/fuzz/corpus/374b9d8c1d285b57c3fe1f99b76472714cc2c69c +++ /dev/null @@ -1,2 +0,0 @@ - -e(o == 0) diff --git a/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c b/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c deleted file mode 100644 index 2b2542414..000000000 --- a/pql/fuzz/corpus/392027b3a650e05b0bc4ca185143138585702c5c +++ /dev/null @@ -1 +0,0 @@ -e(r!=-2) \ No newline at end of file diff --git a/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 b/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 deleted file mode 100644 index 822aa982c..000000000 --- a/pql/fuzz/corpus/3c9cda1dd6ed289bdec524bb9f4995a9c175d656 +++ /dev/null @@ -1 +0,0 @@ -MyCall( y=-12.25, o= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/4 b/pql/fuzz/corpus/4 deleted file mode 100644 index 22982532c..000000000 --- a/pql/fuzz/corpus/4 +++ /dev/null @@ -1 +0,0 @@ -MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null ) \ No newline at end of file diff --git a/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 b/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 deleted file mode 100644 index 65f6e27b8..000000000 --- a/pql/fuzz/corpus/452308054231977c3f6e551b72437500215019b5 +++ /dev/null @@ -1 +0,0 @@ -t(p(w=1,l=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/5 b/pql/fuzz/corpus/5 deleted file mode 100644 index 8075673eb..000000000 --- a/pql/fuzz/corpus/5 +++ /dev/null @@ -1 +0,0 @@ -MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9) \ No newline at end of file diff --git a/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 b/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 deleted file mode 100644 index d2042629e..000000000 --- a/pql/fuzz/corpus/57e5daa393a1de6405e0315abf57cf061bd5dc44 +++ /dev/null @@ -1 +0,0 @@ -e(row=1,field=f,start="1999-12-31T00:00",end="2002-01-01T03:00") \ No newline at end of file diff --git a/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c b/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c deleted file mode 100644 index b22ab81d2..000000000 --- a/pql/fuzz/corpus/597ed3d1cef06f73136921bdd89fc2916cdd287c +++ /dev/null @@ -1 +0,0 @@ -MyCall(ke=foo, x =5, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef b/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef deleted file mode 100644 index a7c359cfc..000000000 --- a/pql/fuzz/corpus/5e982cd2a4acb990e97675afabce72032c1d08ef +++ /dev/null @@ -1 +0,0 @@ -SetValue(invalid_column_name=10,f=100) \ No newline at end of file diff --git a/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 b/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 deleted file mode 100644 index e8d9b2dc2..000000000 --- a/pql/fuzz/corpus/5f6b6920de296ca3a34d3ee14477a9d623d4efc2 +++ /dev/null @@ -1 +0,0 @@ -Range(other!=null) \ No newline at end of file diff --git a/pql/fuzz/corpus/6 b/pql/fuzz/corpus/6 deleted file mode 100644 index 919a949ac..000000000 --- a/pql/fuzz/corpus/6 +++ /dev/null @@ -1 +0,0 @@ -MyCall( key=-12.25, foo= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d b/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d deleted file mode 100644 index 87168a0b5..000000000 --- a/pql/fuzz/corpus/6078ffa2c7287a2fdbb9bca63274a414fd7bc83d +++ /dev/null @@ -1 +0,0 @@ -tRowAttrs(row=1, field=f, baz=13,bat=true) \ No newline at end of file diff --git a/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 b/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 deleted file mode 100644 index 00c36326c..000000000 --- a/pql/fuzz/corpus/6711a6c9ab125b4444c9c03b14e49f416f25180c-1 +++ /dev/null @@ -1 +0,0 @@ -e(w=: \ No newline at end of file diff --git a/pql/fuzz/corpus/7 b/pql/fuzz/corpus/7 deleted file mode 100644 index b5a946470..000000000 --- a/pql/fuzz/corpus/7 +++ /dev/null @@ -1 +0,0 @@ -TopN(field="f", ids=[0,10,30]) \ No newline at end of file diff --git a/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 b/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 deleted file mode 100644 index 9170e620a..000000000 --- a/pql/fuzz/corpus/7209e30b65fb8aa3e31bb84f8f0a2e116af26184-1 +++ /dev/null @@ -1 +0,0 @@ -n(p() , C \ No newline at end of file diff --git a/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b b/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b deleted file mode 100644 index 966f30ab4..000000000 --- a/pql/fuzz/corpus/7282523da2bd624500932760375168ac6d95b08b +++ /dev/null @@ -1 +0,0 @@ -t(p(d=100)) \ No newline at end of file diff --git a/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 b/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 deleted file mode 100644 index 229ba77a9..000000000 --- a/pql/fuzz/corpus/72fca46b66ab75b1b215d42c1f97a6a601e11383-1 +++ /dev/null @@ -1 +0,0 @@ -U(B(,C \ No newline at end of file diff --git a/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 b/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 deleted file mode 100644 index 882c1dac8..000000000 --- a/pql/fuzz/corpus/755ea2169f42a7facac54c6d4228abad4ffdb840-1 +++ /dev/null @@ -1 +0,0 @@ -e(w=12002 \ No newline at end of file diff --git a/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 b/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 deleted file mode 100644 index 201e6ddaa..000000000 --- a/pql/fuzz/corpus/75dcc3426aa51753b37f34acaab56815ae00af91 +++ /dev/null @@ -1 +0,0 @@ -e(o <= 0) diff --git a/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 b/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 deleted file mode 100644 index 394c6b092..000000000 --- a/pql/fuzz/corpus/7cb88de80a430fd4c411e469ded68c8ec2f8fcd3 +++ /dev/null @@ -1 +0,0 @@ -t(p(w=0), p(w=1)) \ No newline at end of file diff --git a/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 b/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 deleted file mode 100644 index 0bf263b2e..000000000 --- a/pql/fuzz/corpus/7e03f5068158432ddc5faa0579f6cbfc09718884-1 +++ /dev/null @@ -1 +0,0 @@ -n(p(),C( \ No newline at end of file diff --git a/pql/fuzz/corpus/8 b/pql/fuzz/corpus/8 deleted file mode 100644 index 29ce05cfd..000000000 --- a/pql/fuzz/corpus/8 +++ /dev/null @@ -1 +0,0 @@ -TopN(Bitmap(id=100, field=other), field=f, n=3) \ No newline at end of file diff --git a/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 b/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 deleted file mode 100644 index dd54822fe..000000000 --- a/pql/fuzz/corpus/80899f5b5670badaff1d2c1820ad1d6c5ece8bf9 +++ /dev/null @@ -1 +0,0 @@ -C(y=12.25,o=13.167,r=2.,z=0.9) \ No newline at end of file diff --git a/pql/fuzz/corpus/9 b/pql/fuzz/corpus/9 deleted file mode 100644 index 870c1835c..000000000 --- a/pql/fuzz/corpus/9 +++ /dev/null @@ -1 +0,0 @@ -MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d b/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d deleted file mode 100644 index f7c988077..000000000 --- a/pql/fuzz/corpus/9456f79011b99928233a5c43c89d9bcabc788a9d +++ /dev/null @@ -1 +0,0 @@ -t(p( )) \ No newline at end of file diff --git a/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 b/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 deleted file mode 100644 index 10e6841d0..000000000 --- a/pql/fuzz/corpus/94ebe178c54a1ed5eced6ee363799261b18740c7-1 +++ /dev/null @@ -1 +0,0 @@ -e(invalid_column_name<0,f=0) \ No newline at end of file diff --git a/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 b/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 deleted file mode 100644 index 090a8a693..000000000 --- a/pql/fuzz/corpus/9cbc01e0a28e963310a3e6b80eeb094a3de77c06 +++ /dev/null @@ -1 +0,0 @@ -tV(f=5) \ No newline at end of file diff --git a/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f b/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f deleted file mode 100644 index 4b2b7b2bb..000000000 --- a/pql/fuzz/corpus/9f974590bac2e9aa23f6e93128263403ca9d109f +++ /dev/null @@ -1 +0,0 @@ -t(Ba(w=0,d=f)) \ No newline at end of file diff --git a/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e b/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e deleted file mode 100644 index dcf964fd5..000000000 --- a/pql/fuzz/corpus/a5ef2ba5c1423d9d03d8293be378b48af8dee79e +++ /dev/null @@ -1 +0,0 @@ -Range(o < 0) \ No newline at end of file diff --git a/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 b/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 deleted file mode 100644 index 74ac16027..000000000 --- a/pql/fuzz/corpus/af209066ba9b25655fadd130ec30aa42f9a6c606 +++ /dev/null @@ -1 +0,0 @@ -n() \ No newline at end of file diff --git a/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 b/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 deleted file mode 100644 index a9bb8167b..000000000 --- a/pql/fuzz/corpus/b9258eb89acc5c62232f5e482449cc155a215125 +++ /dev/null @@ -1 +0,0 @@ -Intersect(Bitmap(row=10),Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 b/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 deleted file mode 100644 index 420a255a2..000000000 --- a/pql/fuzz/corpus/c0d2d3099c28dc8b6e838f1d95a5fd314d6caff5 +++ /dev/null @@ -1 +0,0 @@ -Cl( k=-12.25, f= -13) \ No newline at end of file diff --git a/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 b/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 deleted file mode 100644 index 990d4e833..000000000 --- a/pql/fuzz/corpus/c7b63c3044a6dd7981d72573a970e9f1ac42f5b4 +++ /dev/null @@ -1 +0,0 @@ -e(o<0) \ No newline at end of file diff --git a/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c b/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c deleted file mode 100644 index 6e6fab1ca..000000000 --- a/pql/fuzz/corpus/cd414eb16b1530ef1b9aa16a3b60abc932c4ca6c +++ /dev/null @@ -1 +0,0 @@ -Difference(Bitmap(row=10),Bitmap(row=11)) \ No newline at end of file diff --git a/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 b/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 deleted file mode 100644 index 789a07fc4..000000000 --- a/pql/fuzz/corpus/d20407c02c966d0cac76b72486e892158dce4ba7-1 +++ /dev/null @@ -1 +0,0 @@ -j(w=10375035658,t=R) \ No newline at end of file diff --git a/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd b/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd deleted file mode 100644 index 95edd6a6f..000000000 --- a/pql/fuzz/corpus/d4a4d133499f09ad2d91114f55ed7235e985f7fd +++ /dev/null @@ -1 +0,0 @@ -Range(other!=l) \ No newline at end of file diff --git a/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 b/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 deleted file mode 100644 index e0dfe5315..000000000 --- a/pql/fuzz/corpus/d5dd3b391afdce17c47a2644e536431e3b5b6825 +++ /dev/null @@ -1 +0,0 @@ -e(o<=0) diff --git a/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 b/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 deleted file mode 100644 index 3c37e86b7..000000000 --- a/pql/fuzz/corpus/da588debce70733e48a0f1728ac248ce65e9e8c2-1 +++ /dev/null @@ -1 +0,0 @@ -e(w=T \ No newline at end of file diff --git a/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 b/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 deleted file mode 100644 index cdc7903a6..000000000 --- a/pql/fuzz/corpus/e2c94a638563108995f18d0daadb9d2bd8a5f0c6 +++ /dev/null @@ -1 +0,0 @@ -l(key=oo, x == 12.25, y >= 100, z >< [4,8], m != null) \ No newline at end of file diff --git a/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 b/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 deleted file mode 100644 index a692598ed..000000000 --- a/pql/fuzz/corpus/e373d8c28776b2d1c8740807ffbe46cdd0260f98 +++ /dev/null @@ -1 +0,0 @@ -SB(ow=1, f=f, c=1) \ No newline at end of file diff --git a/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 b/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 deleted file mode 100644 index beb610cd7..000000000 --- a/pql/fuzz/corpus/ee78db5d4e2231cadcf5957d169657ef4658c343 +++ /dev/null @@ -1 +0,0 @@ -Setalue(invalidcolumnnamf=0) \ No newline at end of file diff --git a/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d b/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d deleted file mode 100644 index 5cac44705..000000000 --- a/pql/fuzz/corpus/f1c3a3daa6e74cf6ba1b8a494a21a34aa2aab41d +++ /dev/null @@ -1 +0,0 @@ -N(field="f",ids=[0,10,30]) \ No newline at end of file diff --git a/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 b/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 deleted file mode 100644 index b13f3aff7..000000000 --- a/pql/fuzz/corpus/f8f3c39e99db75ff5c8772a9871185a821a75f29 +++ /dev/null @@ -1 +0,0 @@ -U( B() , C() ) \ No newline at end of file diff --git a/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 b/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 deleted file mode 100644 index adca39514..000000000 --- a/pql/fuzz/corpus/ff41d50e5926d166b2adc0596339201274509856 +++ /dev/null @@ -1 +0,0 @@ -Range(r!=null) \ No newline at end of file diff --git a/pql/internal/oldpql/ast.go b/pql/internal/oldpql/ast.go deleted file mode 100644 index bee778905..000000000 --- a/pql/internal/oldpql/ast.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql - -import ( - "bytes" - "fmt" - "sort" - "strconv" - "strings" - "time" -) - -// Query represents a PQL query. -type Query struct { - Calls []*Call -} - -// WriteCallN returns the number of mutating calls. -func (q *Query) WriteCallN() int { - var n int - for _, call := range q.Calls { - switch call.Name { - case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs": - n++ - } - } - return n -} - -// String returns a string representation of the query. -func (q *Query) String() string { - a := make([]string, len(q.Calls)) - for i, call := range q.Calls { - a[i] = call.String() - } - return strings.Join(a, "\n") -} - -// Call represents a function call in the AST. -type Call struct { - Name string - Args map[string]interface{} - Children []*Call -} - -// UintArg is for reading the value at key from call.Args as a uint64. If the -// key is not in Call.Args, the value of the returned bool will be false, and -// the error will be nil. The value is assumed to be a uint64 or an int64 and -// then cast to a uint64. An error is returned if the value is not an int64 or -// uint64. -func (c *Call) UintArg(key string) (uint64, bool, error) { - val, ok := c.Args[key] - if !ok { - return 0, false, nil - } - switch tval := val.(type) { - case int64: - return uint64(tval), true, nil - case uint64: - return tval, true, nil - default: - return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval) - } -} - -// UintSliceArg reads the value at key from call.Args as a slice of uint64. If -// the key is not in Call.Args, the value of the returned bool will be false, -// and the error will be nil. If the value is a slice of int64 it will convert -// it to []uint64. Otherwise, if it is not a []uint64 it will return an error. -func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { - val, ok := c.Args[key] - if !ok { - return nil, false, nil - } - - switch tval := val.(type) { - case []uint64: - return tval, true, nil - case []int64: - ret := make([]uint64, len(tval)) - for i, v := range tval { - ret[i] = uint64(v) - } - return ret, true, nil - default: - return nil, true, fmt.Errorf("unexpected type %T in UintSliceArg, val %v", tval, tval) - } -} - -// Keys returns a list of argument keys in sorted order. -func (c *Call) Keys() []string { - a := make([]string, 0, len(c.Args)) - for k := range c.Args { - a = append(a, k) - } - sort.Strings(a) - return a -} - -// Clone returns a copy of c. -func (c *Call) Clone() *Call { - if c == nil { - return nil - } - - other := &Call{ - Name: c.Name, - Args: CopyArgs(c.Args), - } - if c.Children != nil { - other.Children = make([]*Call, len(c.Children)) - for i := range c.Children { - other.Children[i] = c.Children[i].Clone() - } - } - return other -} - -// String returns the string representation of the call. -func (c *Call) String() string { - var buf bytes.Buffer - - // Write name. - if c.Name != "" { - buf.WriteString(c.Name) - } else { - buf.WriteString("!UNNAMED") - } - - // Write opening. - buf.WriteByte('(') - - // Write child list. - for i, child := range c.Children { - if i > 0 { - buf.WriteString(", ") - } - buf.WriteString(child.String()) - } - - // Separate children and args, if necessary. - if len(c.Children) > 0 && len(c.Args) > 0 { - buf.WriteString(", ") - } - - // Write arguments in key order. - for i, key := range c.Keys() { - if i > 0 { - buf.WriteString(", ") - } - // If the Arg value is a Condition, then don't include - // the equal sign in the string representation. - switch v := c.Args[key].(type) { - case *Condition: - fmt.Fprintf(&buf, "%v %s", key, v.String()) - default: - fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v)) - } - } - - // Write closing. - buf.WriteByte(')') - - return buf.String() -} - -// HasConditionArg returns true if any arg is a conditional. -func (c *Call) HasConditionArg() bool { - for _, v := range c.Args { - if _, ok := v.(*Condition); ok { - return true - } - } - return false -} - -// Condition represents an operation & value. -// When used in an argument map it represents a binary expression. -type Condition struct { - Op Token - Value interface{} -} - -// String returns the string representation of the condition. -func (cond *Condition) String() string { - return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value)) -} - -// IntSliceValue reads cond.Value as a slice of uint64. -// If the value is a slice of uint64 it will convert -// it to []int64. Otherwise, if it is not a []int64 it will return an error. -func (cond *Condition) IntSliceValue() ([]int64, error) { - val := cond.Value - - switch tval := val.(type) { - case []interface{}: - ret := make([]int64, len(tval)) - for i, v := range tval { - switch tv := v.(type) { - case int64: - ret[i] = tv - case uint64: - ret[i] = int64(tv) - default: - return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv) - } - } - return ret, nil - default: - return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval) - } -} - -func FormatValue(v interface{}) string { - switch v := v.(type) { - case string: - return fmt.Sprintf("%q", v) - case []interface{}: - return fmt.Sprintf("%s", joinInterfaceSlice(v)) - case []uint64: - return fmt.Sprintf("%s", joinUint64Slice(v)) - case time.Time: - return fmt.Sprintf("\"%s\"", v.Format(TimeFormat)) - case *Condition: - return v.String() - default: - return fmt.Sprintf("%v", v) - } -} - -// CopyArgs returns a copy of m. -func CopyArgs(m map[string]interface{}) map[string]interface{} { - other := make(map[string]interface{}, len(m)) - for k, v := range m { - other[k] = v - } - return other -} - -func joinInterfaceSlice(a []interface{}) string { - other := make([]string, len(a)) - for i := range a { - switch v := a[i].(type) { - case string: - other[i] = fmt.Sprintf("%q", v) - default: - other[i] = fmt.Sprintf("%v", v) - } - } - return "[" + strings.Join(other, ",") + "]" -} - -func joinUint64Slice(a []uint64) string { - other := make([]string, len(a)) - for i := range a { - other[i] = strconv.FormatUint(a[i], 10) - } - return "[" + strings.Join(other, ",") + "]" -} diff --git a/pql/internal/oldpql/ast_test.go b/pql/internal/oldpql/ast_test.go deleted file mode 100644 index 1b7c9eba0..000000000 --- a/pql/internal/oldpql/ast_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql_test - -import ( - "reflect" - "testing" - - pql "github.com/pilosa/pilosa/pql/internal/oldpql" -) - -// Ensure call can be converted into a string. -func TestCall_String(t *testing.T) { - t.Run("Empty", func(t *testing.T) { - c := &pql.Call{Name: "Bitmap"} - if s := c.String(); s != `Bitmap()` { - t.Fatalf("unexpected string: %s", s) - } - }) - t.Run("With Args", func(t *testing.T) { - c := &pql.Call{ - Name: "Range", - Args: map[string]interface{}{ - "other": "f", - "field0": &pql.Condition{Op: pql.GTE, Value: 10}, - }, - } - if s := c.String(); s != `Range(field0 >= 10, other="f")` { - t.Fatalf("unexpected string: %s", s) - } - }) -} - -// Ensure condition can handle values for BETWEEN operator. -func TestCondition_Value(t *testing.T) { - t.Run("Between Values", func(t *testing.T) { - for _, tt := range []struct { - val []interface{} - exp []int64 - }{ - {[]interface{}{int64(4), int64(8)}, []int64{4, 8}}, - {[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}}, - {[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}}, - } { - c := &pql.Condition{ - Op: pql.BETWEEN, - Value: tt.val, - } - v, err := c.IntSliceValue() - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(v, tt.exp) { - t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v) - } - } - }) -} diff --git a/pql/internal/oldpql/doc.go b/pql/internal/oldpql/doc.go deleted file mode 100644 index 3e5bd4876..000000000 --- a/pql/internal/oldpql/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -package oldpql defines the Pilosa Query Language. -*/ -package oldpql diff --git a/pql/internal/oldpql/parser.go b/pql/internal/oldpql/parser.go deleted file mode 100644 index d54033ee7..000000000 --- a/pql/internal/oldpql/parser.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql - -import ( - "fmt" - "io" - "strconv" - "strings" -) - -// TimeFormat is the go-style time format used to parse string dates. -const TimeFormat = "2006-01-02T15:04" - -// Parser represents a parser for the PQL language. -type Parser struct { - scanner *bufScanner -} - -// NewParser returns a new instance of Parser. -func NewParser(r io.Reader) *Parser { - return &Parser{ - scanner: newBufScanner(r), - } -} - -// ParseString parses s into a query. -func ParseString(s string) (*Query, error) { - return NewParser(strings.NewReader(s)).Parse() -} - -// Parse parses the next node in the query. -func (p *Parser) Parse() (*Query, error) { - q := &Query{} - for { - call, err := p.parseCall() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - q.Calls = append(q.Calls, call) - } - - // Require at least one call. - if len(q.Calls) == 0 { - return nil, io.ErrUnexpectedEOF - } - - return q, nil -} - -// parseCall parses the next function call. -func (p *Parser) parseCall() (*Call, error) { - var c Call - - // Read call name. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == EOF { - return nil, io.EOF - } else if tok != IDENT { - return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} - } - c.Name = lit - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse children first. - children, err := p.parseChildren() - if err != nil { - return nil, err - } - c.Children = children - - // If next token is a closing paren then exit. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - return &c, nil - } else if tok == IDENT { - p.unscan(1) - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) - } - - // Parse key/value arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - c.Args = args - - // Scan closing parenthesis. - if err := p.expect(RPAREN); err != nil { - return nil, err - } - - return &c, nil -} - -// parseChildren parses call children. -func (p *Parser) parseChildren() ([]*Call, error) { - var offset int - var children []*Call - for { - // Ensure next two tokens are IDENT+LPAREN. - if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { - p.unscanIgnoreWhitespace(1 + offset) - return children, nil - } - if tok, _, _ := p.scan(); tok != LPAREN { - p.unscanIgnoreWhitespace(2 + offset) - return children, nil - } - - // Push tokens back on scanner and parse as a call. - p.unscan(2) - child, err := p.parseCall() - if err != nil { - return nil, err - } - children = append(children, child) - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return children, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - - // Make sure comma is unscanned. - offset = 1 - } -} - -// parseArgs parses key/value arguments. -func (p *Parser) parseArgs() (map[string]interface{}, error) { - args := make(map[string]interface{}) - for { - // Parse key. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != IDENT { - return nil, parseErrorf(pos, "expected argument key, found %q", lit) - } - key := lit - - // Expect '=' or a comparison next. - var op Token - switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { - case ASSIGN: - case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: - op = tok - default: - return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) - } - - // Parse value. - var value interface{} - tok, pos, lit = p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - value = true - } else if lit == "false" { - value = false - } else if lit == "null" { - value = nil - } else { - value = lit - } - case STRING: - value = lit - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - value = v - case FLOAT: - v, err := strconv.ParseFloat(lit, 64) - if err != nil { - return nil, err - } - value = v - case LBRACK: - v, err := p.parseList() - if err != nil { - return nil, err - } - value = v - default: - return nil, parseErrorf(pos, "invalid argument value: %q", lit) - } - - // Ensure key doesn't already exist. - if _, ok := args[key]; ok { - return nil, parseErrorf(pos, "argument key already used: %s", key) - } - - // If op is specified then create a condition. - if op != 0 { - value = &Condition{Op: op, Value: value} - } - - // Add key/value pair to arguments. - args[key] = value - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - } -} - -// parseList parses a list of primitives. This is used by the TopN() filters. -func (p *Parser) parseList() ([]interface{}, error) { - var values []interface{} - for { - // Read next value. - tok, pos, lit := p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - values = append(values, true) - } else if lit == "false" { - values = append(values, false) - } else { - values = append(values, lit) - } - case STRING: - values = append(values, lit) - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - values = append(values, v) - default: - return nil, parseErrorf(pos, "invalid list value: %q", lit) - } - - // Expect a comma or closing bracket next. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { - break - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, found %q", lit) - } - } - return values, nil -} - -// scan returns the next token from the scanner. -func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } - -// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. -func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { - tok, pos, lit = p.scan() - if tok == WS { - tok, pos, lit = p.scan() - } - return -} - -// unscan returns the last n tokens back to the scanner. -func (p *Parser) unscan(n int) { - for i := 0; i < n; i++ { - p.scanner.unscan() - } -} - -// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. -func (p *Parser) unscanIgnoreWhitespace(n int) { - for i := 0; i < n; { - p.scanner.unscan() - if tok, _, _ := p.scanner.curr(); tok != WS { - i++ - } - } -} - -// expect returns an error if the next token is not exp. -func (p *Parser) expect(exp Token) error { - if tok, pos, lit := p.scan(); tok != exp { - return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) - } - return nil -} - -// pos returns the current position. -func (p *Parser) pos() Pos { return p.scanner.pos() } - -// ParseError represents an error that occurred while parsing a PQL query. -type ParseError struct { - Message string - Pos Pos -} - -// Error returns a string representation of e. -func (e *ParseError) Error() string { - return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) -} - -// parseErrorf returns a formatted parse error. -func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { - return &ParseError{ - Message: fmt.Sprintf(format, args...), - Pos: pos, - } -} diff --git a/pql/internal/oldpql/parser_test.go b/pql/internal/oldpql/parser_test.go deleted file mode 100644 index 31613429c..000000000 --- a/pql/internal/oldpql/parser_test.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql_test - -import ( - "reflect" - "testing" - - pql "github.com/pilosa/pilosa/pql/internal/oldpql" - _ "github.com/pilosa/pilosa/test" -) - -// Ensure the parser can parse PQL. -func TestParser_Parse(t *testing.T) { - // Parse with no children or arguments. - t.Run("Empty", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap()`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "Bitmap", - }, - ) { - t.Fatalf("unexpected call: %s", q.Calls[0]) - } - }) - - // Parse with only children. - t.Run("ChildrenOnly", func(t *testing.T) { - q, err := pql.ParseString(`Union( Bitmap() , Count() )`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "Union", - Children: []*pql.Call{ - &pql.Call{Name: "Bitmap"}, - &pql.Call{Name: "Count"}, - }, - }, - ) { - t.Fatalf("unexpected call: %s", q.Calls[0]) - } - }) - - // Parse a single child with a single argument. - t.Run("ChildWithArgument", func(t *testing.T) { - q, err := pql.ParseString(`Count( Bitmap( id=100))`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "Count", - Children: []*pql.Call{ - {Name: "Bitmap", Args: map[string]interface{}{"id": int64(100)}}, - }, - }, - ) { - t.Fatalf("unexpected call: %s", q.Calls[0]) - } - }) - - // Parse with only arguments. - t.Run("ArgumentsOnly", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "MyCall", - Args: map[string]interface{}{ - "key": "value", - "foo": "bar", - "age": int64(12), - "bool0": true, - "bool1": false, - "x": nil, - }, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) - - // Parse with float arguments. - t.Run("WithFloatArgs", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "MyCall", - Args: map[string]interface{}{ - "key": 12.25, - "foo": 13.167, - "bar": 2., - "baz": 0.9, - }, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) - - // Parse with float arguments. - t.Run("WithNegativeArgs", func(t *testing.T) { - q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "MyCall", - Args: map[string]interface{}{ - "key": -12.25, - "foo": int64(-13), - }, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) - - // Parse with both child calls and arguments. - t.Run("ChildrenAndArguments", func(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "TopN", - Children: []*pql.Call{{ - Name: "Bitmap", - Args: map[string]interface{}{"id": int64(100), "field": "other"}, - }}, - Args: map[string]interface{}{"n": int64(3), "field": "f"}, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) - - // Parse a list argument. - t.Run("ListArgument", func(t *testing.T) { - q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "TopN", - Args: map[string]interface{}{ - "field": "f", - "ids": []interface{}{int64(0), int64(10), int64(30)}, - }, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) - - // Parse with condition arguments. - t.Run("WithCondition", func(t *testing.T) { - q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "MyCall", - Args: map[string]interface{}{ - "key": "foo", - "x": &pql.Condition{Op: pql.EQ, Value: 12.25}, - "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, - "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, - "m": &pql.Condition{Op: pql.NEQ, Value: nil}, - }, - }, - ) { - t.Fatalf("unexpected call: %#v", q.Calls[0]) - } - }) -} diff --git a/pql/internal/oldpql/scanner.go b/pql/internal/oldpql/scanner.go deleted file mode 100644 index 27e321a0c..000000000 --- a/pql/internal/oldpql/scanner.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql - -import ( - "bufio" - "bytes" - "io" - "unicode" -) - -// Scanner represents a PQL lexical scanner. -type Scanner struct { - r io.RuneScanner - pos Pos -} - -// NewScanner returns a new instance of Scanner. -func NewScanner(r io.Reader) *Scanner { - return &Scanner{r: bufio.NewReader(r)} -} - -// Scan returns the next token and position from the underlying reader. -func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { - pos = s.pos - - // Read next code point. - ch := s.read() - - // If we see whitespace then consume all contiguous whitespace. - // If we see a letter, or certain acceptable special characters, then consume - // as an ident or reserved word. If we see quotes, then scan as string. - if isWhitespace(ch) { - s.unread() - return s.scanWhitespace() - } else if isIdentFirstChar(ch) { - s.unread() - return s.scanIdent() - } else if isDigit(ch) || ch == '-' { - s.unread() - return s.scanNumber() - } else if ch == '"' || ch == '\'' { - s.unread() - return s.scanString() - } - - // Otherwise parse individual characters. - switch ch { - case eof: - return EOF, pos, "" - case '=': - if next := s.read(); next == '=' { - return EQ, pos, "==" - } - s.unread() - return ASSIGN, pos, string(ch) - case '!': - if next := s.read(); next == '=' { - return NEQ, pos, "!=" - } - s.unread() - return ILLEGAL, pos, string(ch) - case '<': - if next := s.read(); next == '=' { - return LTE, pos, "<=" - } - s.unread() - return LT, pos, string(ch) - case '>': - next := s.read() - if next == '=' { - return GTE, pos, ">=" - } else if next == '<' { - return BETWEEN, pos, "><" - } - s.unread() - return GT, pos, string(ch) - case ',': - return COMMA, pos, string(ch) - case '(': - return LPAREN, pos, string(ch) - case ')': - return RPAREN, pos, string(ch) - case '[': - return LBRACK, pos, string(ch) - case ']': - return RBRACK, pos, string(ch) - default: - return ILLEGAL, pos, string(ch) - } -} - -// read returns the next code point from the underlying reader and updates the pos. -func (s *Scanner) read() rune { - // Read next rune from underlying reader. - ch, _, err := s.r.ReadRune() - if err != nil { - return eof - } - - // Update position information. - if ch == '\n' { - s.pos.Line++ - s.pos.Char = 0 - } else { - s.pos.Char++ - } - - return ch -} - -// unread pushes the previously read rune back onto the reader. -func (s *Scanner) unread() { - if s.pos.Char == 0 { - s.pos.Line-- - } else { - s.pos.Char-- - } - - s.r.UnreadRune() -} - -// scanWhitespace consumes the current rune and all contiguous whitespace. -func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isWhitespace(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - - return WS, pos, buf.String() -} - -func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isIdentChar(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - lit = buf.String() - - // If the literal matches a keyword then return that keyword. - if tok = Lookup(lit); tok != IDENT { - return tok, pos, lit - } - - return IDENT, pos, lit -} - -// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. -func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { - pos = s.pos - tok = INTEGER - - var buf bytes.Buffer - var seenDot bool - first := true - for { - ch := s.read() - if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { - s.unread() - break - } - if ch == '.' { - seenDot = true - tok = FLOAT - } - buf.WriteRune(ch) - first = false - } - return tok, pos, buf.String() -} - -// scanString consumes a single-quoted or double-quoted string. -func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { - pos = s.pos - - // This must be either a single- or double-quote. - ending := s.read() - - var buf bytes.Buffer - for { - ch := s.read() - if ch == ending { - break - } else if ch == '\n' || ch == eof { - return BADSTRING, pos, buf.String() - } else if ch == '\\' { - next := s.read() - if next == 'n' { - buf.WriteRune('\n') - } else if next == '\\' { - buf.WriteRune('\\') - } else if next == '"' { - buf.WriteRune('"') - } else if next == '\'' { - buf.WriteRune('\'') - } else { - return BADSTRING, pos, buf.String() - } - } else { - buf.WriteRune(ch) - } - } - - return STRING, pos, buf.String() -} - -// bufScanner represents a wrapper for scanner to add a buffer. -// It provides a fixed-length circular buffer that can be unread. -type bufScanner struct { - s *Scanner - i int // buffer index - n int // buffer size - buf [8]struct { - tok Token - pos Pos - lit string - } -} - -// newBufScanner returns a new buffered scanner for a reader. -func newBufScanner(r io.Reader) *bufScanner { - return &bufScanner{s: NewScanner(r)} -} - -// Scan reads the next token from the scanner. -func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { - // If we have unread tokens then read them off the buffer first. - if s.n > 0 { - s.n-- - return s.curr() - } - - // Move buffer position forward and save the token. - s.i = (s.i + 1) % len(s.buf) - buf := &s.buf[s.i] - buf.tok, buf.pos, buf.lit = s.s.Scan() - - return s.curr() -} - -// unscan pushes the previously token back onto the buffer. -func (s *bufScanner) unscan() { s.n++ } - -// curr returns the last read token. -func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { - buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] - return buf.tok, buf.pos, buf.lit -} - -// pos returns the current position. -func (s *bufScanner) pos() Pos { - _, pos, _ := s.curr() - return pos -} - -// isWhitespace returns true if the rune a Unicode space character. -func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } - -// isLetter returns true if the rune is a letter. -func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } - -// isDigit returns true if the rune is a digit. -func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } - -// isIdentChar returns true if the rune can be used in an unquoted identifier. -func isIdentChar(ch rune) bool { - return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' -} - -// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. -func isIdentFirstChar(ch rune) bool { return isLetter(ch) } - -const eof = rune(0) diff --git a/pql/internal/oldpql/scanner_test.go b/pql/internal/oldpql/scanner_test.go deleted file mode 100644 index 3a1f462e2..000000000 --- a/pql/internal/oldpql/scanner_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql_test - -import ( - "strings" - "testing" - - pql "github.com/pilosa/pilosa/pql/internal/oldpql" -) - -func TestScanner_Scan(t *testing.T) { - var tests = []struct { - name string - s string - tok pql.Token - lit string - pos pql.Pos - }{ - // Special tokens (EOF, ILLEGAL, WS) - {name: "EOF", s: ``, tok: pql.EOF}, - {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, - {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, - {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, - {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, - - {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, - {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, - {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, - {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, - {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, - {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, - {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, - {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, - {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, - {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, - {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, - {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, - {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, - - {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, - {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, - {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, - - {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, - {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := pql.NewScanner(strings.NewReader(tt.s)) - tok, pos, lit := s.Scan() - if tt.tok != tok { - t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) - } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { - t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) - } else if tt.lit != lit { - t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) - } - }) - } -} diff --git a/pql/internal/oldpql/token.go b/pql/internal/oldpql/token.go deleted file mode 100644 index 4da3b8505..000000000 --- a/pql/internal/oldpql/token.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oldpql - -import "strings" - -// Token is a lexical token of the PQL language. -type Token int - -const ( - // Special tokens - ILLEGAL Token = iota - EOF - WS - - literal_beg - IDENT // main - STRING // "foo" - BADSTRING // bad escape or unclosed string - INTEGER // 12345 - FLOAT // 100.2 - literal_end - - keyword_beg - ALL - keyword_end - - ASSIGN // = - EQ // == - NEQ // != - LT // < - LTE // <= - GT // > - GTE // >= - BETWEEN // >< - COMMA // , - LPAREN // ( - RPAREN // ) - LBRACK // ( - RBRACK // ) -) - -var tokens = [...]string{ - ILLEGAL: "ILLEGAL", - EOF: "EOF", - WS: "WS", - - IDENT: "IDENT", - INTEGER: "INTEGER", - FLOAT: "FLOAT", - - ALL: "ALL", - - ASSIGN: "=", - EQ: "==", - NEQ: "!=", - LT: "<", - LTE: "<=", - GT: ">", - GTE: ">=", - BETWEEN: "><", - COMMA: ",", - LPAREN: "(", - RPAREN: ")", - LBRACK: "(", - RBRACK: ")", -} - -var keywords map[string]Token - -func init() { - keywords = make(map[string]Token) - for tok := keyword_beg + 1; tok < keyword_end; tok++ { - keywords[strings.ToLower(tokens[tok])] = tok - } -} - -// String returns the string representation of the token. -func (tok Token) String() string { - if tok >= 0 && tok < Token(len(tokens)) { - return tokens[tok] - } - return "" -} - -// Lookup returns the token associated with a given string. -func Lookup(ident string) Token { - if tok, ok := keywords[strings.ToLower(ident)]; ok { - return tok - } - return IDENT -} - -// Pos specifies the line and character position of a token. -// The Char and Line are both zero-based indexes. -type Pos struct { - Line int - Char int -} diff --git a/pql/parser_fuzz.go b/pql/parser_fuzz.go deleted file mode 100644 index ffcea44ab..000000000 --- a/pql/parser_fuzz.go +++ /dev/null @@ -1,115 +0,0 @@ -// +build gofuzz - -package pql - -import ( - "bytes" - "fmt" - "reflect" - - "github.com/pilosa/pilosa/pql/internal/oldpql" - "github.com/pkg/errors" -) - -func Fuzz(data []byte) int { - p1 := NewParser(bytes.NewReader(data)) - q1, err1 := p1.Parse() - p2 := oldpql.NewParser(bytes.NewReader(data)) - q2, err2 := p2.Parse() - if err1 != nil && err2 != nil { - return 0 // both error - this is fine - } - if err1 != nil || err2 != nil { - // error in one but not both - need to know this - panic(fmt.Sprintf("Query: '%s' errored one but not both.\n%v\n%v\n", data, err1, err2)) - } - - // if parsers got different results - if err := queriesEqual(q1, q2); err != nil { - panic(fmt.Sprintf(`Query: '%s' parsed, but got different results: -Result New (string) -%s -Result New (hashv) -%#v -Result Old (string) -%s -Result Old (hashv) -%#v -err: -%v -`, data, q1, q1, q2, q2, err)) - } - - // both queries parsed succesfully and got equivalent results - return 1 -} - -func queriesEqual(q1 *Query, q2 *oldpql.Query) (err error) { - if q1.String() != q2.String() { - defer func() { - // golang black magic - if err == nil { - err = errors.New("string reps unequal") - } else { - err = errors.Wrap(err, "string reps unequal") - } - }() - } - if len(q1.Calls) != len(q2.Calls) { - return errors.Errorf("call lengths unequal: %d and %d", len(q1.Calls), len(q2.Calls)) - } - for i, c1 := range q1.Calls { - c2 := q2.Calls[i] - if err := callsEqual(c1, c2); err != nil { - return errors.Wrapf(err, "calls at %d not equal", i) - } - } - return nil -} - -func callsEqual(c1 *Call, c2 *oldpql.Call) error { - if err := argsEqual(c1.Args, c2.Args); err != nil { - return errors.Wrap(err, "args unequal") - } - if c1.Name != c2.Name { - return errors.Errorf("names unequal '%s' != '%s'", c1.Name, c2.Name) - } - if len(c1.Children) != len(c2.Children) { - return errors.Errorf("different child lengths %d and %d", len(c1.Children), len(c2.Children)) - } - - for i, child1 := range c1.Children { - child2 := c2.Children[i] - if err := callsEqual(child1, child2); err != nil { - return errors.Wrapf(err, "children at %d not equal", i) - } - } - - return nil -} - -func argsEqual(a1 map[string]interface{}, a2 map[string]interface{}) error { - if len(a1) != len(a2) { - return errors.Errorf("lengths unequal %d and %d", len(a1), len(a2)) - } - - for k, v1 := range a1 { - v2 := a1[k] - if c1, ok := v1.(Condition); ok { - if c2, ok := v2.(oldpql.Condition); ok { - if int(c1.Op) != int(c2.Op) { - return errors.Errorf("condition ops unequal %d %d", c1, c2) - } - if !reflect.DeepEqual(c1.Value, c2.Value) { - return errors.Errorf("condition values unequal '%v' '%v'", c1.Value, c2.Value) - } - continue - } - return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2) - } - if !reflect.DeepEqual(v1, v2) { - return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2) - } - } - return nil -} From 22a546095ab45f2103906a0d152c174039cdc71c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 22 Jun 2018 10:05:41 -0500 Subject: [PATCH 26/33] fixed reset method on btree plugin --- enterprise/b/containers_btree.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index de001fcb7..7c208b1db 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -162,6 +162,8 @@ func (btc *BTreeContainers) Size() int { func (btc *BTreeContainers) Reset() { btc.tree = TreeNew(cmp) + btc.lastKey = 0 + btc.lastContainer = nil } func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { From 6dd8b9adc637b9683ed4840ab3e12d03c0a6d0f1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 22 Jun 2018 10:20:52 -0500 Subject: [PATCH 27/33] Move server initialization to prevent race condition --- http/handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 9b3d17789..58fdcb8b6 100644 --- a/http/handler.go +++ b/http/handler.go @@ -134,11 +134,12 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { return nil, errors.New("must pass OptHandlerListener") } + handler.server = &http.Server{Handler: handler} + return handler, nil } func (h *Handler) Serve() error { - h.server = &http.Server{Handler: h} err := h.server.Serve(h.ln) if err != nil && err.Error() != "http: Server closed" { h.Logger.Printf("HTTP handler terminated with error: %s\n", err) From 6b57369b511ec53cc9df3b3a9d3ca4118df1081c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 10:44:36 -0500 Subject: [PATCH 28/33] fix stats tests --- server/handler_test.go | 4 +- stats_test.go | 184 +++++++++++++++++------------------------ 2 files changed, 79 insertions(+), 109 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 8d001b529..2b0801585 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -38,6 +38,8 @@ import ( func TestHandler_Endpoints(t *testing.T) { cmd := test.MustRunMainWithCluster(t, 1)[0] h := cmd.Handler.(*http.Handler).Handler + holder := cmd.Server.Holder() + hldr := test.Holder{holder} t.Run("Not Found", func(t *testing.T) { w := httptest.NewRecorder() @@ -57,8 +59,6 @@ func TestHandler_Endpoints(t *testing.T) { } }) - holder := cmd.Server.Holder() - hldr := test.Holder{holder} i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { diff --git a/stats_test.go b/stats_test.go index 1f23dd3f8..7abcfa9c3 100644 --- a/stats_test.go +++ b/stats_test.go @@ -16,12 +16,13 @@ package pilosa_test import ( "context" - "net/http" + "net/http/httptest" "strings" "testing" "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/test" ) @@ -207,120 +208,89 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } } -func TestStatsCount_CreateIndex(t *testing.T) { - t.Skip() - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != "createIndex" { - t.Errorf("Expected createIndex, Results %s", name) - } +func TestStatsCount_APICalls(t *testing.T) { + cmd := test.MustRunMainWithCluster(t, 1)[0] + h := cmd.Handler.(*http.Handler).Handler + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil)) - if !called { - t.Error("Count isn't called") - } -} + t.Run("create index", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCount: func(name string, value int64, rate float64) { + if name != "createIndex" { + t.Errorf("Expected createIndex, Results %s", name) + } + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) -func TestStatsCount_DeleteIndex(t *testing.T) { - t.Skip() - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("create field", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, rate float64, index []string) { + if name != "createField" { + t.Errorf("Expected createField, Results %s", name) + } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != "deleteIndex" { - t.Errorf("Expected deleteIndex, Results %s", name) - } + t.Run("delete field", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, rate float64, index []string) { + if name != "deleteField" { + t.Errorf("Expected deleteField, Results %s", name) + } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } -} + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) -func TestStatsCount_CreateField(t *testing.T) { - t.Skip() - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("delete index", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCount: func(name string, value int64, rate float64) { + if name != "deleteIndex" { + t.Errorf("Expected deleteIndex, Results %s", name) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "createField" { - t.Errorf("Expected createField, Results %s", name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil)) - if !called { - t.Error("Count isn't called") - } -} - -func TestStatsCount_DeleteField(t *testing.T) { - t.Skip() - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - called := false - // Create index. - indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - s.Handler.API.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "deleteField" { - t.Errorf("Expected deleteField, Results %s", name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } } type MockStats struct { From ba9112507daff1450f693e79da4e6fb5314e0fde Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 12:57:28 -0500 Subject: [PATCH 29/33] fix server/handler_test.go for newpql --- executor.go | 6 ++++++ server/handler_test.go | 30 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/executor.go b/executor.go index 027bf50ad..1fbf56960 100644 --- a/executor.go +++ b/executor.go @@ -1618,6 +1618,9 @@ func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { // Translate row key, if field is specified & key exists. if fieldName != "" { field := idx.Field(fieldName) + if field == nil { + return ErrFieldNotFound + } if field.Keys() { if value := callArgString(c, rowKey); value != "" { ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) @@ -1659,6 +1662,9 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res case []Pair: if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) + if field == nil { + return nil, ErrFieldNotFound + } if field.Keys() { other := make([]Pair, len(result)) for i := range result { diff --git a/server/handler_test.go b/server/handler_test.go index 2b0801585..0320b6f09 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -133,7 +133,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Slices args", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { @@ -144,7 +144,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Slices args protobuf", func(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ - Query: "Count(Bitmap(field=f0, row=30))", + Query: "Count(Row(f0=30))", Slices: []uint64{0, 1}, }) if err != nil { @@ -168,7 +168,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query args error", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -178,7 +178,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query params err", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Bitmap(field=f0, row=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { @@ -188,7 +188,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Uint64 protobuf", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Bitmap(field=f0, row=30))")) + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Row(f0=30))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { @@ -205,9 +205,9 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Bitmap JSON", func(t *testing.T) { + t.Run("Row JSON", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Bitmap(field=f0, row=30)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { @@ -226,7 +226,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ColumnAttrs_JSON", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Bitmap(field=f0, row=30)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { @@ -236,7 +236,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Row pbuf", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Bitmap(field=f0, row=30)")) + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { @@ -264,7 +264,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Row columnattrs protobuf", func(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(field=f0, row=30)", + Query: "Row(f0=30)", ColumnAttrs: true, }) if err != nil { @@ -311,7 +311,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query Pairs JSON", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(field=f0, n=2)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" { @@ -321,7 +321,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query Pairs protobuf", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(field=f0, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { @@ -340,7 +340,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query err JSON", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Bitmap(row=30)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"executing: field not found"}`+"\n" { @@ -350,7 +350,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query err protobuf", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Bitmap(row=30)`)) + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { @@ -378,7 +378,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { t.Fatalf("unexpected body: %s", body) } }) From 5d28d2dc3103b666e32c7197d4f310351c04bc81 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 13:00:52 -0500 Subject: [PATCH 30/33] skip new tests which use test.NewServer --- ctl/import_test.go | 2 +- executor_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index eadbdb47f..3d48418cd 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -177,7 +177,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { } func TestImportCommand_BugOverwriteValue(t *testing.T) { - + t.Skip("test.NewServer broken") buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) diff --git a/executor_test.go b/executor_test.go index 71e5b4393..e4500b9a5 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1310,6 +1310,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Ensure a remote query can set RowAttrs func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { + t.Skip("test.NewServer broken") c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. From c9e6d36f94fa9a4815e0ec12a1919cf49e2cae83 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 13:44:50 -0500 Subject: [PATCH 31/33] fix some of the client tests --- http/client_test.go | 51 +++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/http/client_test.go b/http/client_test.go index 03b5b8217..5ac29ec2c 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -219,23 +219,17 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() + cmd := test.MustRunMainWithCluster(t, 1)[0] + host := cmd.Server.Addr().String() + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} // Load bitmap into cache to ensure cache gets updated. hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. hldr.Row("i", "f", 0) - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - // Send import request. - c := MustNewClient(s.Host(), defaultClient) + c := MustNewClient(host, defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -255,13 +249,12 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { - t.Skip() // Until test.NewServer() works - - hldr := test.MustOpenHolder() - defer hldr.Close() + cmd := test.MustRunMainWithCluster(t, 1)[0] + host := cmd.Server.Addr().String() + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} fldName := "f" - fo := pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: -100, @@ -275,14 +268,8 @@ func TestClient_ImportValue(t *testing.T) { t.Fatal(err) } - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - // Send import request. - c := MustNewClient(s.Host(), defaultClient) + c := MustNewClient(host, defaultClient) if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, @@ -334,26 +321,16 @@ func TestClient_ImportValue(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - t.Skip() // Until test.NewServer() works + cmd := test.MustRunMainWithCluster(t, 1)[0] + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Set two bits on blocks 0 & 3. hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. hldr.SetBit("i", "f", 0, 1) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Retrieve blocks. - c := MustNewClient(s.Host(), defaultClient) + c := MustNewClient(cmd.Server.Addr().String(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) From 6093064ac0b00f295cf2184873087e7e04be6d92 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 22 Jun 2018 16:26:45 -0500 Subject: [PATCH 32/33] remove unecessary test and convert import test --- ctl/import_test.go | 17 +++++------------ fragment_internal_test.go | 33 --------------------------------- server/handler_test.go | 2 +- 3 files changed, 6 insertions(+), 46 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index 3d48418cd..717c40342 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -177,7 +177,8 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { } func TestImportCommand_BugOverwriteValue(t *testing.T) { - t.Skip("test.NewServer broken") + cmd := test.MustRunMainWithCluster(t, 1)[0] + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -188,18 +189,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { t.Fatal(err) } - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() + cm.Host = cmd.Server.Addr().String() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() - - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) cm.Index = "i" cm.Field = "f" diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 6da3ded7e..6c733ba4c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -213,39 +213,6 @@ func TestFragment_SetValue(t *testing.T) { t.Fatal(err) } }) - t.Run("Crash", func(t *testing.T) { - f := mustOpenFragment("i", "f", ViewStandard, 0, "") - defer f.Close() - - // Set value. - if changed, err := f.setValue(0, 32, 17); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - if changed, err := f.setValue(0, 32, 16); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - if changed, err := f.setValue(0, 32, 19); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Read value. - if value, exists, err := f.value(0, 32); err != nil { - t.Fatal(err) - } else if value != 19 { - t.Fatalf("unexpected value: %d", value) - } else if !exists { - t.Fatal("expected to exist") - } - }) - } // Ensure a fragment can sum values. diff --git a/server/handler_test.go b/server/handler_test.go index 0320b6f09..070a3176a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -39,7 +39,7 @@ func TestHandler_Endpoints(t *testing.T) { cmd := test.MustRunMainWithCluster(t, 1)[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() - hldr := test.Holder{holder} + hldr := test.Holder{Holder: holder} t.Run("Not Found", func(t *testing.T) { w := httptest.NewRecorder() From 8878b02345d8b74cdadc2077df710d7ad8b18262 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 25 Jun 2018 11:08:00 -0500 Subject: [PATCH 33/33] cleanup - address review feedback --- api.go | 2 +- ctl/import_test.go | 7 +++---- test/handler.go | 7 ------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/api.go b/api.go index c02e67b79..28248230c 100644 --- a/api.go +++ b/api.go @@ -50,7 +50,7 @@ type API struct { } // APIOption is a functional option type for pilosa.API -type APIOption func(s *API) error +type APIOption func(*API) error func OptAPIServer(s *Server) APIOption { return func(a *API) error { diff --git a/ctl/import_test.go b/ctl/import_test.go index 717c40342..5500fdadf 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -87,11 +87,10 @@ func TestImportCommand_RunValue(t *testing.T) { } cmd := test.MustRunMainWithCluster(t, 1)[0] - hostport := cmd.Server.URI.HostPort() - cm.Host = hostport + cm.Host = cmd.Server.URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" cm.Field = "f" diff --git a/test/handler.go b/test/handler.go index 8b58d5a6e..048d9b883 100644 --- a/test/handler.go +++ b/test/handler.go @@ -46,13 +46,6 @@ func NewHandler(opts ...http.HandlerOption) (*Handler, error) { Handler: handler, } - //h.API, err = pilosa.NewAPI(OptAPIServer(s)) - if err != nil { - return nil, err - } - h.Handler.API = h.API - h.Handler.API.Executor = &h.Executor - // Handler test messages can no-op. h.API.Broadcaster = pilosa.NopBroadcaster