featurebase/pql/parser.go
Ben Johnson 569bafdc7d add bulk call support to the query endpoint
This commit changes the `pql.Query` so that it can accept one or more
top-level calls instead of only one.

The query request format change because a query with a single call is
still valid. However, the result format now returns a `results` field
that has one result for each top-level call. The `profiles` field is
still the same, however, it combines all profiles from all bitmap
responses into one return so that there's not duplicate attributes.

Fixes #59
2016-03-08 14:34:37 -07:00

810 lines
18 KiB
Go

package pql
import (
"fmt"
"io"
"strconv"
"strings"
"time"
)
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
scanner *bufScanner
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
scanner: newBufScanner(r),
}
}
// ParseString parses s into a query.
func ParseString(s string) (*Query, error) {
return NewParser(strings.NewReader(s)).Parse()
}
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
q := &Query{}
for {
call, err := p.parseCall()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
q.Calls = append(q.Calls, call)
}
// Require at least one call.
if len(q.Calls) == 0 {
return nil, io.ErrUnexpectedEOF
}
return q, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
switch lit {
case "Bitmap":
return p.parseBitmapCall()
case "Count":
return p.parseCountCall()
case "ClearBit":
return p.parseClearBitCall()
case "Difference":
return p.parseDifferenceCall()
case "Intersect":
return p.parseIntersectCall()
case "Profile":
return p.parseProfileCall()
case "Range":
return p.parseRangeCall()
case "SetBit":
return p.parseSetBitCall()
case "SetBitmapAttrs":
return p.parseSetBitmapAttrsCall()
case "SetProfileAttrs":
return p.parseSetProfileAttrsCall()
case "TopN":
return p.parseTopNCall()
case "Union":
return p.parseUnionCall()
default:
return nil, &ParseError{Message: fmt.Sprintf("function not found: %s", lit), Pos: pos}
}
}
// parseBitmapCall parses a Bitmap() function call.
func (p *Parser) parseBitmapCall() (*Bitmap, error) {
c := &Bitmap{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Bitmap() arg: %v", arg.key)
}
}
return c, nil
}
// parseClearBitCall parses a ClearBit() function call.
func (p *Parser) parseClearBitCall() (*ClearBit, error) {
c := &ClearBit{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid ClearBit() arg: %v", arg.key)
}
}
return c, nil
}
// parseCount parses a Count() function call.
func (p *Parser) parseCountCall() (*Count, error) {
c := &Count{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
} else if len(args) != 1 {
return nil, parseErrorf(pos, "count requires one argument")
}
// Copy argument to AST.
input, ok := args[0].value.(BitmapCall)
if !ok {
return nil, parseErrorf(pos, "invalid count arg: %s", args[0].value)
}
c.Input = input
return c, nil
}
// parseDifference parses a Difference() function call.
func (p *Parser) parseDifferenceCall() (*Difference, error) {
c := &Difference{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Difference() arg: %v", arg.value)
}
}
return c, nil
}
// parseIntersect parses a Intersect() function call.
func (p *Parser) parseIntersectCall() (*Intersect, error) {
c := &Intersect{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Intersect() arg: %v", arg.value)
}
}
return c, nil
}
// parseProfileCall parses a Profile() function call.
func (p *Parser) parseProfileCall() (*Profile, error) {
c := &Profile{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Profile() arg: %v", arg.key)
}
}
return c, nil
}
// parseRangeCall parses a Range() function call.
func (p *Parser) parseRangeCall() (*Range, error) {
c := &Range{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "start":
if err := decodeDate(arg.value, &c.StartTime); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 3, "end":
if err := decodeDate(arg.value, &c.EndTime); err != nil {
return nil, parseErrorf(pos, "end: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Range() arg: %v", arg.key)
}
}
return c, nil
}
// parseSetBitCall parses a SetBit() function call.
func (p *Parser) parseSetBitCall() (*SetBit, error) {
c := &SetBit{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid SetBit() arg: %v", arg.key)
}
}
return c, nil
}
// parseSetBitmapAttrsCall parses a SetBitmapAttrs() function call.
func (p *Parser) parseSetBitmapAttrsCall() (*SetBitmapAttrs, error) {
c := &SetBitmapAttrs{
Attrs: make(map[string]interface{}),
}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
key, ok := arg.key.(string)
if !ok {
return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key)
}
// Special handling for nil values.
if arg.value == nil {
c.Attrs[key] = nil
continue
}
switch v := arg.value.(type) {
case string, bool:
c.Attrs[key] = v
case uint64:
c.Attrs[key] = v
default:
return nil, parseErrorf(pos, "invalid SetBitmapAttrs() arg: %v", arg.key)
}
}
}
return c, nil
}
// parseSetProfileAttrsCall parses a SetProfileAttrs() function call.
func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) {
c := &SetProfileAttrs{
Attrs: make(map[string]interface{}),
}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
default:
key, ok := arg.key.(string)
if !ok {
return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key)
}
// Special handling for nil values.
if arg.value == nil {
c.Attrs[key] = nil
continue
}
switch v := arg.value.(type) {
case string, bool:
c.Attrs[key] = v
case uint64:
c.Attrs[key] = v
default:
return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key)
}
}
}
return c, nil
}
// parseTopNCall parses a TopN() function call.
func (p *Parser) parseTopNCall() (*TopN, error) {
c := &TopN{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Src = v
continue
}
// Assign filter values if there's only a value and no named key.
if v, ok := arg.value.([]interface{}); ok {
if _, ok := arg.key.(string); !ok {
c.Filters = v
continue
}
}
switch arg.key {
case 0, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 1, "n":
if err := decodeInt(arg.value, &c.N); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
case 2, "field":
if err := decodeString(arg.value, &c.Field); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
case "ids":
if err := decodeUint64Slice(arg.value, &c.BitmapIDs); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key)
}
}
return c, nil
}
// parseUnion parses a Union() function call.
func (p *Parser) parseUnionCall() (*Union, error) {
c := &Union{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Union() arg: %v", arg.value)
}
}
return c, nil
}
// parseArgs arguments to a function call.
func (p *Parser) parseArgs() ([]arg, error) {
var i int
var args []arg
for {
// Parse next argument.
arg, err := p.parseArg()
if err != nil {
return nil, err
}
// If it's a primitive type without a key then index it.
if arg.key == nil {
switch arg.value.(type) {
case uint64, string:
arg.key = i
i++
}
}
// Append argument to list.
args = append(args, arg)
// If next token is a closing parenthesis, then exit.
// Otherwise expect a comma.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return args, nil
}
// parseArg parses a single argument to a function call.
func (p *Parser) parseArg() (arg, error) {
var key, value interface{}
// Read identifier and check if there's a following "=" or "(".
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
// If a left paren immediately follows then it's a function call.
if tok, _, _ := p.scan(); tok == LPAREN {
p.unscan(2)
c, err := p.parseCall()
if err != nil {
return arg{}, err
}
return arg{value: c}, nil
}
// If it's not a left paren, rescan ignoring whitespace and look for "=",
p.unscan(1)
if tok, _, _ := p.scanIgnoreWhitespace(); tok == EQ {
key = lit // keyed arg
} else {
p.unscan(1)
}
default:
p.unscan(1)
}
// Read value token.
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
value = true
} else if lit == "false" {
value = false
} else if lit == "null" {
value = nil
} else {
value = lit
}
case STRING:
value = lit
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return arg{}, err
}
value = v
case LBRACK:
v, err := p.parseList()
if err != nil {
return arg{}, err
}
value = v
default:
return arg{}, parseErrorf(pos, "invalid value: %q", lit)
}
return arg{key: key, value: value}, nil
}
// parseListArg parses a list of primitives. This is used by the TopN() filters.
func (p *Parser) parseList() ([]interface{}, error) {
var values []interface{}
for {
// Read next value.
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
values = append(values, true)
} else if lit == "false" {
values = append(values, false)
} else {
values = append(values, lit)
}
case STRING:
values = append(values, lit)
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return nil, err
}
values = append(values, v)
default:
return nil, parseErrorf(pos, "invalid list value: %q", lit)
}
// Expect a comma or closing bracket next.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return values, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
tok, pos, lit = p.scan()
if tok == WS {
tok, pos, lit = p.scan()
}
return
}
// unscan returns the last n tokens back to the scanner.
func (p *Parser) unscan(n int) {
for i := 0; i < n; i++ {
p.scanner.unscan()
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// expectIgnoreWhitespace returns an error if the next non-whitespace token is not exp.
func (p *Parser) expectIgnoreWhitespace(exp Token) error {
if tok, pos, lit := p.scanIgnoreWhitespace(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// arg represents an call argument.
// The key can be the index or the string key.
// The value can be a uint64, []uint64, string, Call, or Calls.
type arg struct {
key interface{}
value interface{}
}
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
Pos Pos
}
// Error returns a string representation of e.
func (e *ParseError) Error() string {
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
}
// parseErrorf returns a formatted parse error.
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
return &ParseError{
Message: fmt.Sprintf(format, args...),
Pos: pos,
}
}
// decodeInt type converts v to target.
func decodeInt(v interface{}, target *int) error {
if v, ok := v.(uint64); ok {
*target = int(v)
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64 type converts v to target.
func decodeUint64(v interface{}, target *uint64) error {
if v, ok := v.(uint64); ok {
*target = v
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64Slice type converts v to target.
func decodeUint64Slice(v interface{}, target *[]uint64) error {
input, ok := v.([]interface{})
if !ok {
return fmt.Errorf("invalid array value: %v", v)
}
a := make([]uint64, len(input))
for i := range input {
elem, ok := input[i].(uint64)
if !ok {
return fmt.Errorf("invalid int element: %v", input[i])
}
a[i] = elem
}
*target = a
return nil
}
// decodeString type converts v to target.
func decodeString(v interface{}, target *string) error {
if v, ok := v.(string); ok {
*target = v
return nil
}
return fmt.Errorf("invalid string value: %v", v)
}
// decodeDate type converts v to target.
func decodeDate(v interface{}, target *time.Time) error {
if v, ok := v.(string); ok {
t, err := time.Parse(TimeFormat, v)
if err != nil {
return fmt.Errorf("invalid date format: %s", v)
}
*target = t
return nil
}
return fmt.Errorf("invalid date value: %v", v)
}