scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor parser & AST to remove compile-time field names.

Browse source 
Consolidates all the pql.Call implementations into a single
pql.Call struct. This is needed to support user-defined schemas
on frames.

This also has the added benefit that a lot of redundant parsing
code has been consolidated.
This commit is contained in:
Ben Johnson 2017-02-02 14:23:07 -07:00
parent c2253a36ea
commit f39768b49b
8 changed files with 547 additions and 1490 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

304
executor.go
View file

@ -80,31 +80,30 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
}
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, db string, c pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(ctx, db, c, slices, opt)
case *pql.ClearBit:
func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
// Special handling for mutation and top-n calls.
switch c.Name {
case "ClearBit":
return e.executeClearBit(ctx, db, c, opt)
case *pql.Count:
case "Count":
return e.executeCount(ctx, db, c, slices, opt)
case *pql.Profile:
case "Profile":
return e.executeProfile(ctx, db, c, opt)
case *pql.SetBit:
case "SetBit":
return e.executeSetBit(ctx, db, c, opt)
case *pql.SetBitmapAttrs:
case "SetBitmapAttrs":
return nil, e.executeSetBitmapAttrs(ctx, db, c, opt)
case *pql.SetProfileAttrs:
case "SetProfileAttrs":
return nil, e.executeSetProfileAttrs(ctx, db, c, opt)
case *pql.TopN:
case "TopN":
return e.executeTopN(ctx, db, c, slices, opt)
default:
panic("unreachable")
return e.executeBitmapCall(ctx, db, c, slices, opt)
}
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.BitmapCall, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeBitmapCallSlice(ctx, db, c, slice)
@ -127,10 +126,13 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.Bitma
// Attach bitmap attributes for Bitmap() calls.
bm, _ := other.(*Bitmap)
if c, ok := c.(*pql.Bitmap); ok {
fr := e.Index.Frame(db, c.Frame)
if c.Name == "Bitmap" {
id, _ := c.Args["id"].(uint64)
frame, _ := c.Args["frame"].(string)
fr := e.Index.Frame(db, frame)
if fr != nil {
attrs, err := fr.BitmapAttrStore().Attrs(c.ID)
attrs, err := fr.BitmapAttrStore().Attrs(id)
if err != nil {
return nil, err
}
@ -142,27 +144,29 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.Bitma
}
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c pql.BitmapCall, slice uint64) (*Bitmap, error) {
switch c := c.(type) {
case *pql.Bitmap:
func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
switch c.Name {
case "Bitmap":
return e.executeBitmapSlice(ctx, db, c, slice)
case *pql.Difference:
case "Difference":
return e.executeDifferenceSlice(ctx, db, c, slice)
case *pql.Intersect:
case "Intersect":
return e.executeIntersectSlice(ctx, db, c, slice)
case *pql.Range:
case "Range":
return e.executeRangeSlice(ctx, db, c, slice)
case *pql.Union:
case "Union":
return e.executeUnionSlice(ctx, db, c, slice)
default:
panic("unreachable")
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
bitmapIDs, _ := c.Args["ids"].([]uint64)
// Execute original query.
pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt)
if err != nil {
@ -171,20 +175,24 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.TopN, slic
// If this call is against specific ids, or we didn't get results,
// or we are part of a larger distributed query then don't refetch.
if len(pairs) == 0 || len(c.BitmapIDs) > 0 || opt.Remote {
if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote {
return pairs, nil
}
// Only the original caller should refetch the full counts.
other := *c
other.N = 0
other.BitmapIDs = Pairs(pairs).Keys()
sort.Sort(uint64Slice(other.BitmapIDs))
other := c.Clone()
other.Args["n"] = 0
return e.executeTopNSlices(ctx, db, &other, slices, opt)
ids := Pairs(pairs).Keys()
sort.Sort(uint64Slice(ids))
other.Args["ids"] = ids
return e.executeTopNSlices(ctx, db, other, slices, opt)
}
func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
n, _ := c.Args["n"].(uint64)
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeTopNSlice(ctx, db, c, slice)
@ -206,27 +214,34 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.TopN
sort.Sort(Pairs(results))
// Only keep the top n after sorting.
if c.N > 0 && len(results) > c.N {
results = results[0:c.N]
if n > 0 && len(results) > int(n) {
results = results[0:n]
}
return results, nil
}
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.TopN, slice uint64) ([]Pair, error) {
func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
n, _ := c.Args["n"].(uint64)
field, _ := c.Args["field"].(string)
bitmapIDs, _ := c.Args["ids"].([]uint64)
filters, _ := c.Args["filters"].([]interface{})
// Retrieve bitmap used to intersect.
var src *Bitmap
if c.Src != nil {
bm, err := e.executeBitmapCallSlice(ctx, db, c.Src, slice)
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
if err != nil {
return nil, err
}
src = bm
} else if len(c.Children) > 1 {
return nil, errors.New("TopN() can only have one input bitmap")
}
// Set default frame.
frame := c.Frame
if frame == "" {
frame = DefaultFrame
}
@ -237,18 +252,18 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.TopN,
}
return f.Top(TopOptions{
N: c.N,
N: int(n),
Src: src,
BitmapIDs: c.BitmapIDs,
FilterField: c.Field,
FilterValues: c.Filters,
BitmapIDs: bitmapIDs,
FilterField: field,
FilterValues: filters,
})
}
// executeDifferenceSlice executes a difference() call for a local slice.
func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Difference, slice uint64) (*Bitmap, error) {
func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
if err != nil {
return nil, err
@ -264,8 +279,9 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql
return other, nil
}
func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Bitmap, slice uint64) (*Bitmap, error) {
frame := c.Frame
func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
id, _ := c.Args["id"].(uint64)
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
@ -274,13 +290,13 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Bit
if f == nil {
return NewBitmap(), nil
}
return f.Bitmap(c.ID), nil
return f.Bitmap(id), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Intersect, slice uint64) (*Bitmap, error) {
func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
if err != nil {
return nil, err
@ -297,8 +313,31 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.
}
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Range, slice uint64) (*Bitmap, error) {
frame := c.Frame
func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
id, _ := c.Args["id"].(uint64)
// Parse start time.
startTimeStr, ok := c.Args["start"].(string)
if !ok {
return nil, errors.New("Range() start time required")
}
startTime, err := time.Parse(TimeFormat, startTimeStr)
if err != nil {
return nil, errors.New("cannot parse Range() start time")
}
// Parse end time.
endTimeStr, _ := c.Args["end"].(string)
if !ok {
return nil, errors.New("Range() end time required")
}
endTime, err := time.Parse(TimeFormat, endTimeStr)
if err != nil {
return nil, errors.New("cannot parse Range() end time")
}
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
@ -317,20 +356,20 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang
// Union bitmaps across all time-based subframes.
bm := &Bitmap{}
for _, subframe := range FramesByTimeRange(frame, c.StartTime, c.EndTime, q) {
for _, subframe := range FramesByTimeRange(frame, startTime, endTime, q) {
f := e.Index.Fragment(db, subframe, slice)
if f == nil {
continue
}
bm = bm.Union(f.Bitmap(c.ID))
bm = bm.Union(f.Bitmap(id))
}
return bm, nil
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Union, slice uint64) (*Bitmap, error) {
func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
if err != nil {
return nil, err
@ -347,10 +386,16 @@ func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Unio
}
// executeCount executes a count() call.
func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Count, slices []uint64, opt *ExecOptions) (uint64, error) {
func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
if len(c.Children) == 0 {
return 0, errors.New("Count() requires an input bitmap")
} else if len(c.Children) > 1 {
return 0, errors.New("Count() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
bm, err := e.executeBitmapCallSlice(ctx, db, c.Input, slice)
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
if err != nil {
return 0, err
}
@ -374,23 +419,38 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Count, sl
// executeProfile executes a Profile() call.
// This call only executes locally since the profile attibutes are stored locally.
func (e *Executor) executeProfile(ctx context.Context, db string, c *pql.Profile, opt *ExecOptions) (*Profile, error) {
func (e *Executor) executeProfile(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (*Profile, error) {
panic("FIXME: impl: e.Index.ProfileAttr(c.ID)")
}
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("ClearBit() frame required")
}
id, ok := c.Args["id"].(uint64)
if !ok {
return false, errors.New("ClearBit() id required")
}
profileID, ok := c.Args["profileID"].(uint64)
if !ok {
return false, errors.New("ClearBit() profileID required")
}
slice := profileID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
// Update locally if host matches.
if node.Host == e.Host {
f := e.Index.Fragment(db, c.Frame, slice)
f := e.Index.Fragment(db, frame, slice)
if f == nil {
return false, nil
}
val, err := f.ClearBit(c.ID, c.ProfileID)
val, err := f.ClearBit(id, profileID)
if err != nil {
return false, err
} else if val {
@ -404,7 +464,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearB
}
// Forward call to remote node otherwise.
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
@ -414,8 +474,23 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearB
}
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("SetBit() frame required")
}
id, ok := c.Args["id"].(uint64)
if !ok {
return false, errors.New("SetBit() id required")
}
profileID, ok := c.Args["profileID"].(uint64)
if !ok {
return false, errors.New("SetBit() profileID required")
}
slice := profileID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
@ -425,7 +500,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit,
if err != nil {
return false, fmt.Errorf("db: %s", err)
}
val, err := db.SetBit(c.Frame, c.ID, c.ProfileID, opt.Timestamp)
val, err := db.SetBit(frame, id, profileID, opt.Timestamp)
if err != nil {
return false, err
} else if val {
@ -440,7 +515,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit,
}
// Forward call to remote node otherwise.
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
@ -450,15 +525,30 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit,
}
// executeSetBitmapAttrs executes a SetBitmapAttrs() call.
func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.SetBitmapAttrs, opt *ExecOptions) error {
func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
frameName, ok := c.Args["frame"].(string)
if !ok {
return errors.New("SetBitmapAttrs() frame required")
}
id, ok := c.Args["id"].(uint64)
if !ok {
return errors.New("SetBitmapAttrs() id required")
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "id")
// Retrieve frame.
frame, err := e.Index.CreateFrameIfNotExists(db, c.Frame)
frame, err := e.Index.CreateFrameIfNotExists(db, frameName)
if err != nil {
return err
}
// Set attributes.
if err := frame.BitmapAttrStore().SetAttrs(c.ID, c.Attrs); err != nil {
if err := frame.BitmapAttrStore().SetAttrs(id, attrs); err != nil {
return err
}
@ -472,7 +562,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt)
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
@ -488,25 +578,38 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
}
// executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls.
func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls pql.Calls, opt *ExecOptions) ([]interface{}, error) {
func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
// Collect attributes by frame/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, call := range calls {
c := call.(*pql.SetBitmapAttrs)
for _, c := range calls {
frame, ok := c.Args["frame"].(string)
if !ok {
return nil, errors.New("SetBitmapAttrs() frame required")
}
id, ok := c.Args["id"].(uint64)
if !ok {
return nil, errors.New("SetBitmapAttrs() id required")
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "id")
// Create frame group, if not exists.
frameMap := m[c.Frame]
frameMap := m[frame]
if frameMap == nil {
frameMap = make(map[uint64]map[string]interface{})
m[c.Frame] = frameMap
m[frame] = frameMap
}
// Set or merge attributes.
attr := frameMap[c.ID]
attr := frameMap[id]
if attr == nil {
frameMap[c.ID] = cloneAttrs(c.Attrs)
frameMap[id] = cloneAttrs(attrs)
} else {
for k, v := range c.Attrs {
for k, v := range attrs {
attr[k] = v
}
}
@ -553,7 +656,16 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
}
// executeSetProfileAttrs executes a SetProfileAttrs() call.
func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.SetProfileAttrs, opt *ExecOptions) error {
func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
id, ok := c.Args["id"].(uint64)
if !ok {
return errors.New("SetProfileAttrs() id required")
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "id")
// Retrieve database.
d, err := e.Index.CreateDBIfNotExists(db)
if err != nil {
@ -561,7 +673,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
}
// Set attributes.
if err := d.ProfileAttrStore().SetAttrs(c.ID, c.Attrs); err != nil {
if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil {
return err
}
@ -575,7 +687,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt)
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
@ -656,21 +768,19 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
var v interface{}
var err error
switch call.(type) {
case pql.BitmapCall:
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
case *pql.TopN:
switch call.Name {
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case *pql.Count:
case "Count":
v, err = pb.Results[i].N, nil
case *pql.SetBit:
case "SetBit":
v, err = pb.Results[i].Changed, nil
case *pql.ClearBit:
case "ClearBit":
v, err = pb.Results[i].Changed, nil
case *pql.SetBitmapAttrs:
case *pql.SetProfileAttrs:
case "SetBitmapAttrs":
case "SetProfileAttrs":
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", call))
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
}
if err != nil {
return nil, err
@ -703,7 +813,7 @@ loop:
//
// If a mapping of slices to a node fails then the slices are resplit across
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
ch := make(chan mapResponse, 0)
// Wrap context with a cancel to kill goroutines on exit.
@ -761,7 +871,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
}
}
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
// Group slices together by nodes.
m, err := e.slicesByNode(nodes, db, slices)
if err != nil {
@ -777,7 +887,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
if n.Host == e.Host {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.exec(ctx, n, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
if len(results) > 0 {
resp.result = results[0]
}
@ -864,13 +974,13 @@ func decodeError(s string) error {
}
// hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls.
func hasOnlySetBitmapAttrs(calls pql.Calls) bool {
func hasOnlySetBitmapAttrs(calls []*pql.Call) bool {
if len(calls) == 0 {
return false
}
for _, call := range calls {
if _, ok := call.(*pql.SetBitmapAttrs); !ok {
if call.Name != "SetBitmapAttrs" {
return false
}
}

8
executor_test.go
View file

@ -287,7 +287,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Bitmap(id=10, frame=f)` {
} else if query.String() != `Bitmap(frame="f", id=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0}) { //TODO: this is incorrect because the calling node doesn't know about slice 2
t.Fatalf("unexpected slices: %+v", slices)
@ -359,7 +359,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(id=10, frame=f, profileID=2)` {
} else if query.String() != `SetBit(frame="f", id=10, profileID=2)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -406,11 +406,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// slices and a second time to get the counts for a set of bitmaps.
switch remoteExecN {
case 0:
if query.String() != `TopN(frame=f, n=3)` {
if query.String() != `TopN(frame="f", n=3)` {
t.Fatalf("unexpected query(0): %s", query.String())
}
case 1:
if query.String() != `TopN(frame=f, ids=[0,10,30])` {
if query.String() != `TopN(frame="f", ids=[0,10,30], n=0)` {
t.Fatalf("unexpected query(1): %s", query.String())
}
default:

26
handler_test.go
View file

@ -93,9 +93,9 @@ func TestHandler_Query_Args_URL(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
t.Fatalf("unexpected status code: %d", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
@ -118,7 +118,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
DB: "db0",
Query: "Count(Bitmap(100))",
Query: "Count(Bitmap(id=100))",
Slices: []uint64{0, 1},
})
if err != nil {
@ -139,7 +139,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("Bitmap(100)")))
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
@ -155,7 +155,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
@ -171,7 +171,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Count(Bitmap(100))"))
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Count(Bitmap(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -196,7 +196,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(100)")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
@ -228,7 +228,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(100)")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
@ -246,7 +246,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Bitmap(100)"))
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Bitmap(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -293,7 +293,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
DB: "d",
Query: "Bitmap(100)",
Query: "Bitmap(id=100)",
Profiles: true,
})
if err != nil {
@ -389,7 +389,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`Bitmap(100)`)))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
@ -436,8 +436,8 @@ func TestHandler_Query_ErrParse(t *testing.T) {
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"function not found: bad_fn occurred at line 1, char 1"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}

395
pql/ast.go
View file

@ -1,16 +1,16 @@
package pql
import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"time"
)
// Query represents a PQL query.
type Query struct {
Calls Calls
Calls []*Call
}
// String returns a string representation of the query.
@ -22,344 +22,119 @@ func (q *Query) String() string {
return strings.Join(a, "\n")
}
// Node represents any node in the AST.
type Node interface {
node()
String() string
}
func (*Bitmap) node() {}
func (*ClearBit) node() {}
func (*Count) node() {}
func (*Difference) node() {}
func (*Intersect) node() {}
func (*Profile) node() {}
func (*Range) node() {}
func (*SetBit) node() {}
func (*SetBitmapAttrs) node() {}
func (*SetProfileAttrs) node() {}
func (*TopN) node() {}
func (*Union) node() {}
// Call represents a function call in the AST.
type Call interface {
Node
call()
type Call struct {
Name string
Args map[string]interface{}
Children []*Call
}
func (*Bitmap) call() {}
func (*ClearBit) call() {}
func (*Count) call() {}
func (*Difference) call() {}
func (*Intersect) call() {}
func (*Profile) call() {}
func (*Range) call() {}
func (*SetBit) call() {}
func (*SetBitmapAttrs) call() {}
func (*SetProfileAttrs) call() {}
func (*TopN) call() {}
func (*Union) call() {}
// Calls represents a list of calls.
type Calls []Call
// String returns a string representation of the calls as a comma-delimited list.
func (a Calls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
// Keys returns a list of argument keys in sorted order.
func (c *Call) Keys() []string {
a := make([]string, 0, len(c.Args))
for k := range c.Args {
a = append(a, k)
}
return strings.Join(args, ", ")
sort.Strings(a)
return a
}
// BitmapCall represents a function call that returns a bitmap.
type BitmapCall interface {
Call
bitmapCall()
}
// BitmapCalls represents a list of bitmap calls.
type BitmapCalls []BitmapCall
// String returns a string representation of the calls as a comma-delimited list.
func (a BitmapCalls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
// Clone returns a copy of c.
func (c *Call) Clone() *Call {
if c == nil {
return nil
}
return strings.Join(args, ", ")
}
func (*Difference) bitmapCall() {}
func (*Bitmap) bitmapCall() {}
func (*Intersect) bitmapCall() {}
func (*Range) bitmapCall() {}
func (*Union) bitmapCall() {}
// Bitmap represents a Bitmap() function call.
type Bitmap struct {
ID uint64
Frame string
other := &Call{
Name: c.Name,
Args: CopyArgs(c.Args),
}
if c.Children != nil {
other.Children = make([]*Call, len(c.Children))
for i := range c.Children {
other.Children[i] = c.Children[i].Clone()
}
}
return other
}
// String returns the string representation of the call.
func (c *Bitmap) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
return fmt.Sprintf("Bitmap(%s)", strings.Join(args, ", "))
}
func (c *Call) String() string {
var buf bytes.Buffer
// ClearBit represents a ClearBit() function call.
type ClearBit struct {
ID uint64
Frame string
ProfileID uint64
}
// String returns the string representation of the call.
func (c *ClearBit) String() string {
args := make([]string, 0, 4)
args = append(args, fmt.Sprintf("id=%d", c.ID))
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
}
return fmt.Sprintf("ClearBit(%s)", strings.Join(args, ", "))
}
// Count represents a count() function call.
type Count struct {
Input BitmapCall
}
// String returns the string representation of the call.
func (c *Count) String() string {
return fmt.Sprintf("Count(%s)", c.Input.String())
}
// Difference represents an difference() function call.
type Difference struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Difference) String() string {
return fmt.Sprintf("Difference(%s)", c.Inputs.String())
}
// Intersect represents an intersect() function call.
type Intersect struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Intersect) String() string {
return fmt.Sprintf("Intersect(%s)", c.Inputs.String())
}
// Profile represents a Profile() function call.
type Profile struct {
ID uint64
}
// String returns the string representation of the call.
func (c *Profile) String() string {
args := make([]string, 0, 1)
args = append(args, fmt.Sprintf("id=%d", c.ID))
return fmt.Sprintf("Profile(%s)", strings.Join(args, ", "))
}
// Range represents a range() function call.
type Range struct {
ID uint64
Frame string
StartTime time.Time
EndTime time.Time
}
// String returns the string representation of the call.
func (c *Range) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if !c.StartTime.IsZero() {
args = append(args, fmt.Sprintf("start=%s", c.StartTime.Format(TimeFormat)))
}
if !c.EndTime.IsZero() {
args = append(args, fmt.Sprintf("end=%s", c.EndTime.Format(TimeFormat)))
}
return fmt.Sprintf("Range(%s)", strings.Join(args, ", "))
}
// SetBit represents a SetBit() function call.
type SetBit struct {
ID uint64
Frame string
ProfileID uint64
}
// String returns the string representation of the call.
func (c *SetBit) String() string {
args := make([]string, 0, 2)
args = append(args, fmt.Sprintf("id=%d", c.ID))
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
}
return fmt.Sprintf("SetBit(%s)", strings.Join(args, ", "))
}
// SetBitmapAttrs represents a SetBitmapAttrs() function call.
type SetBitmapAttrs struct {
ID uint64
Frame string
Attrs map[string]interface{}
}
// String returns the string representation of the call.
func (c *SetBitmapAttrs) String() string {
args := make([]string, 0, 2)
args = append(args, fmt.Sprintf("id=%d", c.ID))
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
// Write name.
if c.Name != "" {
buf.WriteString(c.Name)
} else {
buf.WriteString("!UNNAMED")
}
// Sort keys.
keys := make([]string, 0, len(c.Attrs))
for k := range c.Attrs {
keys = append(keys, k)
}
sort.Strings(keys)
// Write opening.
buf.WriteByte('(')
// Write key/value pairs.
for _, k := range keys {
if c.Attrs[k] == nil {
args = append(args, fmt.Sprintf("%s=null", k))
continue
// Write child list.
for i, child := range c.Children {
if i > 0 {
buf.WriteString(", ")
}
buf.WriteString(child.String())
}
// Separate children and args, if necessary.
if len(c.Children) > 0 && len(c.Args) > 0 {
buf.WriteString(", ")
}
// Write arguments in key order.
for i, key := range c.Keys() {
if i > 0 {
buf.WriteString(", ")
}
switch v := c.Attrs[k].(type) {
switch v := c.Args[key].(type) {
case string:
args = append(args, fmt.Sprintf("%s=\"%s\"", k, v))
fmt.Fprintf(&buf, "%v=%q", key, v)
case []interface{}:
fmt.Fprintf(&buf, "%v=%s", key, joinInterfaceSlice(v))
case []uint64:
fmt.Fprintf(&buf, "%v=%s", key, joinUint64Slice(v))
default:
args = append(args, fmt.Sprintf("%s=%v", k, v))
fmt.Fprintf(&buf, "%v=%v", key, v)
}
}
return fmt.Sprintf("SetBitmapAttrs(%s)", strings.Join(args, ", "))
// Write closing.
buf.WriteByte(')')
return buf.String()
}
// SetProfileAttrs represents a SetProfileAttrs() function call.
type SetProfileAttrs struct {
ID uint64
Attrs map[string]interface{}
}
// String returns the string representation of the call.
func (c *SetProfileAttrs) String() string {
args := make([]string, 0, 2)
args = append(args, fmt.Sprintf("id=%d", c.ID))
// Sort keys.
keys := make([]string, 0, len(c.Attrs))
for k := range c.Attrs {
keys = append(keys, k)
// CopyArgs returns a copy of m.
func CopyArgs(m map[string]interface{}) map[string]interface{} {
other := make(map[string]interface{}, len(m))
for k, v := range m {
other[k] = v
}
sort.Strings(keys)
return other
}
// Write key/value pairs.
for _, k := range keys {
if c.Attrs[k] == nil {
args = append(args, fmt.Sprintf("%s=null", k))
continue
}
switch v := c.Attrs[k].(type) {
func joinInterfaceSlice(a []interface{}) string {
other := make([]string, len(a))
for i := range a {
switch v := a[i].(type) {
case string:
args = append(args, fmt.Sprintf("%s=\"%s\"", k, v))
other[i] = fmt.Sprintf("%q", v)
default:
args = append(args, fmt.Sprintf("%s=%v", k, v))
other[i] = fmt.Sprintf("%v", v)
}
}
return fmt.Sprintf("SetProfileAttrs(%s)", strings.Join(args, ", "))
return "[" + strings.Join(other, ",") + "]"
}
// TopN represents a TopN() function call.
type TopN struct {
Frame string
// Maximum number of results to return.
N int
// Bitmap to use for intersection while computing top results.
// Original bitmap counts are used if no Src is provided.
Src BitmapCall
// Specific bitmaps to retrieve.
BitmapIDs []uint64
// Field name and values to filter on.
Field string
Filters []interface{}
}
// String returns the string representation of the call.
func (c *TopN) String() string {
args := make([]string, 0, 2)
if c.Src != nil {
args = append(args, c.Src.String())
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.N > 0 {
args = append(args, fmt.Sprintf("n=%d", c.N))
}
if len(c.BitmapIDs) > 0 {
strs := make([]string, len(c.BitmapIDs))
for i := range c.BitmapIDs {
strs[i] = strconv.FormatUint(c.BitmapIDs[i], 10)
}
args = append(args, fmt.Sprintf("ids=[%s]", strings.Join(strs, ",")))
}
if c.Field != "" {
args = append(args, fmt.Sprintf("field=%q", c.Field))
}
if len(c.Filters) > 0 {
filters := make([]string, 0, len(c.Filters))
for i := range c.Filters {
switch filter := c.Filters[i].(type) {
case string:
filters = append(filters, fmt.Sprintf("%q", filter))
default:
filters = append(filters, fmt.Sprintf("%v", filter))
}
}
args = append(args, fmt.Sprintf("[%s]", strings.Join(filters, ",")))
}
return fmt.Sprintf("TopN(%s)", strings.Join(args, ", "))
}
// Union represents a union() function call.
type Union struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Union) String() string {
return fmt.Sprintf("Union(%s)", c.Inputs.String())
func joinUint64Slice(a []uint64) string {
other := make([]string, len(a))
for i := range a {
other[i] = strconv.FormatUint(a[i], 10)
}
return "[" + strings.Join(other, ",") + "]"
}

103
pql/ast_test.go
View file

@ -2,103 +2,16 @@ package pql_test
import (
"testing"
"time"
"github.com/pilosa/pilosa/pql"
)
// Ensure the Bitmap call can be converted into a string.
func TestBitmap_String(t *testing.T) {
s := (&pql.Bitmap{ID: 1, Frame: "x.n"}).String()
if s != `Bitmap(id=1, frame=x.n)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the ClearBit call can be converted into a string.
func TestClearBit_String(t *testing.T) {
s := (&pql.ClearBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
if s != `ClearBit(id=1, frame=x.n, profileID=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Count call can be converted into a string.
func TestCount_String(t *testing.T) {
s := (&pql.Count{Input: &pql.Bitmap{ID: 1, Frame: "x.n"}}).String()
if s != `Count(Bitmap(id=1, frame=x.n))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Difference call can be converted into a string.
func TestDifference_String(t *testing.T) {
s := (&pql.Difference{Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1, Frame: "x.n"},
&pql.Bitmap{ID: 2},
},
}).String()
if s != `Difference(Bitmap(id=1, frame=x.n), Bitmap(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Intersect call can be converted into a string.
func TestIntersect_String(t *testing.T) {
s := (&pql.Intersect{Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1, Frame: "x.n"},
&pql.Bitmap{ID: 2},
},
}).String()
if s != `Intersect(Bitmap(id=1, frame=x.n), Bitmap(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Profile call can be converted into a string.
func TestProfile_String(t *testing.T) {
if s := (&pql.Profile{ID: 1}).String(); s != `Profile(id=1)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Range call can be converted into a string.
func TestRange_String(t *testing.T) {
s := (&pql.Range{
ID: 1,
Frame: "x.n",
StartTime: time.Unix(0, 0).UTC(),
EndTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
}).String()
if s != `Range(id=1, frame=x.n, start=1970-01-01T00:00, end=2000-01-02T03:04)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the SetBit call can be converted into a string.
func TestSetBit_String(t *testing.T) {
s := (&pql.SetBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
if s != `SetBit(id=1, frame=x.n, profileID=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the SetBitmapAttrs call can be converted into a string.
func TestSetBitmapAttrs_String(t *testing.T) {
s := (&pql.SetBitmapAttrs{ID: 1, Frame: "x.n", Attrs: map[string]interface{}{"foo": "bar", "baz": 123, "bat": true, "x": nil}}).String()
if s != `SetBitmapAttrs(id=1, frame=x.n, bat=true, baz=123, foo="bar", x=null)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Union call can be converted into a string.
func TestUnion_String(t *testing.T) {
s := (&pql.Union{Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1, Frame: "x.n"},
&pql.Bitmap{ID: 2},
},
}).String()
if s != `Union(Bitmap(id=1, frame=x.n), Bitmap(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
// Ensure call can be converted into a string.
func TestCall_String(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
c := &pql.Call{Name: "Bitmap"}
if s := c.String(); s != `Bitmap()` {
t.Fatalf("unexpected string: %s", s)
}
})
}

775
pql/parser.go
View file

@ -5,7 +5,6 @@ import (
"io"
"strconv"
"strings"
"time"
)
// TimeFormat is the go-style time format used to parse string dates.
@ -50,604 +49,160 @@ func (p *Parser) Parse() (*Query, error) {
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
func (p *Parser) parseCall() (*Call, error) {
var c Call
// Read call name.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
switch lit {
case "Bitmap":
return p.parseBitmapCall()
case "Count":
return p.parseCountCall()
case "ClearBit":
return p.parseClearBitCall()
case "Difference":
return p.parseDifferenceCall()
case "Intersect":
return p.parseIntersectCall()
case "Profile":
return p.parseProfileCall()
case "Range":
return p.parseRangeCall()
case "SetBit":
return p.parseSetBitCall()
case "SetBitmapAttrs":
return p.parseSetBitmapAttrsCall()
case "SetProfileAttrs":
return p.parseSetProfileAttrsCall()
case "TopN":
return p.parseTopNCall()
case "Union":
return p.parseUnionCall()
default:
return nil, &ParseError{Message: fmt.Sprintf("function not found: %s", lit), Pos: pos}
}
}
// parseBitmapCall parses a Bitmap() function call.
func (p *Parser) parseBitmapCall() (*Bitmap, error) {
c := &Bitmap{}
pos := p.pos()
c.Name = lit
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
// Parse children first.
children, err := p.parseChildren()
if err != nil {
return nil, err
}
c.Children = children
// If next token is a closing paren then exit.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
return &c, nil
} else if tok == IDENT {
p.unscan(1)
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit)
}
// Parse key/value arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
c.Args = args
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Bitmap() arg: %v", arg.key)
}
// Scan closing parenthesis.
if err := p.expect(RPAREN); err != nil {
return nil, err
}
return c, nil
return &c, nil
}
// parseClearBitCall parses a ClearBit() function call.
func (p *Parser) parseClearBitCall() (*ClearBit, error) {
c := &ClearBit{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid ClearBit() arg: %v", arg.key)
}
}
return c, nil
}
// parseCount parses a Count() function call.
func (p *Parser) parseCountCall() (*Count, error) {
c := &Count{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
} else if len(args) != 1 {
return nil, parseErrorf(pos, "count requires one argument")
}
// Copy argument to AST.
input, ok := args[0].value.(BitmapCall)
if !ok {
return nil, parseErrorf(pos, "invalid count arg: %s", args[0].value)
}
c.Input = input
return c, nil
}
// parseDifference parses a Difference() function call.
func (p *Parser) parseDifferenceCall() (*Difference, error) {
c := &Difference{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Difference() arg: %v", arg.value)
}
}
return c, nil
}
// parseIntersect parses a Intersect() function call.
func (p *Parser) parseIntersectCall() (*Intersect, error) {
c := &Intersect{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Intersect() arg: %v", arg.value)
}
}
return c, nil
}
// parseProfileCall parses a Profile() function call.
func (p *Parser) parseProfileCall() (*Profile, error) {
c := &Profile{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Profile() arg: %v", arg.key)
}
}
return c, nil
}
// parseRangeCall parses a Range() function call.
func (p *Parser) parseRangeCall() (*Range, error) {
c := &Range{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "start":
if err := decodeDate(arg.value, &c.StartTime); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 3, "end":
if err := decodeDate(arg.value, &c.EndTime); err != nil {
return nil, parseErrorf(pos, "end: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid Range() arg: %v", arg.key)
}
}
return c, nil
}
// parseSetBitCall parses a SetBit() function call.
func (p *Parser) parseSetBitCall() (*SetBit, error) {
c := &SetBit{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid SetBit() arg: %v", arg.key)
}
}
return c, nil
}
// parseSetBitmapAttrsCall parses a SetBitmapAttrs() function call.
func (p *Parser) parseSetBitmapAttrsCall() (*SetBitmapAttrs, error) {
c := &SetBitmapAttrs{
Attrs: make(map[string]interface{}),
}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
key, ok := arg.key.(string)
if !ok {
return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key)
}
// Special handling for nil values.
if arg.value == nil {
c.Attrs[key] = nil
continue
}
switch v := arg.value.(type) {
case string, bool:
c.Attrs[key] = v
case uint64:
c.Attrs[key] = v
default:
return nil, parseErrorf(pos, "invalid SetBitmapAttrs() arg: %v", arg.key)
}
}
}
return c, nil
}
// parseSetProfileAttrsCall parses a SetProfileAttrs() function call.
func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) {
c := &SetProfileAttrs{
Attrs: make(map[string]interface{}),
}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
default:
key, ok := arg.key.(string)
if !ok {
return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key)
}
// Special handling for nil values.
if arg.value == nil {
c.Attrs[key] = nil
continue
}
switch v := arg.value.(type) {
case string, bool:
c.Attrs[key] = v
case uint64:
c.Attrs[key] = v
default:
return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key)
}
}
}
return c, nil
}
// parseTopNCall parses a TopN() function call.
func (p *Parser) parseTopNCall() (*TopN, error) {
c := &TopN{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Src = v
continue
}
// Assign filter values if there's only a value and no named key.
if v, ok := arg.value.([]interface{}); ok {
if _, ok := arg.key.(string); !ok {
c.Filters = v
continue
}
}
switch arg.key {
case 0, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 1, "n":
if err := decodeInt(arg.value, &c.N); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
case 2, "field":
if err := decodeString(arg.value, &c.Field); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
case "ids":
if err := decodeUint64Slice(arg.value, &c.BitmapIDs); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key)
}
}
return c, nil
}
// parseUnion parses a Union() function call.
func (p *Parser) parseUnionCall() (*Union, error) {
c := &Union{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid Union() arg: %v", arg.value)
}
}
return c, nil
}
// parseArgs arguments to a function call.
func (p *Parser) parseArgs() ([]arg, error) {
var i int
var args []arg
// parseChildren parses call children.
func (p *Parser) parseChildren() ([]*Call, error) {
var offset int
var children []*Call
for {
// Parse next argument.
arg, err := p.parseArg()
// Ensure next two tokens are IDENT+LPAREN.
if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT {
p.unscanIgnoreWhitespace(1 + offset)
return children, nil
}
if tok, _, _ := p.scan(); tok != LPAREN {
p.unscanIgnoreWhitespace(2 + offset)
return children, nil
}
// Push tokens back on scanner and parse as a call.
p.unscan(2)
child, err := p.parseCall()
if err != nil {
return nil, err
}
children = append(children, child)
// If it's a primitive type without a key then index it.
if arg.key == nil {
switch arg.value.(type) {
case uint64, string:
arg.key = i
i++
}
}
// Append argument to list.
args = append(args, arg)
// If next token is a closing parenthesis, then exit.
// Otherwise expect a comma.
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return args, nil
}
// parseArg parses a single argument to a function call.
func (p *Parser) parseArg() (arg, error) {
var key, value interface{}
// Read identifier and check if there's a following "=" or "(".
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
// If a left paren immediately follows then it's a function call.
if tok, _, _ := p.scan(); tok == LPAREN {
p.unscan(2)
c, err := p.parseCall()
if err != nil {
return arg{}, err
}
return arg{value: c}, nil
}
// If it's not a left paren, rescan ignoring whitespace and look for "=",
p.unscan(1)
if tok, _, _ := p.scanIgnoreWhitespace(); tok == EQ {
key = lit // keyed arg
} else {
p.unscan(1)
return children, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
default:
p.unscan(1)
}
// Read value token.
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
value = true
} else if lit == "false" {
value = false
} else if lit == "null" {
value = nil
} else {
value = lit
}
case STRING:
value = lit
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return arg{}, err
}
value = v
case LBRACK:
v, err := p.parseList()
if err != nil {
return arg{}, err
}
value = v
default:
return arg{}, parseErrorf(pos, "invalid value: %q", lit)
// Make sure comma is unscanned.
offset = 1
}
return arg{key: key, value: value}, nil
}
// parseListArg parses a list of primitives. This is used by the TopN() filters.
// parseArgs parses key/value arguments.
func (p *Parser) parseArgs() (map[string]interface{}, error) {
args := make(map[string]interface{})
for {
// Parse key.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != IDENT {
return nil, parseErrorf(pos, "expected argument key, found %q", lit)
}
key := lit
// Expect '=' next.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok != EQ {
return nil, parseErrorf(pos, "expected equals sign, found %q", lit)
}
// Parse value.
var value interface{}
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
value = true
} else if lit == "false" {
value = false
} else if lit == "null" {
value = nil
} else {
value = lit
}
case STRING:
value = lit
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return nil, err
}
value = v
case LBRACK:
v, err := p.parseList()
if err != nil {
return nil, err
}
value = v
default:
return nil, parseErrorf(pos, "invalid argument value: %q", lit)
}
// Ensure key doesn't already exist.
if _, ok := args[key]; ok {
return nil, parseErrorf(pos, "argument key already used: %s", key)
}
// Add key/value pair to arguments.
args[key] = value
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
}
}
// parseList parses a list of primitives. This is used by the TopN() filters.
func (p *Parser) parseList() ([]interface{}, error) {
var values []interface{}
for {
@ -678,7 +233,7 @@ func (p *Parser) parseList() ([]interface{}, error) {
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
return nil, parseErrorf(pos, "expected comma, found %q", lit)
}
}
return values, nil
@ -703,6 +258,16 @@ func (p *Parser) unscan(n int) {
}
}
// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner.
func (p *Parser) unscanIgnoreWhitespace(n int) {
for i := 0; i < n; {
p.scanner.unscan()
if tok, _, _ := p.scanner.curr(); tok != WS {
i++
}
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
@ -711,25 +276,9 @@ func (p *Parser) expect(exp Token) error {
return nil
}
// expectIgnoreWhitespace returns an error if the next non-whitespace token is not exp.
func (p *Parser) expectIgnoreWhitespace(exp Token) error {
if tok, pos, lit := p.scanIgnoreWhitespace(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// arg represents an call argument.
// The key can be the index or the string key.
// The value can be a uint64, []uint64, string, Call, or Calls.
type arg struct {
key interface{}
value interface{}
}
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
@ -748,63 +297,3 @@ func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
Pos: pos,
}
}
// decodeInt type converts v to target.
func decodeInt(v interface{}, target *int) error {
if v, ok := v.(uint64); ok {
*target = int(v)
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64 type converts v to target.
func decodeUint64(v interface{}, target *uint64) error {
if v, ok := v.(uint64); ok {
*target = v
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64Slice type converts v to target.
func decodeUint64Slice(v interface{}, target *[]uint64) error {
input, ok := v.([]interface{})
if !ok {
return fmt.Errorf("invalid array value: %v", v)
}
a := make([]uint64, len(input))
for i := range input {
elem, ok := input[i].(uint64)
if !ok {
return fmt.Errorf("invalid int element: %v", input[i])
}
a[i] = elem
}
*target = a
return nil
}
// decodeString type converts v to target.
func decodeString(v interface{}, target *string) error {
if v, ok := v.(string); ok {
*target = v
return nil
}
return fmt.Errorf("invalid string value: %v", v)
}
// decodeDate type converts v to target.
func decodeDate(v interface{}, target *time.Time) error {
if v, ok := v.(string); ok {
t, err := time.Parse(TimeFormat, v)
if err != nil {
return fmt.Errorf("invalid date format: %s", v)
}
*target = t
return nil
}
return fmt.Errorf("invalid date value: %v", v)
}

420
pql/parser_test.go
View file

@ -3,349 +3,117 @@ package pql_test
import (
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/pql"
)
// Ensure the parser can parse a "Bitmap()" function with keyed args.
func TestParser_Parse_Bitmap_Key(t *testing.T) {
q, err := pql.ParseString(`Bitmap(id=1, frame="b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
// Ensure the parser can parse PQL.
func TestParser_Parse(t *testing.T) {
// Parse with no children or arguments.
t.Run("Empty", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap()`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "Bitmap",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
) {
t.Fatalf("unexpected call: %s", q.Calls[0])
}
})
// Ensure the parser can parse a "Bitmap()" function with array args.
func TestParser_Parse_Bitmap_Array(t *testing.T) {
q, err := pql.ParseString(`Bitmap(1, "b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "ClearBit()" function with keyed args.
func TestParser_Parse_ClearBit_Key(t *testing.T) {
q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", profileID = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "ClearBit()" function with array args.
func TestParser_Parse_ClearBit_Array(t *testing.T) {
q, err := pql.ParseString(`ClearBit(1, "b.n", 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "count()" function.
func TestParser_Parse_Count(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap(1))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Count{
Input: &pql.Bitmap{
ID: 1,
// Parse with only children.
t.Run("ChildrenOnly", func(t *testing.T) {
q, err := pql.ParseString(`Union( Bitmap() , Count() )`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "Union",
Children: []*pql.Call{
&pql.Call{Name: "Bitmap"},
&pql.Call{Name: "Count"},
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
) {
t.Fatalf("unexpected call: %s", q.Calls[0])
}
})
// Ensure the parser can parse a "difference()" function.
func TestParser_Parse_Difference(t *testing.T) {
q, err := pql.ParseString(`Difference(Bitmap(1), Bitmap(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
// Parse a single child with a single argument.
t.Run("ChildWithArgument", func(t *testing.T) {
q, err := pql.ParseString(`Count( Bitmap( id=100))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "Count",
Children: []*pql.Call{
{Name: "Bitmap", Args: map[string]interface{}{"id": uint64(100)}},
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
) {
t.Fatalf("unexpected call: %s", q.Calls[0])
}
})
// Ensure the parser can parse a "intersect()" function.
func TestParser_Parse_Intersect(t *testing.T) {
q, err := pql.ParseString(`Intersect(Bitmap(1), Bitmap(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
// Parse with only arguments.
t.Run("ArgumentsOnly", func(t *testing.T) {
q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "MyCall",
Args: map[string]interface{}{
"key": "value",
"foo": "bar",
"age": uint64(12),
"bool0": true,
"bool1": false,
"x": nil,
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
// Ensure the parser can parse a "Profile()" function with keyed args.
func TestParser_Parse_Profile_Key(t *testing.T) {
q, err := pql.ParseString(`Profile(id=1)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "Profile()" function with array args.
func TestParser_Parse_Profile_Array(t *testing.T) {
q, err := pql.ParseString(`Profile(1)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "range()" function with keyed args.
func TestParser_Parse_Range_Key(t *testing.T) {
q, err := pql.ParseString(`Range(start="2000-01-02T03:04", id=20, frame="b.n", end="2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
// Parse with both child calls and arguments.
t.Run("ChildrenAndArguments", func(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Children: []*pql.Call{{
Name: "Bitmap",
Args: map[string]interface{}{"id": uint64(100), "frame": "other"},
}},
Args: map[string]interface{}{"n": uint64(3), "frame": "f"},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
// Ensure the parser can parse a "range()" function with array args.
func TestParser_Parse_Range_Array(t *testing.T) {
q, err := pql.ParseString(`Range(20, "b.n", "2000-01-02T03:04", "2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "SetBit()" function with keyed args.
func TestParser_Parse_SetBit_Key(t *testing.T) {
q, err := pql.ParseString(`SetBit(id=1, frame="b.n", profileID = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "SetBit()" function with array args.
func TestParser_Parse_SetBit_Array(t *testing.T) {
q, err := pql.ParseString(`SetBit(1, "b.n", 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "SetBitmapAttrs()" function with keyed args.
func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) {
q, err := pql.ParseString(`SetBitmapAttrs(id=1, frame="b.n", foo="bar", bar=123, baz=true, bat=false, x=null)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
// Parse a list argument.
t.Run("ListArgument", func(t *testing.T) {
q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Args: map[string]interface{}{
"frame": "f",
"ids": []interface{}{uint64(0), uint64(10), uint64(30)},
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "SetBitmapAttrs()" function with array args.
func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
q, err := pql.ParseString(`SetBitmapAttrs(1, "b.n", foo=bar, bar=123)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "TopN()" function with keyed args.
func TestParser_Parse_TopN_Key(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(100), frame="b.n", n=2, ids=[1,2,3], field="XXX", [5,10,15])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
BitmapIDs: []uint64{1, 2, 3},
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, ids=[1,2,3], field="XXX", [5,10,15])` {
t.Fatalf("unexpected string encoding: %s", s)
}
}
// Ensure the parser can parse a "TopN()" function with array args.
func TestParser_Parse_TopN_Array(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(100), "b.n", 2, "XXX", ["foo",true,false])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", ["foo",true,false])` {
t.Fatalf("unexpected string encoding: %s", s)
}
}
// Ensure the parser can parse a "union()" function.
func TestParser_Parse_Union(t *testing.T) {
q, err := pql.ParseString(`Union(Bitmap(1), Bitmap(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Calls: pql.Calls{
&pql.Union{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
}

6
pql/scanner.go
View file

@ -197,7 +197,7 @@ type bufScanner struct {
s *Scanner
i int // buffer index
n int // buffer size
buf [3]struct {
buf [8]struct {
tok Token
pos Pos
lit string
@ -250,7 +250,9 @@ func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && c
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool { return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' }
func isIdentChar(ch rune) bool {
return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
}
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }