diff --git a/executor.go b/executor.go index 3a8152ff5..53ecc3521 100644 --- a/executor.go +++ b/executor.go @@ -80,31 +80,30 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices } // executeCall executes a call. -func (e *Executor) executeCall(ctx context.Context, db string, c pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { - switch c := c.(type) { - case pql.BitmapCall: - return e.executeBitmapCall(ctx, db, c, slices, opt) - case *pql.ClearBit: +func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { + // Special handling for mutation and top-n calls. + switch c.Name { + case "ClearBit": return e.executeClearBit(ctx, db, c, opt) - case *pql.Count: + case "Count": return e.executeCount(ctx, db, c, slices, opt) - case *pql.Profile: + case "Profile": return e.executeProfile(ctx, db, c, opt) - case *pql.SetBit: + case "SetBit": return e.executeSetBit(ctx, db, c, opt) - case *pql.SetBitmapAttrs: + case "SetBitmapAttrs": return nil, e.executeSetBitmapAttrs(ctx, db, c, opt) - case *pql.SetProfileAttrs: + case "SetProfileAttrs": return nil, e.executeSetProfileAttrs(ctx, db, c, opt) - case *pql.TopN: + case "TopN": return e.executeTopN(ctx, db, c, slices, opt) default: - panic("unreachable") + return e.executeBitmapCall(ctx, db, c, slices, opt) } } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.BitmapCall, slices []uint64, opt *ExecOptions) (*Bitmap, error) { +func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { return e.executeBitmapCallSlice(ctx, db, c, slice) @@ -127,10 +126,13 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.Bitma // Attach bitmap attributes for Bitmap() calls. bm, _ := other.(*Bitmap) - if c, ok := c.(*pql.Bitmap); ok { - fr := e.Index.Frame(db, c.Frame) + if c.Name == "Bitmap" { + id, _ := c.Args["id"].(uint64) + frame, _ := c.Args["frame"].(string) + + fr := e.Index.Frame(db, frame) if fr != nil { - attrs, err := fr.BitmapAttrStore().Attrs(c.ID) + attrs, err := fr.BitmapAttrStore().Attrs(id) if err != nil { return nil, err } @@ -142,27 +144,29 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c pql.Bitma } // executeBitmapCallSlice executes a bitmap call for a single slice. -func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c pql.BitmapCall, slice uint64) (*Bitmap, error) { - switch c := c.(type) { - case *pql.Bitmap: +func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { + switch c.Name { + case "Bitmap": return e.executeBitmapSlice(ctx, db, c, slice) - case *pql.Difference: + case "Difference": return e.executeDifferenceSlice(ctx, db, c, slice) - case *pql.Intersect: + case "Intersect": return e.executeIntersectSlice(ctx, db, c, slice) - case *pql.Range: + case "Range": return e.executeRangeSlice(ctx, db, c, slice) - case *pql.Union: + case "Union": return e.executeUnionSlice(ctx, db, c, slice) default: - panic("unreachable") + return nil, fmt.Errorf("unknown call: %s", c.Name) } } // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { + bitmapIDs, _ := c.Args["ids"].([]uint64) + // Execute original query. pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt) if err != nil { @@ -171,20 +175,24 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.TopN, slic // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(c.BitmapIDs) > 0 || opt.Remote { + if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote { return pairs, nil } // Only the original caller should refetch the full counts. - other := *c - other.N = 0 - other.BitmapIDs = Pairs(pairs).Keys() - sort.Sort(uint64Slice(other.BitmapIDs)) + other := c.Clone() + other.Args["n"] = 0 - return e.executeTopNSlices(ctx, db, &other, slices, opt) + ids := Pairs(pairs).Keys() + sort.Sort(uint64Slice(ids)) + other.Args["ids"] = ids + + return e.executeTopNSlices(ctx, db, other, slices, opt) } -func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { + n, _ := c.Args["n"].(uint64) + // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { return e.executeTopNSlice(ctx, db, c, slice) @@ -206,27 +214,34 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.TopN sort.Sort(Pairs(results)) // Only keep the top n after sorting. - if c.N > 0 && len(results) > c.N { - results = results[0:c.N] + if n > 0 && len(results) > int(n) { + results = results[0:n] } return results, nil } // executeTopNSlice executes a TopN call for a single slice. -func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.TopN, slice uint64) ([]Pair, error) { +func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) { + frame, _ := c.Args["frame"].(string) + n, _ := c.Args["n"].(uint64) + field, _ := c.Args["field"].(string) + bitmapIDs, _ := c.Args["ids"].([]uint64) + filters, _ := c.Args["filters"].([]interface{}) + // Retrieve bitmap used to intersect. var src *Bitmap - if c.Src != nil { - bm, err := e.executeBitmapCallSlice(ctx, db, c.Src, slice) + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice) if err != nil { return nil, err } src = bm + } else if len(c.Children) > 1 { + return nil, errors.New("TopN() can only have one input bitmap") } // Set default frame. - frame := c.Frame if frame == "" { frame = DefaultFrame } @@ -237,18 +252,18 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.TopN, } return f.Top(TopOptions{ - N: c.N, + N: int(n), Src: src, - BitmapIDs: c.BitmapIDs, - FilterField: c.Field, - FilterValues: c.Filters, + BitmapIDs: bitmapIDs, + FilterField: field, + FilterValues: filters, }) } // executeDifferenceSlice executes a difference() call for a local slice. -func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Difference, slice uint64) (*Bitmap, error) { +func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { var other *Bitmap - for i, input := range c.Inputs { + for i, input := range c.Children { bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) if err != nil { return nil, err @@ -264,8 +279,9 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql return other, nil } -func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Bitmap, slice uint64) (*Bitmap, error) { - frame := c.Frame +func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { + id, _ := c.Args["id"].(uint64) + frame, _ := c.Args["frame"].(string) if frame == "" { frame = DefaultFrame } @@ -274,13 +290,13 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Bit if f == nil { return NewBitmap(), nil } - return f.Bitmap(c.ID), nil + return f.Bitmap(id), nil } // executeIntersectSlice executes a intersect() call for a local slice. -func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Intersect, slice uint64) (*Bitmap, error) { +func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { var other *Bitmap - for i, input := range c.Inputs { + for i, input := range c.Children { bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) if err != nil { return nil, err @@ -297,8 +313,31 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql. } // executeRangeSlice executes a range() call for a local slice. -func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Range, slice uint64) (*Bitmap, error) { - frame := c.Frame +func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { + id, _ := c.Args["id"].(uint64) + + // Parse start time. + startTimeStr, ok := c.Args["start"].(string) + if !ok { + return nil, errors.New("Range() start time required") + } + startTime, err := time.Parse(TimeFormat, startTimeStr) + if err != nil { + return nil, errors.New("cannot parse Range() start time") + } + + // Parse end time. + endTimeStr, _ := c.Args["end"].(string) + if !ok { + return nil, errors.New("Range() end time required") + } + endTime, err := time.Parse(TimeFormat, endTimeStr) + if err != nil { + return nil, errors.New("cannot parse Range() end time") + } + + // Parse frame, use default if unset. + frame, _ := c.Args["frame"].(string) if frame == "" { frame = DefaultFrame } @@ -317,20 +356,20 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang // Union bitmaps across all time-based subframes. bm := &Bitmap{} - for _, subframe := range FramesByTimeRange(frame, c.StartTime, c.EndTime, q) { + for _, subframe := range FramesByTimeRange(frame, startTime, endTime, q) { f := e.Index.Fragment(db, subframe, slice) if f == nil { continue } - bm = bm.Union(f.Bitmap(c.ID)) + bm = bm.Union(f.Bitmap(id)) } return bm, nil } // executeUnionSlice executes a union() call for a local slice. -func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Union, slice uint64) (*Bitmap, error) { +func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { var other *Bitmap - for i, input := range c.Inputs { + for i, input := range c.Children { bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) if err != nil { return nil, err @@ -347,10 +386,16 @@ func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Unio } // executeCount executes a count() call. -func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Count, slices []uint64, opt *ExecOptions) (uint64, error) { +func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { + if len(c.Children) == 0 { + return 0, errors.New("Count() requires an input bitmap") + } else if len(c.Children) > 1 { + return 0, errors.New("Count() only accepts a single bitmap input") + } + // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - bm, err := e.executeBitmapCallSlice(ctx, db, c.Input, slice) + bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice) if err != nil { return 0, err } @@ -374,23 +419,38 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Count, sl // executeProfile executes a Profile() call. // This call only executes locally since the profile attibutes are stored locally. -func (e *Executor) executeProfile(ctx context.Context, db string, c *pql.Profile, opt *ExecOptions) (*Profile, error) { +func (e *Executor) executeProfile(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (*Profile, error) { panic("FIXME: impl: e.Index.ProfileAttr(c.ID)") } // executeClearBit executes a ClearBit() call. -func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearBit, opt *ExecOptions) (bool, error) { - slice := c.ProfileID / SliceWidth +func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) { + frame, ok := c.Args["frame"].(string) + if !ok { + return false, errors.New("ClearBit() frame required") + } + + id, ok := c.Args["id"].(uint64) + if !ok { + return false, errors.New("ClearBit() id required") + } + + profileID, ok := c.Args["profileID"].(uint64) + if !ok { + return false, errors.New("ClearBit() profileID required") + } + + slice := profileID / SliceWidth ret := false for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - f := e.Index.Fragment(db, c.Frame, slice) + f := e.Index.Fragment(db, frame, slice) if f == nil { return false, nil } - val, err := f.ClearBit(c.ID, c.ProfileID) + val, err := f.ClearBit(id, profileID) if err != nil { return false, err } else if val { @@ -404,7 +464,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearB } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil { + if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -414,8 +474,23 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.ClearB } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, opt *ExecOptions) (bool, error) { - slice := c.ProfileID / SliceWidth +func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) { + frame, ok := c.Args["frame"].(string) + if !ok { + return false, errors.New("SetBit() frame required") + } + + id, ok := c.Args["id"].(uint64) + if !ok { + return false, errors.New("SetBit() id required") + } + + profileID, ok := c.Args["profileID"].(uint64) + if !ok { + return false, errors.New("SetBit() profileID required") + } + + slice := profileID / SliceWidth ret := false for _, node := range e.Cluster.FragmentNodes(db, slice) { @@ -425,7 +500,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, if err != nil { return false, fmt.Errorf("db: %s", err) } - val, err := db.SetBit(c.Frame, c.ID, c.ProfileID, opt.Timestamp) + val, err := db.SetBit(frame, id, profileID, opt.Timestamp) if err != nil { return false, err } else if val { @@ -440,7 +515,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil { + if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -450,15 +525,30 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, } // executeSetBitmapAttrs executes a SetBitmapAttrs() call. -func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.SetBitmapAttrs, opt *ExecOptions) error { +func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { + frameName, ok := c.Args["frame"].(string) + if !ok { + return errors.New("SetBitmapAttrs() frame required") + } + + id, ok := c.Args["id"].(uint64) + if !ok { + return errors.New("SetBitmapAttrs() id required") + } + + // Copy args and remove reserved fields. + attrs := pql.CopyArgs(c.Args) + delete(attrs, "frame") + delete(attrs, "id") + // Retrieve frame. - frame, err := e.Index.CreateFrameIfNotExists(db, c.Frame) + frame, err := e.Index.CreateFrameIfNotExists(db, frameName) if err != nil { return err } // Set attributes. - if err := frame.BitmapAttrStore().SetAttrs(c.ID, c.Attrs); err != nil { + if err := frame.BitmapAttrStore().SetAttrs(id, attrs); err != nil { return err } @@ -472,7 +562,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt) + _, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -488,25 +578,38 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. } // executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls. -func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls pql.Calls, opt *ExecOptions) ([]interface{}, error) { +func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { // Collect attributes by frame/id. m := make(map[string]map[uint64]map[string]interface{}) - for _, call := range calls { - c := call.(*pql.SetBitmapAttrs) + for _, c := range calls { + frame, ok := c.Args["frame"].(string) + if !ok { + return nil, errors.New("SetBitmapAttrs() frame required") + } + + id, ok := c.Args["id"].(uint64) + if !ok { + return nil, errors.New("SetBitmapAttrs() id required") + } + + // Copy args and remove reserved fields. + attrs := pql.CopyArgs(c.Args) + delete(attrs, "frame") + delete(attrs, "id") // Create frame group, if not exists. - frameMap := m[c.Frame] + frameMap := m[frame] if frameMap == nil { frameMap = make(map[uint64]map[string]interface{}) - m[c.Frame] = frameMap + m[frame] = frameMap } // Set or merge attributes. - attr := frameMap[c.ID] + attr := frameMap[id] if attr == nil { - frameMap[c.ID] = cloneAttrs(c.Attrs) + frameMap[id] = cloneAttrs(attrs) } else { - for k, v := range c.Attrs { + for k, v := range attrs { attr[k] = v } } @@ -553,7 +656,16 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal } // executeSetProfileAttrs executes a SetProfileAttrs() call. -func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.SetProfileAttrs, opt *ExecOptions) error { +func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { + id, ok := c.Args["id"].(uint64) + if !ok { + return errors.New("SetProfileAttrs() id required") + } + + // Copy args and remove reserved fields. + attrs := pql.CopyArgs(c.Args) + delete(attrs, "id") + // Retrieve database. d, err := e.Index.CreateDBIfNotExists(db) if err != nil { @@ -561,7 +673,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql } // Set attributes. - if err := d.ProfileAttrStore().SetAttrs(c.ID, c.Attrs); err != nil { + if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil { return err } @@ -575,7 +687,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt) + _, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -656,21 +768,19 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query var v interface{} var err error - switch call.(type) { - case pql.BitmapCall: - v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil - case *pql.TopN: + switch call.Name { + case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil - case *pql.Count: + case "Count": v, err = pb.Results[i].N, nil - case *pql.SetBit: + case "SetBit": v, err = pb.Results[i].Changed, nil - case *pql.ClearBit: + case "ClearBit": v, err = pb.Results[i].Changed, nil - case *pql.SetBitmapAttrs: - case *pql.SetProfileAttrs: + case "SetBitmapAttrs": + case "SetProfileAttrs": default: - panic(fmt.Sprintf("invalid node for remote exec: %T", call)) + v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil } if err != nil { return nil, err @@ -703,7 +813,7 @@ loop: // // If a mapping of slices to a node fails then the slices are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. -func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { +func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { ch := make(chan mapResponse, 0) // Wrap context with a cancel to kill goroutines on exit. @@ -761,7 +871,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c } } -func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { +func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { // Group slices together by nodes. m, err := e.slicesByNode(nodes, db, slices) if err != nil { @@ -777,7 +887,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.Host == e.Host { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.exec(ctx, n, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt) + results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] } @@ -864,13 +974,13 @@ func decodeError(s string) error { } // hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls. -func hasOnlySetBitmapAttrs(calls pql.Calls) bool { +func hasOnlySetBitmapAttrs(calls []*pql.Call) bool { if len(calls) == 0 { return false } for _, call := range calls { - if _, ok := call.(*pql.SetBitmapAttrs); !ok { + if call.Name != "SetBitmapAttrs" { return false } } diff --git a/executor_test.go b/executor_test.go index 164d35acf..2be36d561 100644 --- a/executor_test.go +++ b/executor_test.go @@ -287,7 +287,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `Bitmap(id=10, frame=f)` { + } else if query.String() != `Bitmap(frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{0}) { //TODO: this is incorrect because the calling node doesn't know about slice 2 t.Fatalf("unexpected slices: %+v", slices) @@ -359,7 +359,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `SetBit(id=10, frame=f, profileID=2)` { + } else if query.String() != `SetBit(frame="f", id=10, profileID=2)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -406,11 +406,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // slices and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: - if query.String() != `TopN(frame=f, n=3)` { + if query.String() != `TopN(frame="f", n=3)` { t.Fatalf("unexpected query(0): %s", query.String()) } case 1: - if query.String() != `TopN(frame=f, ids=[0,10,30])` { + if query.String() != `TopN(frame="f", ids=[0,10,30], n=0)` { t.Fatalf("unexpected query(1): %s", query.String()) } default: diff --git a/handler_test.go b/handler_test.go index cc86df4b8..49fa2160f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -93,9 +93,9 @@ func TestHandler_Query_Args_URL(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) + t.Fatalf("unexpected status code: %d", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) } @@ -118,7 +118,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ DB: "db0", - Query: "Count(Bitmap(100))", + Query: "Count(Bitmap(id=100))", Slices: []uint64{0, 1}, }) if err != nil { @@ -139,7 +139,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Ensure the handler returns an error when parsing bad arguments. func TestHandler_Query_Args_Err(t *testing.T) { w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("Bitmap(100)"))) + NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -155,7 +155,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -171,7 +171,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Count(Bitmap(100))")) + r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -196,7 +196,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" { @@ -228,7 +228,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { @@ -246,7 +246,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Bitmap(100)")) + r := MustNewHTTPRequest("POST", "/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -293,7 +293,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ DB: "d", - Query: "Bitmap(100)", + Query: "Bitmap(id=100)", Profiles: true, }) if err != nil { @@ -389,7 +389,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`Bitmap(100)`))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`Bitmap(id=100)`))) if w.Code != http.StatusInternalServerError { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { @@ -436,8 +436,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"function not found: bad_fn occurred at line 1, char 1"}`+"\n" { - t.Fatalf("unexpected body: %q", body) + } else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { + t.Fatalf("unexpected body: %s", body) } } diff --git a/pql/ast.go b/pql/ast.go index 0659e9d3d..300ebe8f9 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -1,16 +1,16 @@ package pql import ( + "bytes" "fmt" "sort" "strconv" "strings" - "time" ) // Query represents a PQL query. type Query struct { - Calls Calls + Calls []*Call } // String returns a string representation of the query. @@ -22,344 +22,119 @@ func (q *Query) String() string { return strings.Join(a, "\n") } -// Node represents any node in the AST. -type Node interface { - node() - String() string -} - -func (*Bitmap) node() {} -func (*ClearBit) node() {} -func (*Count) node() {} -func (*Difference) node() {} -func (*Intersect) node() {} -func (*Profile) node() {} -func (*Range) node() {} -func (*SetBit) node() {} -func (*SetBitmapAttrs) node() {} -func (*SetProfileAttrs) node() {} -func (*TopN) node() {} -func (*Union) node() {} - // Call represents a function call in the AST. -type Call interface { - Node - call() +type Call struct { + Name string + Args map[string]interface{} + Children []*Call } -func (*Bitmap) call() {} -func (*ClearBit) call() {} -func (*Count) call() {} -func (*Difference) call() {} -func (*Intersect) call() {} -func (*Profile) call() {} -func (*Range) call() {} -func (*SetBit) call() {} -func (*SetBitmapAttrs) call() {} -func (*SetProfileAttrs) call() {} -func (*TopN) call() {} -func (*Union) call() {} - -// Calls represents a list of calls. -type Calls []Call - -// String returns a string representation of the calls as a comma-delimited list. -func (a Calls) String() string { - args := make([]string, len(a)) - for i, c := range a { - args[i] = c.String() +// Keys returns a list of argument keys in sorted order. +func (c *Call) Keys() []string { + a := make([]string, 0, len(c.Args)) + for k := range c.Args { + a = append(a, k) } - return strings.Join(args, ", ") + sort.Strings(a) + return a } -// BitmapCall represents a function call that returns a bitmap. -type BitmapCall interface { - Call - bitmapCall() -} - -// BitmapCalls represents a list of bitmap calls. -type BitmapCalls []BitmapCall - -// String returns a string representation of the calls as a comma-delimited list. -func (a BitmapCalls) String() string { - args := make([]string, len(a)) - for i, c := range a { - args[i] = c.String() +// Clone returns a copy of c. +func (c *Call) Clone() *Call { + if c == nil { + return nil } - return strings.Join(args, ", ") -} -func (*Difference) bitmapCall() {} -func (*Bitmap) bitmapCall() {} -func (*Intersect) bitmapCall() {} -func (*Range) bitmapCall() {} -func (*Union) bitmapCall() {} - -// Bitmap represents a Bitmap() function call. -type Bitmap struct { - ID uint64 - Frame string + other := &Call{ + Name: c.Name, + Args: CopyArgs(c.Args), + } + if c.Children != nil { + other.Children = make([]*Call, len(c.Children)) + for i := range c.Children { + other.Children[i] = c.Children[i].Clone() + } + } + return other } // String returns the string representation of the call. -func (c *Bitmap) String() string { - args := make([]string, 0, 2) - if c.ID != 0 { - args = append(args, fmt.Sprintf("id=%d", c.ID)) - } - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) - } - return fmt.Sprintf("Bitmap(%s)", strings.Join(args, ", ")) -} +func (c *Call) String() string { + var buf bytes.Buffer -// ClearBit represents a ClearBit() function call. -type ClearBit struct { - ID uint64 - Frame string - ProfileID uint64 -} - -// String returns the string representation of the call. -func (c *ClearBit) String() string { - args := make([]string, 0, 4) - args = append(args, fmt.Sprintf("id=%d", c.ID)) - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) - } - if c.ProfileID != 0 { - args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID)) - } - return fmt.Sprintf("ClearBit(%s)", strings.Join(args, ", ")) -} - -// Count represents a count() function call. -type Count struct { - Input BitmapCall -} - -// String returns the string representation of the call. -func (c *Count) String() string { - return fmt.Sprintf("Count(%s)", c.Input.String()) -} - -// Difference represents an difference() function call. -type Difference struct { - Inputs BitmapCalls -} - -// String returns the string representation of the call. -func (c *Difference) String() string { - return fmt.Sprintf("Difference(%s)", c.Inputs.String()) -} - -// Intersect represents an intersect() function call. -type Intersect struct { - Inputs BitmapCalls -} - -// String returns the string representation of the call. -func (c *Intersect) String() string { - return fmt.Sprintf("Intersect(%s)", c.Inputs.String()) -} - -// Profile represents a Profile() function call. -type Profile struct { - ID uint64 -} - -// String returns the string representation of the call. -func (c *Profile) String() string { - args := make([]string, 0, 1) - args = append(args, fmt.Sprintf("id=%d", c.ID)) - return fmt.Sprintf("Profile(%s)", strings.Join(args, ", ")) -} - -// Range represents a range() function call. -type Range struct { - ID uint64 - Frame string - StartTime time.Time - EndTime time.Time -} - -// String returns the string representation of the call. -func (c *Range) String() string { - args := make([]string, 0, 2) - if c.ID != 0 { - args = append(args, fmt.Sprintf("id=%d", c.ID)) - } - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) - } - if !c.StartTime.IsZero() { - args = append(args, fmt.Sprintf("start=%s", c.StartTime.Format(TimeFormat))) - } - if !c.EndTime.IsZero() { - args = append(args, fmt.Sprintf("end=%s", c.EndTime.Format(TimeFormat))) - } - return fmt.Sprintf("Range(%s)", strings.Join(args, ", ")) -} - -// SetBit represents a SetBit() function call. -type SetBit struct { - ID uint64 - Frame string - ProfileID uint64 -} - -// String returns the string representation of the call. -func (c *SetBit) String() string { - args := make([]string, 0, 2) - args = append(args, fmt.Sprintf("id=%d", c.ID)) - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) - } - if c.ProfileID != 0 { - args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID)) - } - return fmt.Sprintf("SetBit(%s)", strings.Join(args, ", ")) -} - -// SetBitmapAttrs represents a SetBitmapAttrs() function call. -type SetBitmapAttrs struct { - ID uint64 - Frame string - Attrs map[string]interface{} -} - -// String returns the string representation of the call. -func (c *SetBitmapAttrs) String() string { - args := make([]string, 0, 2) - args = append(args, fmt.Sprintf("id=%d", c.ID)) - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) + // Write name. + if c.Name != "" { + buf.WriteString(c.Name) + } else { + buf.WriteString("!UNNAMED") } - // Sort keys. - keys := make([]string, 0, len(c.Attrs)) - for k := range c.Attrs { - keys = append(keys, k) - } - sort.Strings(keys) + // Write opening. + buf.WriteByte('(') - // Write key/value pairs. - for _, k := range keys { - if c.Attrs[k] == nil { - args = append(args, fmt.Sprintf("%s=null", k)) - continue + // Write child list. + for i, child := range c.Children { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(child.String()) + } + + // Separate children and args, if necessary. + if len(c.Children) > 0 && len(c.Args) > 0 { + buf.WriteString(", ") + } + + // Write arguments in key order. + for i, key := range c.Keys() { + if i > 0 { + buf.WriteString(", ") } - switch v := c.Attrs[k].(type) { + switch v := c.Args[key].(type) { case string: - args = append(args, fmt.Sprintf("%s=\"%s\"", k, v)) + fmt.Fprintf(&buf, "%v=%q", key, v) + case []interface{}: + fmt.Fprintf(&buf, "%v=%s", key, joinInterfaceSlice(v)) + case []uint64: + fmt.Fprintf(&buf, "%v=%s", key, joinUint64Slice(v)) default: - args = append(args, fmt.Sprintf("%s=%v", k, v)) + fmt.Fprintf(&buf, "%v=%v", key, v) } } - return fmt.Sprintf("SetBitmapAttrs(%s)", strings.Join(args, ", ")) + // Write closing. + buf.WriteByte(')') + + return buf.String() } -// SetProfileAttrs represents a SetProfileAttrs() function call. -type SetProfileAttrs struct { - ID uint64 - Attrs map[string]interface{} -} - -// String returns the string representation of the call. -func (c *SetProfileAttrs) String() string { - args := make([]string, 0, 2) - args = append(args, fmt.Sprintf("id=%d", c.ID)) - - // Sort keys. - keys := make([]string, 0, len(c.Attrs)) - for k := range c.Attrs { - keys = append(keys, k) +// CopyArgs returns a copy of m. +func CopyArgs(m map[string]interface{}) map[string]interface{} { + other := make(map[string]interface{}, len(m)) + for k, v := range m { + other[k] = v } - sort.Strings(keys) + return other +} - // Write key/value pairs. - for _, k := range keys { - if c.Attrs[k] == nil { - args = append(args, fmt.Sprintf("%s=null", k)) - continue - } - - switch v := c.Attrs[k].(type) { +func joinInterfaceSlice(a []interface{}) string { + other := make([]string, len(a)) + for i := range a { + switch v := a[i].(type) { case string: - args = append(args, fmt.Sprintf("%s=\"%s\"", k, v)) + other[i] = fmt.Sprintf("%q", v) default: - args = append(args, fmt.Sprintf("%s=%v", k, v)) + other[i] = fmt.Sprintf("%v", v) } } - - return fmt.Sprintf("SetProfileAttrs(%s)", strings.Join(args, ", ")) + return "[" + strings.Join(other, ",") + "]" } -// TopN represents a TopN() function call. -type TopN struct { - Frame string - - // Maximum number of results to return. - N int - - // Bitmap to use for intersection while computing top results. - // Original bitmap counts are used if no Src is provided. - Src BitmapCall - - // Specific bitmaps to retrieve. - BitmapIDs []uint64 - - // Field name and values to filter on. - Field string - Filters []interface{} -} - -// String returns the string representation of the call. -func (c *TopN) String() string { - args := make([]string, 0, 2) - if c.Src != nil { - args = append(args, c.Src.String()) - } - if c.Frame != "" { - args = append(args, fmt.Sprintf("frame=%s", c.Frame)) - } - if c.N > 0 { - args = append(args, fmt.Sprintf("n=%d", c.N)) - } - if len(c.BitmapIDs) > 0 { - strs := make([]string, len(c.BitmapIDs)) - for i := range c.BitmapIDs { - strs[i] = strconv.FormatUint(c.BitmapIDs[i], 10) - } - args = append(args, fmt.Sprintf("ids=[%s]", strings.Join(strs, ","))) - } - if c.Field != "" { - args = append(args, fmt.Sprintf("field=%q", c.Field)) - } - if len(c.Filters) > 0 { - filters := make([]string, 0, len(c.Filters)) - for i := range c.Filters { - switch filter := c.Filters[i].(type) { - case string: - filters = append(filters, fmt.Sprintf("%q", filter)) - default: - filters = append(filters, fmt.Sprintf("%v", filter)) - } - } - args = append(args, fmt.Sprintf("[%s]", strings.Join(filters, ","))) - } - return fmt.Sprintf("TopN(%s)", strings.Join(args, ", ")) -} - -// Union represents a union() function call. -type Union struct { - Inputs BitmapCalls -} - -// String returns the string representation of the call. -func (c *Union) String() string { - return fmt.Sprintf("Union(%s)", c.Inputs.String()) +func joinUint64Slice(a []uint64) string { + other := make([]string, len(a)) + for i := range a { + other[i] = strconv.FormatUint(a[i], 10) + } + return "[" + strings.Join(other, ",") + "]" } diff --git a/pql/ast_test.go b/pql/ast_test.go index aa2e68e79..653bd5af5 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -2,103 +2,16 @@ package pql_test import ( "testing" - "time" "github.com/pilosa/pilosa/pql" ) -// Ensure the Bitmap call can be converted into a string. -func TestBitmap_String(t *testing.T) { - s := (&pql.Bitmap{ID: 1, Frame: "x.n"}).String() - if s != `Bitmap(id=1, frame=x.n)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the ClearBit call can be converted into a string. -func TestClearBit_String(t *testing.T) { - s := (&pql.ClearBit{ID: 1, Frame: "x.n", ProfileID: 3}).String() - if s != `ClearBit(id=1, frame=x.n, profileID=3)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Count call can be converted into a string. -func TestCount_String(t *testing.T) { - s := (&pql.Count{Input: &pql.Bitmap{ID: 1, Frame: "x.n"}}).String() - if s != `Count(Bitmap(id=1, frame=x.n))` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Difference call can be converted into a string. -func TestDifference_String(t *testing.T) { - s := (&pql.Difference{Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1, Frame: "x.n"}, - &pql.Bitmap{ID: 2}, - }, - }).String() - if s != `Difference(Bitmap(id=1, frame=x.n), Bitmap(id=2))` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Intersect call can be converted into a string. -func TestIntersect_String(t *testing.T) { - s := (&pql.Intersect{Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1, Frame: "x.n"}, - &pql.Bitmap{ID: 2}, - }, - }).String() - if s != `Intersect(Bitmap(id=1, frame=x.n), Bitmap(id=2))` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Profile call can be converted into a string. -func TestProfile_String(t *testing.T) { - if s := (&pql.Profile{ID: 1}).String(); s != `Profile(id=1)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Range call can be converted into a string. -func TestRange_String(t *testing.T) { - s := (&pql.Range{ - ID: 1, - Frame: "x.n", - StartTime: time.Unix(0, 0).UTC(), - EndTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), - }).String() - if s != `Range(id=1, frame=x.n, start=1970-01-01T00:00, end=2000-01-02T03:04)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the SetBit call can be converted into a string. -func TestSetBit_String(t *testing.T) { - s := (&pql.SetBit{ID: 1, Frame: "x.n", ProfileID: 3}).String() - if s != `SetBit(id=1, frame=x.n, profileID=3)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the SetBitmapAttrs call can be converted into a string. -func TestSetBitmapAttrs_String(t *testing.T) { - s := (&pql.SetBitmapAttrs{ID: 1, Frame: "x.n", Attrs: map[string]interface{}{"foo": "bar", "baz": 123, "bat": true, "x": nil}}).String() - if s != `SetBitmapAttrs(id=1, frame=x.n, bat=true, baz=123, foo="bar", x=null)` { - t.Fatalf("unexpected string: %s", s) - } -} - -// Ensure the Union call can be converted into a string. -func TestUnion_String(t *testing.T) { - s := (&pql.Union{Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1, Frame: "x.n"}, - &pql.Bitmap{ID: 2}, - }, - }).String() - if s != `Union(Bitmap(id=1, frame=x.n), Bitmap(id=2))` { - t.Fatalf("unexpected string: %s", s) - } +// Ensure call can be converted into a string. +func TestCall_String(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + c := &pql.Call{Name: "Bitmap"} + if s := c.String(); s != `Bitmap()` { + t.Fatalf("unexpected string: %s", s) + } + }) } diff --git a/pql/parser.go b/pql/parser.go index a1a40f793..0f1ff1e87 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -5,7 +5,6 @@ import ( "io" "strconv" "strings" - "time" ) // TimeFormat is the go-style time format used to parse string dates. @@ -50,604 +49,160 @@ func (p *Parser) Parse() (*Query, error) { } // parseCall parses the next function call. -func (p *Parser) parseCall() (Call, error) { +func (p *Parser) parseCall() (*Call, error) { + var c Call + + // Read call name. tok, pos, lit := p.scanIgnoreWhitespace() if tok == EOF { return nil, io.EOF } else if tok != IDENT { return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} } - - switch lit { - case "Bitmap": - return p.parseBitmapCall() - case "Count": - return p.parseCountCall() - case "ClearBit": - return p.parseClearBitCall() - case "Difference": - return p.parseDifferenceCall() - case "Intersect": - return p.parseIntersectCall() - case "Profile": - return p.parseProfileCall() - case "Range": - return p.parseRangeCall() - case "SetBit": - return p.parseSetBitCall() - case "SetBitmapAttrs": - return p.parseSetBitmapAttrsCall() - case "SetProfileAttrs": - return p.parseSetProfileAttrsCall() - case "TopN": - return p.parseTopNCall() - case "Union": - return p.parseUnionCall() - default: - return nil, &ParseError{Message: fmt.Sprintf("function not found: %s", lit), Pos: pos} - } -} - -// parseBitmapCall parses a Bitmap() function call. -func (p *Parser) parseBitmapCall() (*Bitmap, error) { - c := &Bitmap{} - pos := p.pos() + c.Name = lit // Scan opening parenthesis. if err := p.expect(LPAREN); err != nil { return nil, err } - // Parse arguments. + // Parse children first. + children, err := p.parseChildren() + if err != nil { + return nil, err + } + c.Children = children + + // If next token is a closing paren then exit. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + return &c, nil + } else if tok == IDENT { + p.unscan(1) + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + } + + // Parse key/value arguments. args, err := p.parseArgs() if err != nil { return nil, err } + c.Args = args - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - case 1, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid Bitmap() arg: %v", arg.key) - } + // Scan closing parenthesis. + if err := p.expect(RPAREN); err != nil { + return nil, err } - return c, nil + return &c, nil } -// parseClearBitCall parses a ClearBit() function call. -func (p *Parser) parseClearBitCall() (*ClearBit, error) { - c := &ClearBit{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - case 1, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - case 2, "profileID": - if err := decodeUint64(arg.value, &c.ProfileID); err != nil { - return nil, parseErrorf(pos, "profileID: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid ClearBit() arg: %v", arg.key) - } - } - - return c, nil -} - -// parseCount parses a Count() function call. -func (p *Parser) parseCountCall() (*Count, error) { - c := &Count{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } else if len(args) != 1 { - return nil, parseErrorf(pos, "count requires one argument") - } - - // Copy argument to AST. - input, ok := args[0].value.(BitmapCall) - if !ok { - return nil, parseErrorf(pos, "invalid count arg: %s", args[0].value) - } - c.Input = input - - return c, nil -} - -// parseDifference parses a Difference() function call. -func (p *Parser) parseDifferenceCall() (*Difference, error) { - c := &Difference{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - if v, ok := arg.value.(BitmapCall); ok { - c.Inputs = append(c.Inputs, v) - } else { - return nil, parseErrorf(pos, "invalid Difference() arg: %v", arg.value) - } - } - - return c, nil -} - -// parseIntersect parses a Intersect() function call. -func (p *Parser) parseIntersectCall() (*Intersect, error) { - c := &Intersect{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - if v, ok := arg.value.(BitmapCall); ok { - c.Inputs = append(c.Inputs, v) - } else { - return nil, parseErrorf(pos, "invalid Intersect() arg: %v", arg.value) - } - } - - return c, nil -} - -// parseProfileCall parses a Profile() function call. -func (p *Parser) parseProfileCall() (*Profile, error) { - c := &Profile{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid Profile() arg: %v", arg.key) - } - } - - return c, nil -} - -// parseRangeCall parses a Range() function call. -func (p *Parser) parseRangeCall() (*Range, error) { - c := &Range{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "start: %s", err) - } - case 1, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - case 2, "start": - if err := decodeDate(arg.value, &c.StartTime); err != nil { - return nil, parseErrorf(pos, "start: %s", err) - } - case 3, "end": - if err := decodeDate(arg.value, &c.EndTime); err != nil { - return nil, parseErrorf(pos, "end: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid Range() arg: %v", arg.key) - } - } - - return c, nil -} - -// parseSetBitCall parses a SetBit() function call. -func (p *Parser) parseSetBitCall() (*SetBit, error) { - c := &SetBit{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - case 1, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - case 2, "profileID": - if err := decodeUint64(arg.value, &c.ProfileID); err != nil { - return nil, parseErrorf(pos, "profileID: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid SetBit() arg: %v", arg.key) - } - } - - return c, nil -} - -// parseSetBitmapAttrsCall parses a SetBitmapAttrs() function call. -func (p *Parser) parseSetBitmapAttrsCall() (*SetBitmapAttrs, error) { - c := &SetBitmapAttrs{ - Attrs: make(map[string]interface{}), - } - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - case 1, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - default: - key, ok := arg.key.(string) - if !ok { - return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key) - } - - // Special handling for nil values. - if arg.value == nil { - c.Attrs[key] = nil - continue - } - - switch v := arg.value.(type) { - case string, bool: - c.Attrs[key] = v - case uint64: - c.Attrs[key] = v - default: - return nil, parseErrorf(pos, "invalid SetBitmapAttrs() arg: %v", arg.key) - } - } - } - - return c, nil -} - -// parseSetProfileAttrsCall parses a SetProfileAttrs() function call. -func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) { - c := &SetProfileAttrs{ - Attrs: make(map[string]interface{}), - } - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - switch arg.key { - case 0, "id": - if err := decodeUint64(arg.value, &c.ID); err != nil { - return nil, parseErrorf(pos, "id: %s", err) - } - default: - key, ok := arg.key.(string) - if !ok { - return nil, parseErrorf(pos, "invalid attr arg: %v", arg.key) - } - - // Special handling for nil values. - if arg.value == nil { - c.Attrs[key] = nil - continue - } - - switch v := arg.value.(type) { - case string, bool: - c.Attrs[key] = v - case uint64: - c.Attrs[key] = v - default: - return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key) - } - } - } - - return c, nil -} - -// parseTopNCall parses a TopN() function call. -func (p *Parser) parseTopNCall() (*TopN, error) { - c := &TopN{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - if v, ok := arg.value.(BitmapCall); ok { - c.Src = v - continue - } - - // Assign filter values if there's only a value and no named key. - if v, ok := arg.value.([]interface{}); ok { - if _, ok := arg.key.(string); !ok { - c.Filters = v - continue - } - } - - switch arg.key { - case 0, "frame": - if err := decodeString(arg.value, &c.Frame); err != nil { - return nil, parseErrorf(pos, "frame: %s", err) - } - case 1, "n": - if err := decodeInt(arg.value, &c.N); err != nil { - return nil, parseErrorf(pos, "n: %s", err) - } - case 2, "field": - if err := decodeString(arg.value, &c.Field); err != nil { - return nil, parseErrorf(pos, "n: %s", err) - } - case "ids": - if err := decodeUint64Slice(arg.value, &c.BitmapIDs); err != nil { - return nil, parseErrorf(pos, "n: %s", err) - } - default: - return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key) - } - } - - return c, nil -} - -// parseUnion parses a Union() function call. -func (p *Parser) parseUnionCall() (*Union, error) { - c := &Union{} - pos := p.pos() - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse arguments. - args, err := p.parseArgs() - if err != nil { - return nil, err - } - - // Copy arguments to AST. - for _, arg := range args { - if v, ok := arg.value.(BitmapCall); ok { - c.Inputs = append(c.Inputs, v) - } else { - return nil, parseErrorf(pos, "invalid Union() arg: %v", arg.value) - } - } - - return c, nil -} - -// parseArgs arguments to a function call. -func (p *Parser) parseArgs() ([]arg, error) { - var i int - var args []arg +// parseChildren parses call children. +func (p *Parser) parseChildren() ([]*Call, error) { + var offset int + var children []*Call for { - // Parse next argument. - arg, err := p.parseArg() + // Ensure next two tokens are IDENT+LPAREN. + if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { + p.unscanIgnoreWhitespace(1 + offset) + return children, nil + } + if tok, _, _ := p.scan(); tok != LPAREN { + p.unscanIgnoreWhitespace(2 + offset) + return children, nil + } + + // Push tokens back on scanner and parse as a call. + p.unscan(2) + child, err := p.parseCall() if err != nil { return nil, err } + children = append(children, child) - // If it's a primitive type without a key then index it. - if arg.key == nil { - switch arg.value.(type) { - case uint64, string: - arg.key = i - i++ - } - } - - // Append argument to list. - args = append(args, arg) - - // If next token is a closing parenthesis, then exit. - // Otherwise expect a comma. + // Exit if closing paren. if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - break - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected COMMA, found %q", lit) - } - } - - return args, nil -} - -// parseArg parses a single argument to a function call. -func (p *Parser) parseArg() (arg, error) { - var key, value interface{} - - // Read identifier and check if there's a following "=" or "(". - tok, pos, lit := p.scanIgnoreWhitespace() - switch tok { - case IDENT: - // If a left paren immediately follows then it's a function call. - if tok, _, _ := p.scan(); tok == LPAREN { - p.unscan(2) - c, err := p.parseCall() - if err != nil { - return arg{}, err - } - return arg{value: c}, nil - } - - // If it's not a left paren, rescan ignoring whitespace and look for "=", - p.unscan(1) - if tok, _, _ := p.scanIgnoreWhitespace(); tok == EQ { - key = lit // keyed arg - } else { p.unscan(1) + return children, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) } - default: - p.unscan(1) - } - // Read value token. - tok, pos, lit = p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - value = true - } else if lit == "false" { - value = false - } else if lit == "null" { - value = nil - } else { - value = lit - } - case STRING: - value = lit - case NUMBER: - v, err := strconv.ParseUint(lit, 10, 64) - if err != nil { - return arg{}, err - } - value = v - case LBRACK: - v, err := p.parseList() - if err != nil { - return arg{}, err - } - value = v - default: - return arg{}, parseErrorf(pos, "invalid value: %q", lit) + // Make sure comma is unscanned. + offset = 1 } - - return arg{key: key, value: value}, nil } -// parseListArg parses a list of primitives. This is used by the TopN() filters. +// parseArgs parses key/value arguments. +func (p *Parser) parseArgs() (map[string]interface{}, error) { + args := make(map[string]interface{}) + for { + // Parse key. + tok, pos, lit := p.scanIgnoreWhitespace() + if tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != IDENT { + return nil, parseErrorf(pos, "expected argument key, found %q", lit) + } + key := lit + + // Expect '=' next. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok != EQ { + return nil, parseErrorf(pos, "expected equals sign, found %q", lit) + } + + // Parse value. + var value interface{} + tok, pos, lit = p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + value = true + } else if lit == "false" { + value = false + } else if lit == "null" { + value = nil + } else { + value = lit + } + case STRING: + value = lit + case NUMBER: + v, err := strconv.ParseUint(lit, 10, 64) + if err != nil { + return nil, err + } + value = v + case LBRACK: + v, err := p.parseList() + if err != nil { + return nil, err + } + value = v + default: + return nil, parseErrorf(pos, "invalid argument value: %q", lit) + } + + // Ensure key doesn't already exist. + if _, ok := args[key]; ok { + return nil, parseErrorf(pos, "argument key already used: %s", key) + } + + // Add key/value pair to arguments. + args[key] = value + + // Exit if closing paren. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) + } + } +} + +// parseList parses a list of primitives. This is used by the TopN() filters. func (p *Parser) parseList() ([]interface{}, error) { var values []interface{} for { @@ -678,7 +233,7 @@ func (p *Parser) parseList() ([]interface{}, error) { if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { break } else if tok != COMMA { - return nil, parseErrorf(pos, "expected COMMA, found %q", lit) + return nil, parseErrorf(pos, "expected comma, found %q", lit) } } return values, nil @@ -703,6 +258,16 @@ func (p *Parser) unscan(n int) { } } +// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. +func (p *Parser) unscanIgnoreWhitespace(n int) { + for i := 0; i < n; { + p.scanner.unscan() + if tok, _, _ := p.scanner.curr(); tok != WS { + i++ + } + } +} + // expect returns an error if the next token is not exp. func (p *Parser) expect(exp Token) error { if tok, pos, lit := p.scan(); tok != exp { @@ -711,25 +276,9 @@ func (p *Parser) expect(exp Token) error { return nil } -// expectIgnoreWhitespace returns an error if the next non-whitespace token is not exp. -func (p *Parser) expectIgnoreWhitespace(exp Token) error { - if tok, pos, lit := p.scanIgnoreWhitespace(); tok != exp { - return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) - } - return nil -} - // pos returns the current position. func (p *Parser) pos() Pos { return p.scanner.pos() } -// arg represents an call argument. -// The key can be the index or the string key. -// The value can be a uint64, []uint64, string, Call, or Calls. -type arg struct { - key interface{} - value interface{} -} - // ParseError represents an error that occurred while parsing a PQL query. type ParseError struct { Message string @@ -748,63 +297,3 @@ func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { Pos: pos, } } - -// decodeInt type converts v to target. -func decodeInt(v interface{}, target *int) error { - if v, ok := v.(uint64); ok { - *target = int(v) - return nil - } - return fmt.Errorf("invalid int value: %v", v) -} - -// decodeUint64 type converts v to target. -func decodeUint64(v interface{}, target *uint64) error { - if v, ok := v.(uint64); ok { - *target = v - return nil - } - return fmt.Errorf("invalid int value: %v", v) -} - -// decodeUint64Slice type converts v to target. -func decodeUint64Slice(v interface{}, target *[]uint64) error { - input, ok := v.([]interface{}) - if !ok { - return fmt.Errorf("invalid array value: %v", v) - } - - a := make([]uint64, len(input)) - for i := range input { - elem, ok := input[i].(uint64) - if !ok { - return fmt.Errorf("invalid int element: %v", input[i]) - } - a[i] = elem - } - - *target = a - return nil -} - -// decodeString type converts v to target. -func decodeString(v interface{}, target *string) error { - if v, ok := v.(string); ok { - *target = v - return nil - } - return fmt.Errorf("invalid string value: %v", v) -} - -// decodeDate type converts v to target. -func decodeDate(v interface{}, target *time.Time) error { - if v, ok := v.(string); ok { - t, err := time.Parse(TimeFormat, v) - if err != nil { - return fmt.Errorf("invalid date format: %s", v) - } - *target = t - return nil - } - return fmt.Errorf("invalid date value: %v", v) -} diff --git a/pql/parser_test.go b/pql/parser_test.go index f303e9944..f4db973bc 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -3,349 +3,117 @@ package pql_test import ( "reflect" "testing" - "time" - "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/pql" ) -// Ensure the parser can parse a "Bitmap()" function with keyed args. -func TestParser_Parse_Bitmap_Key(t *testing.T) { - q, err := pql.ParseString(`Bitmap(id=1, frame="b.n")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Bitmap{ - ID: 1, - Frame: "b.n", +// Ensure the parser can parse PQL. +func TestParser_Parse(t *testing.T) { + // Parse with no children or arguments. + t.Run("Empty", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap()`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Bitmap", }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) -// Ensure the parser can parse a "Bitmap()" function with array args. -func TestParser_Parse_Bitmap_Array(t *testing.T) { - q, err := pql.ParseString(`Bitmap(1, "b.n")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Bitmap{ - ID: 1, - Frame: "b.n", - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "ClearBit()" function with keyed args. -func TestParser_Parse_ClearBit_Key(t *testing.T) { - q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", profileID = 3)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.ClearBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "ClearBit()" function with array args. -func TestParser_Parse_ClearBit_Array(t *testing.T) { - q, err := pql.ParseString(`ClearBit(1, "b.n", 3)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.ClearBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "count()" function. -func TestParser_Parse_Count(t *testing.T) { - q, err := pql.ParseString(`Count(Bitmap(1))`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Count{ - Input: &pql.Bitmap{ - ID: 1, + // Parse with only children. + t.Run("ChildrenOnly", func(t *testing.T) { + q, err := pql.ParseString(`Union( Bitmap() , Count() )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Union", + Children: []*pql.Call{ + &pql.Call{Name: "Bitmap"}, + &pql.Call{Name: "Count"}, }, }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) -// Ensure the parser can parse a "difference()" function. -func TestParser_Parse_Difference(t *testing.T) { - q, err := pql.ParseString(`Difference(Bitmap(1), Bitmap(2))`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Difference{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, + // Parse a single child with a single argument. + t.Run("ChildWithArgument", func(t *testing.T) { + q, err := pql.ParseString(`Count( Bitmap( id=100))`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Count", + Children: []*pql.Call{ + {Name: "Bitmap", Args: map[string]interface{}{"id": uint64(100)}}, }, }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) -// Ensure the parser can parse a "intersect()" function. -func TestParser_Parse_Intersect(t *testing.T) { - q, err := pql.ParseString(`Intersect(Bitmap(1), Bitmap(2))`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Intersect{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, + // Parse with only arguments. + t.Run("ArgumentsOnly", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": "value", + "foo": "bar", + "age": uint64(12), + "bool0": true, + "bool1": false, + "x": nil, }, }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) -// Ensure the parser can parse a "Profile()" function with keyed args. -func TestParser_Parse_Profile_Key(t *testing.T) { - q, err := pql.ParseString(`Profile(id=1)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Profile{ID: 1}, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "Profile()" function with array args. -func TestParser_Parse_Profile_Array(t *testing.T) { - q, err := pql.ParseString(`Profile(1)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Profile{ID: 1}, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "range()" function with keyed args. -func TestParser_Parse_Range_Key(t *testing.T) { - q, err := pql.ParseString(`Range(start="2000-01-02T03:04", id=20, frame="b.n", end="2001-01-02T03:04")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Range{ - ID: 20, - Frame: "b.n", - StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), - EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), + // Parse with both child calls and arguments. + t.Run("ChildrenAndArguments", func(t *testing.T) { + q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Children: []*pql.Call{{ + Name: "Bitmap", + Args: map[string]interface{}{"id": uint64(100), "frame": "other"}, + }}, + Args: map[string]interface{}{"n": uint64(3), "frame": "f"}, }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) -// Ensure the parser can parse a "range()" function with array args. -func TestParser_Parse_Range_Array(t *testing.T) { - q, err := pql.ParseString(`Range(20, "b.n", "2000-01-02T03:04", "2001-01-02T03:04")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Range{ - ID: 20, - Frame: "b.n", - StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), - EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "SetBit()" function with keyed args. -func TestParser_Parse_SetBit_Key(t *testing.T) { - q, err := pql.ParseString(`SetBit(id=1, frame="b.n", profileID = 3)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.SetBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "SetBit()" function with array args. -func TestParser_Parse_SetBit_Array(t *testing.T) { - q, err := pql.ParseString(`SetBit(1, "b.n", 3)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.SetBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "SetBitmapAttrs()" function with keyed args. -func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) { - q, err := pql.ParseString(`SetBitmapAttrs(id=1, frame="b.n", foo="bar", bar=123, baz=true, bat=false, x=null)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.SetBitmapAttrs{ - ID: 1, - Frame: "b.n", - Attrs: map[string]interface{}{ - "foo": "bar", - "bar": uint64(123), - "baz": true, - "bat": false, - "x": nil, + // Parse a list argument. + t.Run("ListArgument", func(t *testing.T) { + q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Args: map[string]interface{}{ + "frame": "f", + "ids": []interface{}{uint64(0), uint64(10), uint64(30)}, }, }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "SetBitmapAttrs()" function with array args. -func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) { - q, err := pql.ParseString(`SetBitmapAttrs(1, "b.n", foo=bar, bar=123)`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.SetBitmapAttrs{ - ID: 1, - Frame: "b.n", - Attrs: map[string]interface{}{ - "foo": "bar", - "bar": uint64(123), - }, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } -} - -// Ensure the parser can parse a "TopN()" function with keyed args. -func TestParser_Parse_TopN_Key(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(100), frame="b.n", n=2, ids=[1,2,3], field="XXX", [5,10,15])`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.TopN{ - Src: &pql.Bitmap{ID: 100}, - Frame: "b.n", - N: 2, - BitmapIDs: []uint64{1, 2, 3}, - Field: "XXX", - Filters: []interface{}{uint64(5), uint64(10), uint64(15)}, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } - - if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, ids=[1,2,3], field="XXX", [5,10,15])` { - t.Fatalf("unexpected string encoding: %s", s) - } -} - -// Ensure the parser can parse a "TopN()" function with array args. -func TestParser_Parse_TopN_Array(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(100), "b.n", 2, "XXX", ["foo",true,false])`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.TopN{ - Src: &pql.Bitmap{ID: 100}, - Frame: "b.n", - N: 2, - Field: "XXX", - Filters: []interface{}{"foo", true, false}, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } - - if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", ["foo",true,false])` { - t.Fatalf("unexpected string encoding: %s", s) - } -} - -// Ensure the parser can parse a "union()" function. -func TestParser_Parse_Union(t *testing.T) { - q, err := pql.ParseString(`Union(Bitmap(1), Bitmap(2))`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q, &pql.Query{ - Calls: pql.Calls{ - &pql.Union{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, - }, - }, - }, - }) { - t.Fatalf("unexpected query: %s", spew.Sdump(q)) - } + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) } diff --git a/pql/scanner.go b/pql/scanner.go index c3b846ba6..2b1add022 100644 --- a/pql/scanner.go +++ b/pql/scanner.go @@ -197,7 +197,7 @@ type bufScanner struct { s *Scanner i int // buffer index n int // buffer size - buf [3]struct { + buf [8]struct { tok Token pos Pos lit string @@ -250,7 +250,9 @@ func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && c func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } // isIdentChar returns true if the rune can be used in an unquoted identifier. -func isIdentChar(ch rune) bool { return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' } +func isIdentChar(ch rune) bool { + return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' +} // isIdentFirstChar returns true if the rune can be used as the first char in an identifier. func isIdentFirstChar(ch rune) bool { return isLetter(ch) }