diff --git a/cache.go b/cache.go index ccd5d3fb4..17a69f75b 100644 --- a/cache.go +++ b/cache.go @@ -286,3 +286,32 @@ type uint64Slice []uint64 func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p uint64Slice) Len() int { return len(p) } func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } + +// merge combines p and other to a unique sorted set of values. +// p and other must both have unique sets and be sorted. +func (p uint64Slice) merge(other []uint64) []uint64 { + ret := make([]uint64, 0, len(p)) + + i, j := 0, 0 + for i < len(p) && j < len(other) { + a, b := p[i], other[j] + if a == b { + ret = append(ret, a) + i, j = i+1, j+1 + } else if a < b { + ret = append(ret, a) + i++ + } else { + ret = append(ret, b) + j++ + } + } + + if i < len(p) { + ret = append(ret, p[i:]...) + } else if j < len(other) { + ret = append(ret, other[j:]...) + } + + return ret +} diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go index 6bdbcbef0..fab082030 100644 --- a/cmd/pilosa/main_test.go +++ b/cmd/pilosa/main_test.go @@ -40,9 +40,11 @@ func TestMain_Set_Quick(t *testing.T) { for frame, frameSet := range SetCommands(cmds).Frames() { for id, profileIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ - "result": map[string]interface{}{ - "bits": profileIDs, - "attrs": map[string]interface{}{}, + "results": []interface{}{ + map[string]interface{}{ + "bits": profileIDs, + "attrs": map[string]interface{}{}, + }, }, }) + "\n" if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { @@ -61,9 +63,11 @@ func TestMain_Set_Quick(t *testing.T) { for frame, frameSet := range SetCommands(cmds).Frames() { for id, profileIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ - "result": map[string]interface{}{ - "bits": profileIDs, - "attrs": map[string]interface{}{}, + "results": []interface{}{ + map[string]interface{}{ + "bits": profileIDs, + "attrs": map[string]interface{}{}, + }, }, }) + "\n" if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { @@ -110,14 +114,14 @@ func TestMain_SetBitmapAttrs(t *testing.T) { // Query bitmap x.n/1. if res, err := m.Query("db=d", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query bitmap x.n/2. if res, err := m.Query("db=d", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"result":{"attrs":{"x":200},"bits":[100]}}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -128,7 +132,7 @@ func TestMain_SetBitmapAttrs(t *testing.T) { // Query bitmap after reopening. if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } @@ -153,7 +157,7 @@ func TestMain_SetProfileAttrs(t *testing.T) { // Query bitmap. if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -164,7 +168,7 @@ func TestMain_SetProfileAttrs(t *testing.T) { // Query bitmap after reopening. if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } diff --git a/executor.go b/executor.go index 39f305dbd..9ded8b5c9 100644 --- a/executor.go +++ b/executor.go @@ -42,7 +42,7 @@ func NewExecutor(index *Index) *Executor { func (e *Executor) Index() *Index { return e.index } // Execute executes a PQL query. -func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) { +func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Verify that a database is set. if db == "" { return nil, ErrDatabaseRequired @@ -53,16 +53,6 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp opt = &ExecOptions{} } - // Ignore slices for set calls. - switch root := q.Root.(type) { - case *pql.SetBit: - return e.executeSetBit(db, root, opt) - case *pql.SetBitmapAttrs: - return nil, e.executeSetBitmapAttrs(db, root) - case *pql.SetProfileAttrs: - return nil, e.executeSetProfileAttrs(db, root) - } - // If slices aren't specified, then include all of them. if len(slices) == 0 { // Round up the number of slices. @@ -76,7 +66,17 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp } } - return e.executeCall(db, q.Root, slices, opt) + // Execute each call serially. + results := make([]interface{}, 0, len(q.Calls)) + for _, call := range q.Calls { + v, err := e.executeCall(db, call, slices, opt) + if err != nil { + return nil, err + } + results = append(results, v) + } + + return results, nil } // executeCall executes a call. @@ -84,10 +84,18 @@ func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *Exec switch c := c.(type) { case pql.BitmapCall: return e.executeBitmapCall(db, c, slices, opt) + case *pql.ClearBit: + return e.executeClearBit(db, c, opt) case *pql.Count: return e.executeCount(db, c, slices, opt) case *pql.Profile: return e.executeProfile(db, c, opt) + case *pql.SetBit: + return e.executeSetBit(db, c, opt) + case *pql.SetBitmapAttrs: + return nil, e.executeSetBitmapAttrs(db, c) + case *pql.SetProfileAttrs: + return nil, e.executeSetProfileAttrs(db, c) case *pql.TopN: return e.executeTopN(db, c, slices, opt) default: @@ -112,11 +120,11 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6 } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) + res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt) if err != nil { return nil, err } - other.Merge(res.(*Bitmap)) + other.Merge(res[0].(*Bitmap)) } // Attach bitmap attributes for Bitmap() calls. @@ -193,11 +201,11 @@ func (e *Executor) executeTopNSlices(db string, c *pql.TopN, slices []uint64, op } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) + res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt) if err != nil { return nil, err } - results = Pairs(results).Add(res.([]Pair)) + results = Pairs(results).Add(res[0].([]Pair)) } // Sort final merged results. @@ -344,11 +352,11 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *E } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) + res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt) if err != nil { return 0, err } - n += res.(uint64) + n += res[0].(uint64) } return n, nil } @@ -359,6 +367,37 @@ func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) ( panic("FIXME: impl: e.Index().ProfileAttr(c.ID)") } +// executeClearBit executes a ClearBit() call. +func (e *Executor) executeClearBit(db string, c *pql.ClearBit, opt *ExecOptions) (bool, error) { + slice := c.ProfileID / SliceWidth + ret := false + for _, node := range e.Cluster.SliceNodes(slice) { + // Update locally if host matches. + if node.Host == e.Host { + f, err := e.Index().CreateFragmentIfNotExists(db, c.Frame, slice) + if err != nil { + return false, fmt.Errorf("fragment: %s", err) + } + val, err := f.ClearBit(c.ID, c.ProfileID) + if err != nil { + return false, err + } + if val { + ret = true + } + continue + } + + // Forward call to remote node otherwise. + if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil { + return false, err + } else { + ret = res[0].(bool) + } + } + return ret, nil +} + // executeSetBit executes a SetBit() call. func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) { slice := c.ProfileID / SliceWidth @@ -381,10 +420,10 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bo } // Forward call to remote node otherwise. - if res, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil { + if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil { return false, err } else { - ret = res.(bool) + ret = res[0].(bool) } } return ret, nil @@ -427,7 +466,7 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) { +func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ DB: proto.String(db), @@ -488,18 +527,30 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op } // Return appropriate data for the query. - switch q.Root.(type) { - case pql.BitmapCall: - return decodeBitmap(pb.GetBitmap()), nil - case *pql.TopN: - return decodePairs(pb.GetPairs()), nil - case *pql.Count: - return pb.GetN(), nil - case *pql.SetBit: - return pb.GetChanged(), nil - default: - panic(fmt.Sprintf("invalid node for remote exec: %T", q.Root)) + results = make([]interface{}, len(q.Calls)) + for i, call := range q.Calls { + var v interface{} + var err error + + switch call.(type) { + case pql.BitmapCall: + v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil + case *pql.TopN: + v, err = decodePairs(pb.Results[i].GetPairs()), nil + case *pql.Count: + v, err = pb.Results[i].GetN(), nil + case *pql.SetBit: + v, err = pb.Results[i].GetChanged(), nil + default: + panic(fmt.Sprintf("invalid node for remote exec: %T", call)) + } + if err != nil { + return nil, err + } + + results[i] = v } + return results, nil } // slicesByNode returns a mapping of nodes to slices. diff --git a/executor_test.go b/executor_test.go index 49b70e057..88c4dca33 100644 --- a/executor_test.go +++ b/executor_test.go @@ -24,13 +24,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { + } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) } else if chunks[0].Value[0] != 8 { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) } else if chunks[1].Value[0] != 2 { t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) - } else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { + } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } } @@ -47,7 +47,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 { + } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) } else if chunks[0].Value[0] != 10 { // b1010 t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) @@ -69,7 +69,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { + } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) } else if chunks[0].Value[0] != 2 { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) @@ -92,7 +92,7 @@ func TestExecutor_Execute_Union(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { + } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) } else if chunks[0].Value[0] != 5 { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) @@ -110,10 +110,10 @@ func TestExecutor_Execute_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) - } else if n != uint64(3) { - t.Fatalf("unexpected n: %d", n) + } else if res[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res[0]) } } @@ -131,7 +131,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { t.Fatal(err) } else { - if !res.(bool) { + if !res[0].(bool) { t.Fatalf("expected bit changed") } } @@ -142,7 +142,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { t.Fatal(err) } else { - if res.(bool) { + if res[0].(bool) { t.Fatalf("expected bit unchanged") } } @@ -197,7 +197,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []pilosa.Pair{ + } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {Key: 0, Count: 5}, {Key: 10, Count: 2}, }) { @@ -220,9 +220,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []pilosa.Pair{ + } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {Key: 0, Count: 4}, - }) { + }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } } @@ -251,11 +251,11 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []pilosa.Pair{ + } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {Key: 20, Count: 3}, {Key: 10, Count: 2}, {Key: 0, Count: 1}, - }) { + }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } } @@ -273,7 +273,7 @@ func TestExecutor_Execute_Range(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected bits: %+v", bits) } } @@ -288,7 +288,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(id=10, frame=f)` { @@ -303,7 +303,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { (0*SliceWidth)+2, (2*SliceWidth)+4, ) - return bm, nil + return []interface{}{bm}, nil } // Create local executor data. @@ -315,7 +315,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { e := NewExecutor(idx.Index, c) if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 { + } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) } else if chunks[0].Value[0] != 6 { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) @@ -334,8 +334,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { - return uint64(10), nil + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{uint64(10)}, nil } // Create local executor data. The local node owns slice 1. @@ -345,10 +345,10 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) - } else if n != uint64(12) { - t.Fatalf("unexpected n: %d", n) + } else if res[0] != uint64(12) { + t.Fatalf("unexpected n: %d", res[0]) } } @@ -364,14 +364,14 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `SetBit(id=10, frame=f, profileID=2)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true - return nil, nil + return []interface{}{nil}, nil } // Create local executor data. @@ -403,7 +403,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) { @@ -427,11 +427,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { remoteExecN++ // Return pair counts. - return []pilosa.Pair{ + return []interface{}{[]pilosa.Pair{ {Key: 0, Count: 5}, {Key: 10, Count: 2}, {Key: 30, Count: 2}, - }, nil + }}, nil } // Create local executor data on slice 1 & 3. @@ -443,11 +443,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { e := NewExecutor(idx.Index, c) if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(res, []pilosa.Pair{ + } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {Key: 0, Count: 5}, {Key: 30, Count: 4}, {Key: 10, Count: 2}, - }) { + }}) { t.Fatalf("unexpected results: %s", spew.Sdump(res)) } } diff --git a/handler.go b/handler.go index 6846766e6..cf3c68fc0 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,7 @@ type Handler struct { // The execution engine for running queries. Executor interface { - Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) + Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } // The version to report on the /version endpoint. @@ -127,12 +127,23 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Execute the query. - result, err := h.Executor.Execute(req.DB, q, req.Slices, opt) - resp := &QueryResponse{Result: result, Err: err} + results, err := h.Executor.Execute(req.DB, q, req.Slices, opt) + resp := &QueryResponse{Results: results, Err: err} // Fill profile attributes if requested. - if bm, ok := result.(*Bitmap); ok && req.Profiles { - profiles, err := h.readProfiles(h.Index.DB(req.DB), bm.Bits()) + if req.Profiles { + // Consolidate all profile ids across all calls. + var profileIDs []uint64 + for _, result := range results { + bm, ok := result.(*Bitmap) + if !ok { + continue + } + profileIDs = uint64Slice(profileIDs).merge(bm.Bits()) + } + + // Retrieve profile attributes across all calls. + profiles, err := h.readProfiles(h.Index.DB(req.DB), profileIDs) if err != nil { w.WriteHeader(http.StatusInternalServerError) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) @@ -502,9 +513,9 @@ func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { // QueryResponse represent a response from a processed query. type QueryResponse struct { - // Query execution results. + // Result for each top-level query call. // Can be a Bitmap, Pairs, or uint64. - Result interface{} + Results []interface{} // Set of profiles matching IDs returned in Result. Profiles []*Profile @@ -515,11 +526,11 @@ type QueryResponse struct { func (resp *QueryResponse) MarshalJSON() ([]byte, error) { var output struct { - Result interface{} `json:"result,omitempty"` - Profiles []*Profile `json:"profiles,omitempty"` - Err string `json:"error,omitempty"` + Results []interface{} `json:"results,omitempty"` + Profiles []*Profile `json:"profiles,omitempty"` + Err string `json:"error,omitempty"` } - output.Result = resp.Result + output.Results = resp.Results output.Profiles = resp.Profiles if resp.Err != nil { @@ -530,21 +541,22 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(resp.Results)), Profiles: encodeProfiles(resp.Profiles), } - if resp.Result != nil { - switch result := resp.Result.(type) { + for i := range resp.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := resp.Results[i].(type) { case *Bitmap: - pb.Bitmap = encodeBitmap(result) + pb.Results[i].Bitmap = encodeBitmap(result) case []Pair: - pb.Pairs = encodePairs(result) + pb.Results[i].Pairs = encodePairs(result) case uint64: - pb.N = proto.Uint64(result) + pb.Results[i].N = proto.Uint64(result) case bool: - pb.Changed = proto.Bool(result) - default: - panic(fmt.Sprintf("invalid query result type: %T", resp.Result)) + pb.Results[i].Changed = proto.Bool(result) } } diff --git a/handler_test.go b/handler_test.go index c07f3e597..c56c0a4da 100644 --- a/handler_test.go +++ b/handler_test.go @@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -37,14 +37,14 @@ func TestHandler_Query_Args_URL(t *testing.T) { } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { t.Fatalf("unexpected slices: %+v", slices) } - return uint64(100), nil + return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"result":100}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) } } @@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -60,7 +60,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { t.Fatalf("unexpected slices: %+v", slices) } - return uint64(100), nil + return []interface{}{uint64(100)}, nil } // Generate request body. @@ -98,15 +98,15 @@ func TestHandler_Query_Args_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { - return uint64(100), nil + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"result":100}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) } } @@ -114,8 +114,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { - return uint64(100), nil + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() @@ -129,25 +129,25 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if resp.GetN() != 100 { - t.Fatalf("unexpected n: %d", resp.GetN()) + } else if n := resp.Results[0].GetN(); n != 100 { + t.Fatalf("unexpected n: %d", n) } } // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return bm, nil + return []interface{}{bm}, nil } w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -169,17 +169,17 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return bm, nil + return []interface{}{bm}, nil } w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]},"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -187,10 +187,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return bm, nil + return []interface{}{bm}, nil } w := httptest.NewRecorder() @@ -204,9 +204,9 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if a := resp.GetBitmap().GetChunks(); len(a) != 2 { + } else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 { t.Fatalf("unexpected bitmap chunk length: %d", len(a)) - } else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) @@ -232,10 +232,10 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return bm, nil + return []interface{}{bm}, nil } // Encode request body. @@ -261,9 +261,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if a := resp.GetBitmap().GetChunks(); len(a) != 2 { + if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 { t.Fatalf("unexpected bitmap chunk length: %d", len(a)) - } else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) @@ -287,18 +287,18 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { - return []pilosa.Pair{ + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{[]pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, - }, nil + }}, nil } w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`TopN(frame=x, n=2)`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"result":[{"key":1,"count":2},{"key":3,"count":4}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[[{"key":1,"count":2},{"key":3,"count":4}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) } } @@ -306,11 +306,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { - return []pilosa.Pair{ + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return []interface{}{[]pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, - }, nil + }}, nil } w := httptest.NewRecorder() @@ -324,7 +324,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if a := resp.GetPairs(); len(a) != 2 { + } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } } @@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -493,12 +493,12 @@ func NewHandler() *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) + ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { +func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return c.ExecuteFn(db, query, slices, opt) } diff --git a/internal/internal.pb.go b/internal/internal.pb.go index a1ab93a9c..475fa879e 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -18,6 +18,7 @@ It has these top-level messages: AttrMap QueryRequest QueryResponse + QueryResult ImportRequest ImportResponse Cache @@ -273,13 +274,10 @@ func (m *QueryRequest) GetRemote() bool { } type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"` - Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"` - Profiles []*Profile `protobuf:"bytes,5,rep" json:"Profiles,omitempty"` - Changed *bool `protobuf:"varint,6,opt" json:"Changed,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep" json:"Profiles,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } @@ -293,23 +291,9 @@ func (m *QueryResponse) GetErr() string { return "" } -func (m *QueryResponse) GetBitmap() *Bitmap { +func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { - return m.Bitmap - } - return nil -} - -func (m *QueryResponse) GetN() uint64 { - if m != nil && m.N != nil { - return *m.N - } - return 0 -} - -func (m *QueryResponse) GetPairs() []*Pair { - if m != nil { - return m.Pairs + return m.Results } return nil } @@ -321,7 +305,40 @@ func (m *QueryResponse) GetProfiles() []*Profile { return nil } -func (m *QueryResponse) GetChanged() bool { +type QueryResult struct { + Bitmap *Bitmap `protobuf:"bytes,1,opt" json:"Bitmap,omitempty"` + N *uint64 `protobuf:"varint,2,opt" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep" json:"Pairs,omitempty"` + Changed *bool `protobuf:"varint,4,opt" json:"Changed,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *QueryResult) Reset() { *m = QueryResult{} } +func (m *QueryResult) String() string { return proto.CompactTextString(m) } +func (*QueryResult) ProtoMessage() {} + +func (m *QueryResult) GetBitmap() *Bitmap { + if m != nil { + return m.Bitmap + } + return nil +} + +func (m *QueryResult) GetN() uint64 { + if m != nil && m.N != nil { + return *m.N + } + return 0 +} + +func (m *QueryResult) GetPairs() []*Pair { + if m != nil { + return m.Pairs + } + return nil +} + +func (m *QueryResult) GetChanged() bool { if m != nil && m.Changed != nil { return *m.Changed } diff --git a/internal/internal.proto b/internal/internal.proto index de85efa30..127095f50 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -47,12 +47,16 @@ message QueryRequest { } message QueryResponse { - optional string Err = 1; - optional Bitmap Bitmap = 2; - optional uint64 N = 3; - repeated Pair Pairs = 4; - repeated Profile Profiles = 5; - optional bool Changed = 6; + optional string Err = 1; + repeated QueryResult Results = 2; + repeated Profile Profiles = 3; +} + +message QueryResult { + optional Bitmap Bitmap = 1; + optional uint64 N = 2; + repeated Pair Pairs = 3; + optional bool Changed = 4; } message ImportRequest { diff --git a/pql/ast.go b/pql/ast.go index aa27f06d5..0659e9d3d 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -10,11 +10,17 @@ import ( // Query represents a PQL query. type Query struct { - Root Call + Calls Calls } // String returns a string representation of the query. -func (q *Query) String() string { return q.Root.String() } +func (q *Query) String() string { + a := make([]string, len(q.Calls)) + for i, call := range q.Calls { + a[i] = call.String() + } + return strings.Join(a, "\n") +} // Node represents any node in the AST. type Node interface { diff --git a/pql/parser.go b/pql/parser.go index 5e44c07f4..a1a40f793 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -30,17 +30,31 @@ func ParseString(s string) (*Query, error) { // Parse parses the next node in the query. func (p *Parser) Parse() (*Query, error) { - fn, err := p.parseCall() - if err != nil { - return nil, err + q := &Query{} + for { + call, err := p.parseCall() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + q.Calls = append(q.Calls, call) } - return &Query{Root: fn}, nil + + // Require at least one call. + if len(q.Calls) == 0 { + return nil, io.ErrUnexpectedEOF + } + + return q, nil } // parseCall parses the next function call. func (p *Parser) parseCall() (Call, error) { tok, pos, lit := p.scanIgnoreWhitespace() - if tok != IDENT { + if tok == EOF { + return nil, io.EOF + } else if tok != IDENT { return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} } diff --git a/pql/parser_test.go b/pql/parser_test.go index 59628af49..3904455d2 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -15,9 +15,11 @@ func TestParser_Parse_Bitmap_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Bitmap{ - ID: 1, - Frame: "b.n", + Calls: pql.Calls{ + &pql.Bitmap{ + ID: 1, + Frame: "b.n", + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -30,9 +32,11 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Bitmap{ - ID: 1, - Frame: "b.n", + Calls: pql.Calls{ + &pql.Bitmap{ + ID: 1, + Frame: "b.n", + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -45,10 +49,12 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.ClearBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, + Calls: pql.Calls{ + &pql.ClearBit{ + ID: 1, + Frame: "b.n", + ProfileID: 3, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -61,10 +67,12 @@ func TestParser_Parse_ClearBit_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.ClearBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, + Calls: pql.Calls{ + &pql.ClearBit{ + ID: 1, + Frame: "b.n", + ProfileID: 3, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -77,9 +85,11 @@ func TestParser_Parse_Count(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Count{ - Input: &pql.Bitmap{ - ID: 1, + Calls: pql.Calls{ + &pql.Count{ + Input: &pql.Bitmap{ + ID: 1, + }, }, }, }) { @@ -93,10 +103,12 @@ func TestParser_Parse_Difference(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Difference{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, + Calls: pql.Calls{ + &pql.Difference{ + Inputs: pql.BitmapCalls{ + &pql.Bitmap{ID: 1}, + &pql.Bitmap{ID: 2}, + }, }, }, }) { @@ -110,10 +122,12 @@ func TestParser_Parse_Intersect(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Intersect{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, + Calls: pql.Calls{ + &pql.Intersect{ + Inputs: pql.BitmapCalls{ + &pql.Bitmap{ID: 1}, + &pql.Bitmap{ID: 2}, + }, }, }, }) { @@ -127,7 +141,9 @@ func TestParser_Parse_Profile_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Profile{ID: 1}, + Calls: pql.Calls{ + &pql.Profile{ID: 1}, + }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) } @@ -139,7 +155,9 @@ func TestParser_Parse_Profile_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Profile{ID: 1}, + Calls: pql.Calls{ + &pql.Profile{ID: 1}, + }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) } @@ -151,11 +169,13 @@ func TestParser_Parse_Range_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Range{ - ID: 20, - Frame: "b.n", - StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), - EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), + Calls: pql.Calls{ + &pql.Range{ + ID: 20, + Frame: "b.n", + StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), + EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -168,11 +188,13 @@ func TestParser_Parse_Range_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Range{ - ID: 20, - Frame: "b.n", - StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), - EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), + Calls: pql.Calls{ + &pql.Range{ + ID: 20, + Frame: "b.n", + StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC), + EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC), + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -185,10 +207,12 @@ func TestParser_Parse_SetBit_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.SetBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, + Calls: pql.Calls{ + &pql.SetBit{ + ID: 1, + Frame: "b.n", + ProfileID: 3, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -201,10 +225,12 @@ func TestParser_Parse_SetBit_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.SetBit{ - ID: 1, - Frame: "b.n", - ProfileID: 3, + Calls: pql.Calls{ + &pql.SetBit{ + ID: 1, + Frame: "b.n", + ProfileID: 3, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -217,15 +243,17 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.SetBitmapAttrs{ - ID: 1, - Frame: "b.n", - Attrs: map[string]interface{}{ - "foo": "bar", - "bar": uint64(123), - "baz": true, - "bat": false, - "x": nil, + Calls: pql.Calls{ + &pql.SetBitmapAttrs{ + ID: 1, + Frame: "b.n", + Attrs: map[string]interface{}{ + "foo": "bar", + "bar": uint64(123), + "baz": true, + "bat": false, + "x": nil, + }, }, }, }) { @@ -239,12 +267,14 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.SetBitmapAttrs{ - ID: 1, - Frame: "b.n", - Attrs: map[string]interface{}{ - "foo": "bar", - "bar": uint64(123), + Calls: pql.Calls{ + &pql.SetBitmapAttrs{ + ID: 1, + Frame: "b.n", + Attrs: map[string]interface{}{ + "foo": "bar", + "bar": uint64(123), + }, }, }, }) { @@ -258,13 +288,15 @@ func TestParser_Parse_TopN_Key(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.TopN{ - Src: &pql.Bitmap{ID: 100}, - Frame: "b.n", - N: 2, - BitmapIDs: []uint64{1, 2, 3}, - Field: "XXX", - Filters: []interface{}{uint64(5), uint64(10), uint64(15)}, + Calls: pql.Calls{ + &pql.TopN{ + Src: &pql.Bitmap{ID: 100}, + Frame: "b.n", + N: 2, + BitmapIDs: []uint64{1, 2, 3}, + Field: "XXX", + Filters: []interface{}{uint64(5), uint64(10), uint64(15)}, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -281,12 +313,14 @@ func TestParser_Parse_TopN_Array(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.TopN{ - Src: &pql.Bitmap{ID: 100}, - Frame: "b.n", - N: 2, - Field: "XXX", - Filters: []interface{}{"foo", true, false}, + Calls: pql.Calls{ + &pql.TopN{ + Src: &pql.Bitmap{ID: 100}, + Frame: "b.n", + N: 2, + Field: "XXX", + Filters: []interface{}{"foo", true, false}, + }, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) @@ -303,10 +337,12 @@ func TestParser_Parse_Union(t *testing.T) { if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ - Root: &pql.Union{ - Inputs: pql.BitmapCalls{ - &pql.Bitmap{ID: 1}, - &pql.Bitmap{ID: 2}, + Calls: pql.Calls{ + &pql.Union{ + Inputs: pql.BitmapCalls{ + &pql.Bitmap{ID: 1}, + &pql.Bitmap{ID: 2}, + }, }, }, }) { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c25268878..fc22e8d13 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -165,7 +165,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { // Add more values to bitmap. for _, v := range a1 { set[v] = struct{}{} - if err := bm.Add(v); err != nil { + if _, err := bm.Add(v); err != nil { t.Fatal(err) }