scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 00:25:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #60 from benbjohnson/multi-call

Browse source 
Add bulk call support to the query endpoint
This commit is contained in:
tgruben 2016-03-09 10:53:07 -06:00
commit 8b86901d90
12 changed files with 419 additions and 246 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
cache.go
View file

@ -286,3 +286,32 @@ type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}

26
cmd/pilosa/main_test.go
View file

@ -40,9 +40,11 @@ func TestMain_Set_Quick(t *testing.T) {
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"result": map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
@ -61,9 +63,11 @@ func TestMain_Set_Quick(t *testing.T) {
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"result": map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
@ -110,14 +114,14 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
// Query bitmap x.n/1.
if res, err := m.Query("db=d", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query bitmap x.n/2.
if res, err := m.Query("db=d", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":200},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -128,7 +132,7 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
// Query bitmap after reopening.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
@ -153,7 +157,7 @@ func TestMain_SetProfileAttrs(t *testing.T) {
// Query bitmap.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -164,7 +168,7 @@ func TestMain_SetProfileAttrs(t *testing.T) {
// Query bitmap after reopening.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}

115
executor.go
View file

@ -42,7 +42,7 @@ func NewExecutor(index *Index) *Executor {
func (e *Executor) Index() *Index { return e.index }
// Execute executes a PQL query.
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) {
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
@ -53,16 +53,6 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp
opt = &ExecOptions{}
}
// Ignore slices for set calls.
switch root := q.Root.(type) {
case *pql.SetBit:
return e.executeSetBit(db, root, opt)
case *pql.SetBitmapAttrs:
return nil, e.executeSetBitmapAttrs(db, root)
case *pql.SetProfileAttrs:
return nil, e.executeSetProfileAttrs(db, root)
}
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Round up the number of slices.
@ -76,7 +66,17 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp
}
}
return e.executeCall(db, q.Root, slices, opt)
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
v, err := e.executeCall(db, call, slices, opt)
if err != nil {
return nil, err
}
results = append(results, v)
}
return results, nil
}
// executeCall executes a call.
@ -84,10 +84,18 @@ func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *Exec
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(db, c, slices, opt)
case *pql.ClearBit:
return e.executeClearBit(db, c, opt)
case *pql.Count:
return e.executeCount(db, c, slices, opt)
case *pql.Profile:
return e.executeProfile(db, c, opt)
case *pql.SetBit:
return e.executeSetBit(db, c, opt)
case *pql.SetBitmapAttrs:
return nil, e.executeSetBitmapAttrs(db, c)
case *pql.SetProfileAttrs:
return nil, e.executeSetProfileAttrs(db, c)
case *pql.TopN:
return e.executeTopN(db, c, slices, opt)
default:
@ -112,11 +120,11 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return nil, err
}
other.Merge(res.(*Bitmap))
other.Merge(res[0].(*Bitmap))
}
// Attach bitmap attributes for Bitmap() calls.
@ -193,11 +201,11 @@ func (e *Executor) executeTopNSlices(db string, c *pql.TopN, slices []uint64, op
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return nil, err
}
results = Pairs(results).Add(res.([]Pair))
results = Pairs(results).Add(res[0].([]Pair))
}
// Sort final merged results.
@ -344,11 +352,11 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *E
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return 0, err
}
n += res.(uint64)
n += res[0].(uint64)
}
return n, nil
}
@ -359,6 +367,37 @@ func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) (
panic("FIXME: impl: e.Index().ProfileAttr(c.ID)")
}
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(db string, c *pql.ClearBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(slice) {
// Update locally if host matches.
if node.Host == e.Host {
f, err := e.Index().CreateFragmentIfNotExists(db, c.Frame, slice)
if err != nil {
return false, fmt.Errorf("fragment: %s", err)
}
val, err := f.ClearBit(c.ID, c.ProfileID)
if err != nil {
return false, err
}
if val {
ret = true
}
continue
}
// Forward call to remote node otherwise.
if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
}
}
return ret, nil
}
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
@ -381,10 +420,10 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bo
}
// Forward call to remote node otherwise.
if res, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil {
if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res.(bool)
ret = res[0].(bool)
}
}
return ret, nil
@ -427,7 +466,7 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) {
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
DB: proto.String(db),
@ -488,18 +527,30 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op
}
// Return appropriate data for the query.
switch q.Root.(type) {
case pql.BitmapCall:
return decodeBitmap(pb.GetBitmap()), nil
case *pql.TopN:
return decodePairs(pb.GetPairs()), nil
case *pql.Count:
return pb.GetN(), nil
case *pql.SetBit:
return pb.GetChanged(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", q.Root))
results = make([]interface{}, len(q.Calls))
for i, call := range q.Calls {
var v interface{}
var err error
switch call.(type) {
case pql.BitmapCall:
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
case *pql.TopN:
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case *pql.Count:
v, err = pb.Results[i].GetN(), nil
case *pql.SetBit:
v, err = pb.Results[i].GetChanged(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", call))
}
if err != nil {
return nil, err
}
results[i] = v
}
return results, nil
}
// slicesByNode returns a mapping of nodes to slices.

62
executor_test.go
View file

@ -24,13 +24,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 8 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
}
@ -47,7 +47,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 10 { // b1010
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -69,7 +69,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 2 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -92,7 +92,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 5 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -110,10 +110,10 @@ func TestExecutor_Execute_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(3) {
t.Fatalf("unexpected n: %d", n)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
}
}
@ -131,7 +131,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res.(bool) {
if !res[0].(bool) {
t.Fatalf("expected bit changed")
}
}
@ -142,7 +142,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res.(bool) {
if res[0].(bool) {
t.Fatalf("expected bit unchanged")
}
}
@ -197,7 +197,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
}) {
@ -220,9 +220,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{Key: 0, Count: 4},
}) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
@ -251,11 +251,11 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{Key: 20, Count: 3},
{Key: 10, Count: 2},
{Key: 0, Count: 1},
}) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
@ -273,7 +273,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) {
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -288,7 +288,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Bitmap(id=10, frame=f)` {
@ -303,7 +303,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
(0*SliceWidth)+2,
(2*SliceWidth)+4,
)
return bm, nil
return []interface{}{bm}, nil
}
// Create local executor data.
@ -315,7 +315,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 6 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -334,8 +334,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(10), nil
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(10)}, nil
}
// Create local executor data. The local node owns slice 1.
@ -345,10 +345,10 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(12) {
t.Fatalf("unexpected n: %d", n)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
}
}
@ -364,14 +364,14 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(id=10, frame=f, profileID=2)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
return nil, nil
return []interface{}{nil}, nil
}
// Create local executor data.
@ -403,7 +403,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) {
@ -427,11 +427,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
remoteExecN++
// Return pair counts.
return []pilosa.Pair{
return []interface{}{[]pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
{Key: 30, Count: 2},
}, nil
}}, nil
}
// Create local executor data on slice 1 & 3.
@ -443,11 +443,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []pilosa.Pair{
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{Key: 0, Count: 5},
{Key: 30, Count: 4},
{Key: 10, Count: 2},
}) {
}}) {
t.Fatalf("unexpected results: %s", spew.Sdump(res))
}
}

50
handler.go
View file

@ -29,7 +29,7 @@ type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error)
Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
}
// The version to report on the /version endpoint.
@ -127,12 +127,23 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
// Execute the query.
result, err := h.Executor.Execute(req.DB, q, req.Slices, opt)
resp := &QueryResponse{Result: result, Err: err}
results, err := h.Executor.Execute(req.DB, q, req.Slices, opt)
resp := &QueryResponse{Results: results, Err: err}
// Fill profile attributes if requested.
if bm, ok := result.(*Bitmap); ok && req.Profiles {
profiles, err := h.readProfiles(h.Index.DB(req.DB), bm.Bits())
if req.Profiles {
// Consolidate all profile ids across all calls.
var profileIDs []uint64
for _, result := range results {
bm, ok := result.(*Bitmap)
if !ok {
continue
}
profileIDs = uint64Slice(profileIDs).merge(bm.Bits())
}
// Retrieve profile attributes across all calls.
profiles, err := h.readProfiles(h.Index.DB(req.DB), profileIDs)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
@ -502,9 +513,9 @@ func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
// QueryResponse represent a response from a processed query.
type QueryResponse struct {
// Query execution results.
// Result for each top-level query call.
// Can be a Bitmap, Pairs, or uint64.
Result interface{}
Results []interface{}
// Set of profiles matching IDs returned in Result.
Profiles []*Profile
@ -515,11 +526,11 @@ type QueryResponse struct {
func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
var output struct {
Result interface{} `json:"result,omitempty"`
Profiles []*Profile `json:"profiles,omitempty"`
Err string `json:"error,omitempty"`
Results []interface{} `json:"results,omitempty"`
Profiles []*Profile `json:"profiles,omitempty"`
Err string `json:"error,omitempty"`
}
output.Result = resp.Result
output.Results = resp.Results
output.Profiles = resp.Profiles
if resp.Err != nil {
@ -530,21 +541,22 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
Profiles: encodeProfiles(resp.Profiles),
}
if resp.Result != nil {
switch result := resp.Result.(type) {
for i := range resp.Results {
pb.Results[i] = &internal.QueryResult{}
switch result := resp.Results[i].(type) {
case *Bitmap:
pb.Bitmap = encodeBitmap(result)
pb.Results[i].Bitmap = encodeBitmap(result)
case []Pair:
pb.Pairs = encodePairs(result)
pb.Results[i].Pairs = encodePairs(result)
case uint64:
pb.N = proto.Uint64(result)
pb.Results[i].N = proto.Uint64(result)
case bool:
pb.Changed = proto.Bool(result)
default:
panic(fmt.Sprintf("invalid query result type: %T", resp.Result))
pb.Results[i].Changed = proto.Bool(result)
}
}

76
handler_test.go
View file

@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) {
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -37,14 +37,14 @@ func TestHandler_Query_Args_URL(t *testing.T) {
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -60,7 +60,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
return []interface{}{uint64(100)}, nil
}
// Generate request body.
@ -98,15 +98,15 @@ func TestHandler_Query_Args_Err(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -114,8 +114,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
@ -129,25 +129,25 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if resp.GetN() != 100 {
t.Fatalf("unexpected n: %d", resp.GetN())
} else if n := resp.Results[0].GetN(); n != 100 {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -169,17 +169,17 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]},"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -187,10 +187,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
@ -204,9 +204,9 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
} else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
} else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 {
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
@ -232,10 +232,10 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
return []interface{}{bm}, nil
}
// Encode request body.
@ -261,9 +261,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
} else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 {
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
@ -287,18 +287,18 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`TopN(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":[{"key":1,"count":2},{"key":3,"count":4}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[[{"key":1,"count":2},{"key":3,"count":4}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -306,11 +306,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}}, nil
}
w := httptest.NewRecorder()
@ -324,7 +324,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetPairs(); len(a) != 2 {
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -493,12 +493,12 @@ func NewHandler() *Handler {
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error)
ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(db, query, slices, opt)
}

65
internal/internal.pb.go
View file

@ -18,6 +18,7 @@ It has these top-level messages:
AttrMap
QueryRequest
QueryResponse
QueryResult
ImportRequest
ImportResponse
Cache
@ -273,13 +274,10 @@ func (m *QueryRequest) GetRemote() bool {
}
type QueryResponse struct {
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"`
Profiles []*Profile `protobuf:"bytes,5,rep" json:"Profiles,omitempty"`
Changed *bool `protobuf:"varint,6,opt" json:"Changed,omitempty"`
XXX_unrecognized []byte `json:"-"`
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Results []*QueryResult `protobuf:"bytes,2,rep" json:"Results,omitempty"`
Profiles []*Profile `protobuf:"bytes,3,rep" json:"Profiles,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
@ -293,23 +291,9 @@ func (m *QueryResponse) GetErr() string {
return ""
}
func (m *QueryResponse) GetBitmap() *Bitmap {
func (m *QueryResponse) GetResults() []*QueryResult {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResponse) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResponse) GetPairs() []*Pair {
if m != nil {
return m.Pairs
return m.Results
}
return nil
}
@ -321,7 +305,40 @@ func (m *QueryResponse) GetProfiles() []*Profile {
return nil
}
func (m *QueryResponse) GetChanged() bool {
type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,2,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep" json:"Pairs,omitempty"`
Changed *bool `protobuf:"varint,4,opt" json:"Changed,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (m *QueryResult) GetBitmap() *Bitmap {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResult) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResult) GetPairs() []*Pair {
if m != nil {
return m.Pairs
}
return nil
}
func (m *QueryResult) GetChanged() bool {
if m != nil && m.Changed != nil {
return *m.Changed
}

16
internal/internal.proto
View file

@ -47,12 +47,16 @@ message QueryRequest {
}
message QueryResponse {
optional string Err = 1;
optional Bitmap Bitmap = 2;
optional uint64 N = 3;
repeated Pair Pairs = 4;
repeated Profile Profiles = 5;
optional bool Changed = 6;
optional string Err = 1;
repeated QueryResult Results = 2;
repeated Profile Profiles = 3;
}
message QueryResult {
optional Bitmap Bitmap = 1;
optional uint64 N = 2;
repeated Pair Pairs = 3;
optional bool Changed = 4;
}
message ImportRequest {

10
pql/ast.go
View file

@ -10,11 +10,17 @@ import (
// Query represents a PQL query.
type Query struct {
Root Call
Calls Calls
}
// String returns a string representation of the query.
func (q *Query) String() string { return q.Root.String() }
func (q *Query) String() string {
a := make([]string, len(q.Calls))
for i, call := range q.Calls {
a[i] = call.String()
}
return strings.Join(a, "\n")
}
// Node represents any node in the AST.
type Node interface {

24
pql/parser.go
View file

@ -30,17 +30,31 @@ func ParseString(s string) (*Query, error) {
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
fn, err := p.parseCall()
if err != nil {
return nil, err
q := &Query{}
for {
call, err := p.parseCall()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
q.Calls = append(q.Calls, call)
}
return &Query{Root: fn}, nil
// Require at least one call.
if len(q.Calls) == 0 {
return nil, io.ErrUnexpectedEOF
}
return q, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
tok, pos, lit := p.scanIgnoreWhitespace()
if tok != IDENT {
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}

190
pql/parser_test.go
View file

@ -15,9 +15,11 @@ func TestParser_Parse_Bitmap_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Bitmap{
ID: 1,
Frame: "b.n",
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -30,9 +32,11 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Bitmap{
ID: 1,
Frame: "b.n",
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -45,10 +49,12 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -61,10 +67,12 @@ func TestParser_Parse_ClearBit_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -77,9 +85,11 @@ func TestParser_Parse_Count(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Count{
Input: &pql.Bitmap{
ID: 1,
Calls: pql.Calls{
&pql.Count{
Input: &pql.Bitmap{
ID: 1,
},
},
},
}) {
@ -93,10 +103,12 @@ func TestParser_Parse_Difference(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {
@ -110,10 +122,12 @@ func TestParser_Parse_Intersect(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {
@ -127,7 +141,9 @@ func TestParser_Parse_Profile_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Profile{ID: 1},
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
@ -139,7 +155,9 @@ func TestParser_Parse_Profile_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Profile{ID: 1},
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
@ -151,11 +169,13 @@ func TestParser_Parse_Range_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -168,11 +188,13 @@ func TestParser_Parse_Range_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -185,10 +207,12 @@ func TestParser_Parse_SetBit_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -201,10 +225,12 @@ func TestParser_Parse_SetBit_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -217,15 +243,17 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
},
},
},
}) {
@ -239,12 +267,14 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
},
},
},
}) {
@ -258,13 +288,15 @@ func TestParser_Parse_TopN_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
BitmapIDs: []uint64{1, 2, 3},
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
BitmapIDs: []uint64{1, 2, 3},
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -281,12 +313,14 @@ func TestParser_Parse_TopN_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -303,10 +337,12 @@ func TestParser_Parse_Union(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Union{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Union{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {

2
roaring/roaring_test.go
View file

@ -165,7 +165,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
// Add more values to bitmap.
for _, v := range a1 {
set[v] = struct{}{}
if err := bm.Add(v); err != nil {
if _, err := bm.Add(v); err != nil {
t.Fatal(err)
}