scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 00:25:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add bulk call support to the query endpoint

Browse source 
This commit changes the `pql.Query` so that it can accept one or more
top-level calls instead of only one.

The query request format change because a query with a single call is
still valid. However, the result format now returns a `results` field
that has one result for each top-level call. The `profiles` field is
still the same, however, it combines all profiles from all bitmap
responses into one return so that there's not duplicate attributes.

Fixes #59
This commit is contained in:
Ben Johnson 2016-03-08 14:34:37 -07:00
parent e7b47efd8d
commit ee521e2bbf
12 changed files with 419 additions and 246 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
cache.go
View file

@ -286,3 +286,32 @@ type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}

26
cmd/pilosa/main_test.go
View file

@ -40,9 +40,11 @@ func TestMain_Set_Quick(t *testing.T) {
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"result": map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
@ -61,9 +63,11 @@ func TestMain_Set_Quick(t *testing.T) {
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"result": map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
if res, err := m.Query("db=d", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
@ -110,14 +114,14 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
// Query bitmap x.n/1.
if res, err := m.Query("db=d", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query bitmap x.n/2.
if res, err := m.Query("db=d", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":200},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -128,7 +132,7 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
// Query bitmap after reopening.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{"x":100},"bits":[100]}}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
@ -153,7 +157,7 @@ func TestMain_SetProfileAttrs(t *testing.T) {
// Query bitmap.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -164,7 +168,7 @@ func TestMain_SetProfileAttrs(t *testing.T) {
// Query bitmap after reopening.
if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"result":{"attrs":{},"bits":[100,101]},"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}

115
executor.go
View file

@ -42,7 +42,7 @@ func NewExecutor(index *Index) *Executor {
func (e *Executor) Index() *Index { return e.index }
// Execute executes a PQL query.
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) {
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
@ -53,16 +53,6 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp
opt = &ExecOptions{}
}
// Ignore slices for set calls.
switch root := q.Root.(type) {
case *pql.SetBit:
return e.executeSetBit(db, root, opt)
case *pql.SetBitmapAttrs:
return nil, e.executeSetBitmapAttrs(db, root)
case *pql.SetProfileAttrs:
return nil, e.executeSetProfileAttrs(db, root)
}
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Round up the number of slices.
@ -76,7 +66,17 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOp
}
}
return e.executeCall(db, q.Root, slices, opt)
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
v, err := e.executeCall(db, call, slices, opt)
if err != nil {
return nil, err
}
results = append(results, v)
}
return results, nil
}
// executeCall executes a call.
@ -84,10 +84,18 @@ func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *Exec
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(db, c, slices, opt)
case *pql.ClearBit:
return e.executeClearBit(db, c, opt)
case *pql.Count:
return e.executeCount(db, c, slices, opt)
case *pql.Profile:
return e.executeProfile(db, c, opt)
case *pql.SetBit:
return e.executeSetBit(db, c, opt)
case *pql.SetBitmapAttrs:
return nil, e.executeSetBitmapAttrs(db, c)
case *pql.SetProfileAttrs:
return nil, e.executeSetProfileAttrs(db, c)
case *pql.TopN:
return e.executeTopN(db, c, slices, opt)
default:
@ -112,11 +120,11 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return nil, err
}
other.Merge(res.(*Bitmap))
other.Merge(res[0].(*Bitmap))
}
// Attach bitmap attributes for Bitmap() calls.
@ -193,11 +201,11 @@ func (e *Executor) executeTopNSlices(db string, c *pql.TopN, slices []uint64, op
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return nil, err
}
results = Pairs(results).Add(res.([]Pair))
results = Pairs(results).Add(res[0].([]Pair))
}
// Sort final merged results.
@ -344,11 +352,11 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *E
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
if err != nil {
return 0, err
}
n += res.(uint64)
n += res[0].(uint64)
}
return n, nil
}
@ -359,6 +367,37 @@ func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) (
panic("FIXME: impl: e.Index().ProfileAttr(c.ID)")
}
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(db string, c *pql.ClearBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(slice) {
// Update locally if host matches.
if node.Host == e.Host {
f, err := e.Index().CreateFragmentIfNotExists(db, c.Frame, slice)
if err != nil {
return false, fmt.Errorf("fragment: %s", err)
}
val, err := f.ClearBit(c.ID, c.ProfileID)
if err != nil {
return false, err
}
if val {
ret = true
}
continue
}
// Forward call to remote node otherwise.
if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
}
}
return ret, nil
}
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
@ -381,10 +420,10 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bo
}
// Forward call to remote node otherwise.
if res, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil {
if res, err := e.exec(node, db, &pql.Query{Calls: pql.Calls{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res.(bool)
ret = res[0].(bool)
}
}
return ret, nil
@ -427,7 +466,7 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) {
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
DB: proto.String(db),
@ -488,18 +527,30 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op
}
// Return appropriate data for the query.
switch q.Root.(type) {
case pql.BitmapCall:
return decodeBitmap(pb.GetBitmap()), nil
case *pql.TopN:
return decodePairs(pb.GetPairs()), nil
case *pql.Count:
return pb.GetN(), nil
case *pql.SetBit:
return pb.GetChanged(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", q.Root))
results = make([]interface{}, len(q.Calls))
for i, call := range q.Calls {
var v interface{}
var err error
switch call.(type) {
case pql.BitmapCall:
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
case *pql.TopN:
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case *pql.Count:
v, err = pb.Results[i].GetN(), nil
case *pql.SetBit:
v, err = pb.Results[i].GetChanged(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", call))
}
if err != nil {
return nil, err
}
results[i] = v
}
return results, nil
}
// slicesByNode returns a mapping of nodes to slices.

62
executor_test.go
View file

@ -24,13 +24,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 8 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
}
@ -47,7 +47,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 10 { // b1010
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -69,7 +69,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 2 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -92,7 +92,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 5 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -110,10 +110,10 @@ func TestExecutor_Execute_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(3) {
t.Fatalf("unexpected n: %d", n)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
}
}
@ -131,7 +131,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res.(bool) {
if !res[0].(bool) {
t.Fatalf("expected bit changed")
}
}
@ -142,7 +142,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if res, err := e.Execute("d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res.(bool) {
if res[0].(bool) {
t.Fatalf("expected bit unchanged")
}
}
@ -197,7 +197,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
}) {
@ -220,9 +220,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{Key: 0, Count: 4},
}) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
@ -251,11 +251,11 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{Key: 20, Count: 3},
{Key: 10, Count: 2},
{Key: 0, Count: 1},
}) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
@ -273,7 +273,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) {
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -288,7 +288,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Bitmap(id=10, frame=f)` {
@ -303,7 +303,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
(0*SliceWidth)+2,
(2*SliceWidth)+4,
)
return bm, nil
return []interface{}{bm}, nil
}
// Create local executor data.
@ -315,7 +315,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 6 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
@ -334,8 +334,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(10), nil
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(10)}, nil
}
// Create local executor data. The local node owns slice 1.
@ -345,10 +345,10 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(12) {
t.Fatalf("unexpected n: %d", n)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
}
}
@ -364,14 +364,14 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(id=10, frame=f, profileID=2)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
return nil, nil
return []interface{}{nil}, nil
}
// Create local executor data.
@ -403,7 +403,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) {
@ -427,11 +427,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
remoteExecN++
// Return pair counts.
return []pilosa.Pair{
return []interface{}{[]pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
{Key: 30, Count: 2},
}, nil
}}, nil
}
// Create local executor data on slice 1 & 3.
@ -443,11 +443,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []pilosa.Pair{
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{Key: 0, Count: 5},
{Key: 30, Count: 4},
{Key: 10, Count: 2},
}) {
}}) {
t.Fatalf("unexpected results: %s", spew.Sdump(res))
}
}

50
handler.go
View file

@ -29,7 +29,7 @@ type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error)
Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
}
// The version to report on the /version endpoint.
@ -127,12 +127,23 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
// Execute the query.
result, err := h.Executor.Execute(req.DB, q, req.Slices, opt)
resp := &QueryResponse{Result: result, Err: err}
results, err := h.Executor.Execute(req.DB, q, req.Slices, opt)
resp := &QueryResponse{Results: results, Err: err}
// Fill profile attributes if requested.
if bm, ok := result.(*Bitmap); ok && req.Profiles {
profiles, err := h.readProfiles(h.Index.DB(req.DB), bm.Bits())
if req.Profiles {
// Consolidate all profile ids across all calls.
var profileIDs []uint64
for _, result := range results {
bm, ok := result.(*Bitmap)
if !ok {
continue
}
profileIDs = uint64Slice(profileIDs).merge(bm.Bits())
}
// Retrieve profile attributes across all calls.
profiles, err := h.readProfiles(h.Index.DB(req.DB), profileIDs)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
@ -502,9 +513,9 @@ func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
// QueryResponse represent a response from a processed query.
type QueryResponse struct {
// Query execution results.
// Result for each top-level query call.
// Can be a Bitmap, Pairs, or uint64.
Result interface{}
Results []interface{}
// Set of profiles matching IDs returned in Result.
Profiles []*Profile
@ -515,11 +526,11 @@ type QueryResponse struct {
func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
var output struct {
Result interface{} `json:"result,omitempty"`
Profiles []*Profile `json:"profiles,omitempty"`
Err string `json:"error,omitempty"`
Results []interface{} `json:"results,omitempty"`
Profiles []*Profile `json:"profiles,omitempty"`
Err string `json:"error,omitempty"`
}
output.Result = resp.Result
output.Results = resp.Results
output.Profiles = resp.Profiles
if resp.Err != nil {
@ -530,21 +541,22 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
Profiles: encodeProfiles(resp.Profiles),
}
if resp.Result != nil {
switch result := resp.Result.(type) {
for i := range resp.Results {
pb.Results[i] = &internal.QueryResult{}
switch result := resp.Results[i].(type) {
case *Bitmap:
pb.Bitmap = encodeBitmap(result)
pb.Results[i].Bitmap = encodeBitmap(result)
case []Pair:
pb.Pairs = encodePairs(result)
pb.Results[i].Pairs = encodePairs(result)
case uint64:
pb.N = proto.Uint64(result)
pb.Results[i].N = proto.Uint64(result)
case bool:
pb.Changed = proto.Bool(result)
default:
panic(fmt.Sprintf("invalid query result type: %T", resp.Result))
pb.Results[i].Changed = proto.Bool(result)
}
}

76
handler_test.go
View file

@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) {
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -37,14 +37,14 @@ func TestHandler_Query_Args_URL(t *testing.T) {
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -60,7 +60,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
return []interface{}{uint64(100)}, nil
}
// Generate request body.
@ -98,15 +98,15 @@ func TestHandler_Query_Args_Err(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("Count( Bitmap( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -114,8 +114,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
@ -129,25 +129,25 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if resp.GetN() != 100 {
t.Fatalf("unexpected n: %d", resp.GetN())
} else if n := resp.Results[0].GetN(); n != 100 {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d", strings.NewReader("Bitmap(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -169,17 +169,17 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=d&profiles=true", strings.NewReader("Bitmap(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]},"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,65537]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -187,10 +187,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
@ -204,9 +204,9 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
} else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
} else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 {
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
@ -232,10 +232,10 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
return []interface{}{bm}, nil
}
// Encode request body.
@ -261,9 +261,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
} else if attrs := resp.GetBitmap().GetAttrs(); len(attrs) != 3 {
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
@ -287,18 +287,18 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`TopN(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":[{"key":1,"count":2},{"key":3,"count":4}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[[{"key":1,"count":2},{"key":3,"count":4}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
@ -306,11 +306,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}}, nil
}
w := httptest.NewRecorder()
@ -324,7 +324,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetPairs(); len(a) != 2 {
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -493,12 +493,12 @@ func NewHandler() *Handler {
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error)
ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(db, query, slices, opt)
}

65
internal/internal.pb.go
View file

@ -18,6 +18,7 @@ It has these top-level messages:
AttrMap
QueryRequest
QueryResponse
QueryResult
ImportRequest
ImportResponse
Cache
@ -273,13 +274,10 @@ func (m *QueryRequest) GetRemote() bool {
}
type QueryResponse struct {
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"`
Profiles []*Profile `protobuf:"bytes,5,rep" json:"Profiles,omitempty"`
Changed *bool `protobuf:"varint,6,opt" json:"Changed,omitempty"`
XXX_unrecognized []byte `json:"-"`
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Results []*QueryResult `protobuf:"bytes,2,rep" json:"Results,omitempty"`
Profiles []*Profile `protobuf:"bytes,3,rep" json:"Profiles,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
@ -293,23 +291,9 @@ func (m *QueryResponse) GetErr() string {
return ""
}
func (m *QueryResponse) GetBitmap() *Bitmap {
func (m *QueryResponse) GetResults() []*QueryResult {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResponse) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResponse) GetPairs() []*Pair {
if m != nil {
return m.Pairs
return m.Results
}
return nil
}
@ -321,7 +305,40 @@ func (m *QueryResponse) GetProfiles() []*Profile {
return nil
}
func (m *QueryResponse) GetChanged() bool {
type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,2,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep" json:"Pairs,omitempty"`
Changed *bool `protobuf:"varint,4,opt" json:"Changed,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (m *QueryResult) GetBitmap() *Bitmap {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResult) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResult) GetPairs() []*Pair {
if m != nil {
return m.Pairs
}
return nil
}
func (m *QueryResult) GetChanged() bool {
if m != nil && m.Changed != nil {
return *m.Changed
}

16
internal/internal.proto
View file

@ -47,12 +47,16 @@ message QueryRequest {
}
message QueryResponse {
optional string Err = 1;
optional Bitmap Bitmap = 2;
optional uint64 N = 3;
repeated Pair Pairs = 4;
repeated Profile Profiles = 5;
optional bool Changed = 6;
optional string Err = 1;
repeated QueryResult Results = 2;
repeated Profile Profiles = 3;
}
message QueryResult {
optional Bitmap Bitmap = 1;
optional uint64 N = 2;
repeated Pair Pairs = 3;
optional bool Changed = 4;
}
message ImportRequest {

10
pql/ast.go
View file

@ -10,11 +10,17 @@ import (
// Query represents a PQL query.
type Query struct {
Root Call
Calls Calls
}
// String returns a string representation of the query.
func (q *Query) String() string { return q.Root.String() }
func (q *Query) String() string {
a := make([]string, len(q.Calls))
for i, call := range q.Calls {
a[i] = call.String()
}
return strings.Join(a, "\n")
}
// Node represents any node in the AST.
type Node interface {

24
pql/parser.go
View file

@ -30,17 +30,31 @@ func ParseString(s string) (*Query, error) {
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
fn, err := p.parseCall()
if err != nil {
return nil, err
q := &Query{}
for {
call, err := p.parseCall()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
q.Calls = append(q.Calls, call)
}
return &Query{Root: fn}, nil
// Require at least one call.
if len(q.Calls) == 0 {
return nil, io.ErrUnexpectedEOF
}
return q, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
tok, pos, lit := p.scanIgnoreWhitespace()
if tok != IDENT {
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}

190
pql/parser_test.go
View file

@ -15,9 +15,11 @@ func TestParser_Parse_Bitmap_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Bitmap{
ID: 1,
Frame: "b.n",
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -30,9 +32,11 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Bitmap{
ID: 1,
Frame: "b.n",
Calls: pql.Calls{
&pql.Bitmap{
ID: 1,
Frame: "b.n",
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -45,10 +49,12 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -61,10 +67,12 @@ func TestParser_Parse_ClearBit_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.ClearBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -77,9 +85,11 @@ func TestParser_Parse_Count(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Count{
Input: &pql.Bitmap{
ID: 1,
Calls: pql.Calls{
&pql.Count{
Input: &pql.Bitmap{
ID: 1,
},
},
},
}) {
@ -93,10 +103,12 @@ func TestParser_Parse_Difference(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {
@ -110,10 +122,12 @@ func TestParser_Parse_Intersect(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {
@ -127,7 +141,9 @@ func TestParser_Parse_Profile_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Profile{ID: 1},
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
@ -139,7 +155,9 @@ func TestParser_Parse_Profile_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Profile{ID: 1},
Calls: pql.Calls{
&pql.Profile{ID: 1},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
@ -151,11 +169,13 @@ func TestParser_Parse_Range_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -168,11 +188,13 @@ func TestParser_Parse_Range_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
Calls: pql.Calls{
&pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -185,10 +207,12 @@ func TestParser_Parse_SetBit_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -201,10 +225,12 @@ func TestParser_Parse_SetBit_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
Calls: pql.Calls{
&pql.SetBit{
ID: 1,
Frame: "b.n",
ProfileID: 3,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -217,15 +243,17 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
},
},
},
}) {
@ -239,12 +267,14 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
Calls: pql.Calls{
&pql.SetBitmapAttrs{
ID: 1,
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": uint64(123),
},
},
},
}) {
@ -258,13 +288,15 @@ func TestParser_Parse_TopN_Key(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
BitmapIDs: []uint64{1, 2, 3},
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
BitmapIDs: []uint64{1, 2, 3},
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -281,12 +313,14 @@ func TestParser_Parse_TopN_Array(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
Calls: pql.Calls{
&pql.TopN{
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
@ -303,10 +337,12 @@ func TestParser_Parse_Union(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Union{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
Calls: pql.Calls{
&pql.Union{
Inputs: pql.BitmapCalls{
&pql.Bitmap{ID: 1},
&pql.Bitmap{ID: 2},
},
},
},
}) {

2
roaring/roaring_test.go
View file

@ -165,7 +165,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
// Add more values to bitmap.
for _, v := range a1 {
set[v] = struct{}{}
if err := bm.Add(v); err != nil {
if _, err := bm.Add(v); err != nil {
t.Fatal(err)
}