scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Query planner - make work with new args from query parser, remove "N" variable

Browse source
This commit is contained in:
Cody Soyland 2014-03-18 18:40:40 -05:00
parent f375ad9de5
commit c858da3fa7
7 changed files with 217 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/query.go
View file

@ -45,7 +45,7 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) {
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N, categoryleaves)
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, categoryleaves)
if err != nil {
spew.Dump(err)
}

4
query/lexer.go
View file

@ -43,7 +43,9 @@ type Lexer struct {
}
func (lexer *Lexer) emit(typ int) {
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
if lexer.start < lexer.pos {
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
}
lexer.start = lexer.pos
}

11
query/parser.go
View file

@ -111,6 +111,12 @@ ArgLoop:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "top-n":
i, err := strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
query.Args["n"] = i
default:
spew.Dump("UNPROCESSED VALUE", token)
}
@ -135,6 +141,11 @@ ArgLoop:
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
} else if keyword == "n" {
value, err = strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
} else {
value = token.Text
}

4
query/parser_test.go
View file

@ -64,14 +64,14 @@ func TestQueryParser(t *testing.T) {
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "brands"})
})
Convey("Lists", t, func() {
tokens, err := Lex("top-n(get(10, general), [1,2,3])")
tokens, err := Lex("top-n(get(10, general), [1,2,3], 50)")
So(err, ShouldBeNil)
query, err := Parse(tokens)
So(err, ShouldBeNil)
So(query.Operation, ShouldEqual, "top-n")
So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}})
So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}, "n": 50})
So(len(query.Subqueries), ShouldEqual, 1)
So(query.Subqueries[0].Operation, ShouldEqual, "get")

109
query/planner.go
View file

@ -2,26 +2,12 @@ package query
import (
"encoding/gob"
"fmt"
"math/rand"
"pilosa/db"
"tux21b.org/v1/gocql/uuid"
)
// A single step in the query plan.
type QueryStep struct {
id *uuid.UUID
operation string
inputs []QueryInput
location *db.Location
destination *db.Location
}
func (q QueryStep) StringHOLD() string {
return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.destination)
}
type PortableQueryStep interface {
GetId() *uuid.UUID
GetLocation() *db.Location
@ -105,6 +91,7 @@ type TopNQueryResult struct {
type TopNQueryTree struct {
subquery QueryTree
location *db.Location
N int
}
// Uses consistent hashing function to select node containing data for GET operation
@ -191,6 +178,7 @@ type CatQueryResult struct {
type CatQueryTree struct {
subqueries []QueryTree
location *db.Location
N int
}
// Uses consistent hashing function to select node containing data for GET operation
@ -292,44 +280,31 @@ type QueryPlan []interface{}
type QueryPlanner struct {
Database *db.Database
Query *Query
}
type QueryTree interface {
getLocation(d *db.Database) *db.Location
}
// QueryTree for UNION and INTERSECT queries
type CompositeQueryTree struct {
operation string
subqueries []QueryTree
location *db.Location
}
// Randomly select location from subqueries (so subqueries roll up into composite queries while minimizing inter-node data traffic)
func (qt *CompositeQueryTree) getLocation(d *db.Database) *db.Location {
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location = subquery.getLocation(d)
}
}
return qt.location
}
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
var tree QueryTree
// handle SET operation regardless of the slice
if query.Operation == "set" {
tree = &SetQueryTree{query.Inputs[0].(*db.Bitmap), query.ProfileId}
tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, query.Args["profile_id"].(uint64)}
return tree
}
// handle the remaining operations, taking slice into consideration
if slice == -1 {
tree = &CatQueryTree{}
var n int
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
tree = &CatQueryTree{N: n}
numSlices, err := qp.Database.NumSlices()
if err != nil {
panic(err)
@ -342,57 +317,48 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
}
} else {
if query.Operation == "get" {
tree = &GetQueryTree{query.Inputs[0].(*db.Bitmap), slice}
tree = &GetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, slice}
return tree
} else if query.Operation == "count" {
subquery := qp.buildTree(query.Inputs[0].(*Query), slice)
subquery := qp.buildTree(&query.Subqueries[0], slice)
tree = &CountQueryTree{subquery: subquery}
} else if query.Operation == "top-n" {
subquery := qp.buildTree(query.Inputs[0].(*Query), slice)
tree = &TopNQueryTree{subquery: subquery}
var n int
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
subquery := qp.buildTree(&query.Subqueries[0], slice)
tree = &TopNQueryTree{subquery: subquery, N: n}
} else if query.Operation == "union" {
subqueries := make([]QueryTree, len(query.Inputs))
for i, input := range query.Inputs {
subqueries[i] = qp.buildTree(input.(*Query), slice)
subqueries := make([]QueryTree, len(query.Subqueries))
for i, query := range query.Subqueries {
subqueries[i] = qp.buildTree(&query, slice)
}
tree = &UnionQueryTree{subqueries: subqueries}
} else if query.Operation == "intersect" {
subqueries := make([]QueryTree, len(query.Inputs))
for i, input := range query.Inputs {
subqueries[i] = qp.buildTree(input.(*Query), slice)
subqueries := make([]QueryTree, len(query.Subqueries))
for i, query := range query.Subqueries {
subqueries[i] = qp.buildTree(&query, slice)
}
tree = &IntersectQueryTree{subqueries: subqueries}
} else {
subqueries := make([]QueryTree, len(query.Inputs))
for i, input := range query.Inputs {
subqueries[i] = qp.buildTree(input.(*Query), slice)
}
tree = &CompositeQueryTree{operation: query.Operation, subqueries: subqueries}
panic("invalid operation")
}
}
return tree
}
// Produces flattened QueryPlan from QueryTree input
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location, n int) *QueryPlan {
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location) *QueryPlan {
plan := QueryPlan{}
if composite, ok := qt.(*CompositeQueryTree); ok {
inputs := make([]QueryInput, len(composite.subqueries))
step := QueryStep{id, composite.operation, inputs, composite.getLocation(qp.Database), location}
for index, subq := range composite.subqueries {
sub_id := uuid.RandomUUID()
step.inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if cat, ok := qt.(*CatQueryTree); ok {
if cat, ok := qt.(*CatQueryTree); ok {
inputs := make([]*uuid.UUID, len(cat.subqueries))
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, n}
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, cat.N}
for index, subq := range cat.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database), n)
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -402,7 +368,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
for index, subq := range union.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database), n)
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -412,7 +378,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
for index, subq := range intersect.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database), n)
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -427,13 +393,13 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
} else if cnt, ok := qt.(*CountQueryTree); ok {
sub_id := uuid.RandomUUID()
step := &CountQueryStep{&BaseQueryStep{id, "count", cnt.getLocation(qp.Database), location}, &sub_id}
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database), n)
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := uuid.RandomUUID()
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, n}
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database), n)
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.N}
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
plan = append(plan, step)
}
@ -443,6 +409,5 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
// Transforms Query into QueryTree and flattens to QueryPlan object
func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan {
queryTree := qp.buildTree(query, -1)
//return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value
return qp.flatten(queryTree, query.Id, destination, query.N)
return qp.flatten(queryTree, query.Id, destination)
}

207
query/planner_test.go
View file

@ -4,67 +4,184 @@ import (
"pilosa/db"
"pilosa/util"
"testing"
"github.com/davecgh/go-spew/spew"
. "github.com/smartystreets/goconvey/convey"
"tux21b.org/v1/gocql/uuid"
)
func basic_database() (*db.Database, *db.Fragment) {
// create an empty database
cluster := db.NewCluster()
database := cluster.GetOrCreateDatabase("main")
frame := database.GetOrCreateFrame("general")
slice1 := database.GetOrCreateSlice(0)
fragment_id1 := util.Id()
fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1)
process_id1 := uuid.RandomUUID()
process1 := db.NewProcess(&process_id1)
process1.SetHost("----192.1.1.0----")
fragment1.SetProcess(process1)
slice2 := database.GetOrCreateSlice(1)
fragment_id2 := util.Id()
fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2)
process_id2 := uuid.RandomUUID()
process2 := db.NewProcess(&process_id2)
process2.SetHost("----192.1.1.1----")
fragment2.SetProcess(process2)
return database, fragment1
}
func TestQueryPlanner(t *testing.T) {
Convey("Basic query plan", t, func() {
Convey("Union query plan", t, func() {
id1 := uuid.RandomUUID()
bm1 := db.Bitmap{10, "general"}
inputs1 := []QueryInput{&bm1}
query1 := Query{&id1, "get", inputs1, 0, 0}
query1 := Query{Id: &id1, Operation: "get", Args: map[string]interface{}{"id": uint64(10), "frame": "general"}}
id2 := uuid.RandomUUID()
bm2 := db.Bitmap{20, "general"}
inputs2 := []QueryInput{&bm2}
query2 := Query{&id2, "get", inputs2, 0, 0}
query2 := Query{Id: &id2, Operation: "get", Args: map[string]interface{}{"id": uint64(20), "frame": "general"}}
id3 := uuid.RandomUUID()
inputs := []QueryInput{&query1, &query2}
query := Query{&id3, "union", inputs, 0, 0}
/*
query := Query{Id: &id3, Operation: "union", Subqueries: []Query{query1, query2}}
bm1 := db.Bitmap{10, "general"}
inputs1 := []QueryInput{&bm1}
query1 := Query{"get", inputs1}
query := query1
*/
database, fragment1 := basic_database()
// create an empty database
cluster := db.NewCluster()
database := cluster.GetOrCreateDatabase("main")
frame := database.GetOrCreateFrame("general")
slice1 := database.GetOrCreateSlice(0)
fragment_id1 := util.Id()
fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1)
process_id1 := uuid.RandomUUID()
process1 := db.NewProcess(&process_id1)
process1.SetHost("----192.1.1.0----")
fragment1.SetProcess(process1)
slice2 := database.GetOrCreateSlice(1)
fragment_id2 := util.Id()
fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2)
process_id2 := uuid.RandomUUID()
process2 := db.NewProcess(&process_id2)
process2.SetHost("----192.1.1.1----")
fragment2.SetProcess(process2)
qplanner := QueryPlanner{Database: database}
qplanner := QueryPlanner{Database: database, Query: &query}
destination := fragment1.GetLocation()
id := uuid.RandomUUID()
qp := qplanner.Plan(&query, &id, destination)
qp := *qplanner.Plan(&query, &id, destination)
for i, qs := range *qp {
//spew.Dump(i, qs, qs.inputs)
spew.Dump(i, qs)
spew.Dump("**************************************************************")
}
So(len(qp), ShouldEqual, 7)
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0)
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union")
So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[0].(GetQueryStep).Id,
qp[1].(GetQueryStep).Id,
})
So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1)
So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1)
So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union")
So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[3].(GetQueryStep).Id,
qp[4].(GetQueryStep).Id,
})
So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat")
So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[2].(UnionQueryStep).Id,
qp[5].(UnionQueryStep).Id,
})
})
Convey("Get query plan - including parsing", t, func() {
query := QueryForPQL("get(10,general)")
database, fragment1 := basic_database()
qplanner := QueryPlanner{Database: database, Query: query}
destination := fragment1.GetLocation()
id := uuid.RandomUUID()
qp := *qplanner.Plan(query, &id, destination)
So(len(qp), ShouldEqual, 3)
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 1)
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[2].(CatQueryStep).Operation, ShouldEqual, "cat")
So(qp[2].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[0].(GetQueryStep).Id,
qp[1].(GetQueryStep).Id,
})
})
Convey("Union query plan - including parsing", t, func() {
query := QueryForPQL("union(get(10, general), get(20, general))")
database, fragment1 := basic_database()
qplanner := QueryPlanner{Database: database, Query: query}
destination := fragment1.GetLocation()
id := uuid.RandomUUID()
qp := *qplanner.Plan(query, &id, destination)
So(len(qp), ShouldEqual, 7)
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0)
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union")
So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[0].(GetQueryStep).Id,
qp[1].(GetQueryStep).Id,
})
So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1)
So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get")
So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1)
So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union")
So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[3].(GetQueryStep).Id,
qp[4].(GetQueryStep).Id,
})
So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat")
So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
qp[2].(UnionQueryStep).Id,
qp[5].(UnionQueryStep).Id,
})
})
Convey("Set query plan - including parsing", t, func() {
query := QueryForPQL("set(10, general, 100)")
database, fragment1 := basic_database()
qplanner := QueryPlanner{Database: database, Query: query}
destination := fragment1.GetLocation()
id := uuid.RandomUUID()
qp := *qplanner.Plan(query, &id, destination)
So(len(qp), ShouldEqual, 1)
So(qp[0].(SetQueryStep).Operation, ShouldEqual, "set")
So(qp[0].(SetQueryStep).ProfileId, ShouldEqual, 100)
So(*(qp[0].(SetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
})
Convey("Top-n query plan - including parsing", t, func() {
query := QueryForPQL("top-n(get(10, general), [1,2,3], 50)")
database, fragment1 := basic_database()
qplanner := QueryPlanner{Database: database, Query: query}
destination := fragment1.GetLocation()
id := uuid.RandomUUID()
qp := *qplanner.Plan(query, &id, destination)
So(len(qp), ShouldEqual, 5)
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[1].(*TopNQueryStep).Operation, ShouldEqual, "top-n")
So(qp[1].(*TopNQueryStep).Input, ShouldEqual, qp[0].(GetQueryStep).Id)
So(qp[1].(*TopNQueryStep).N, ShouldEqual, 50)
So(qp[2].(GetQueryStep).Operation, ShouldEqual, "get")
So(*(qp[2].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
So(qp[3].(*TopNQueryStep).Operation, ShouldEqual, "top-n")
So(qp[3].(*TopNQueryStep).Input, ShouldEqual, qp[2].(GetQueryStep).Id)
So(qp[3].(*TopNQueryStep).N, ShouldEqual, 50)
})
}

9
query/query.go
View file

@ -26,13 +26,6 @@ type Query struct {
Operation string
Args map[string]interface{}
Subqueries []Query
// deprecated:
Inputs []QueryInput //"strconv"
// Represents a parsed query. Inputs can be Query or Bitmap objects
// Maybe Bitmap and Query objects should have different fields to avoid using interface{}
ProfileId uint64 // used only for set() queries
N int // TODO: I think we should make this a generic map for any attributes related to the query
}
func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan {
@ -65,7 +58,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L
}
func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) *QueryPlan {
query_planner := QueryPlanner{Database: database}
query_planner := QueryPlanner{Database: database, Query: query}
id := uuid.RandomUUID()
query_plan := query_planner.Plan(query, &id, destination)
return query_plan