scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of ops:nuevo-pilosa

Browse source
This commit is contained in:
Todd Gruben 2014-02-06 07:08:03 -06:00
commit 4d17ff5242
9 changed files with 69 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
core/query.go
View file

@ -42,7 +42,7 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) {
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, 8) //TODO: get the N from the query (default is 8)
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N)
if err != nil {
spew.Dump(err)
}
@ -137,10 +137,11 @@ func (self *Service) CatQueryStepHandler(msg *db.Message) {
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
case uint64:
//spew.Dump(val)
return_type = "sum"
sum += val
case []index.Pair:
spew.Dump(val)
//spew.Dump(val)
return_type = "pair-list"
for _, pair := range val {
_, ok := merge_map[pair.Key]
@ -175,7 +176,7 @@ func (self *Service) CatQueryStepHandler(msg *db.Message) {
for _, r := range rank_list {
pair_list = append(pair_list, *r.Pair)
}
result = pair_list[:8] //TODO: get the N from the query (default is 8)
result = pair_list[:qs.N]
} else {
result = "NONE"
}
@ -209,10 +210,7 @@ func (self *Service) GetQueryStepHandler(msg *db.Message) {
func (self *Service) SetQueryStepHandler(msg *db.Message) {
//spew.Dump("SET QUERYSTEP")
qs := msg.Data.(query.SetQueryStep)
//result, err := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
result, _ := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
//spew.Dump("result:", result)
//spew.Dump("err:", err)
result_message := db.Message{Data: query.SetQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: result}}}
self.Transport.Send(&result_message, qs.Destination.ProcessId)

1
executor/executor.go
View file

@ -122,7 +122,6 @@ func (self *Executor) RunPQL(database_name string, pql string) interface{} {
if err != nil {
spew.Dump(err)
}
spew.Dump(final)
final_result[query_list[i].Label] = final
}

1
index/brand.go
View file

@ -176,7 +176,6 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) {
func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []int) []Pair {
self.rank_counter = 0
println("RANK")
self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP
is := new(IntSet)
for _, v := range categories {

21
query/lexer.go
View file

@ -16,6 +16,7 @@ const (
TYPE_FRAME = iota
TYPE_PROFILE = iota
TYPE_COMMA = iota
TYPE_LIMIT = iota
)
type Token struct {
@ -158,7 +159,7 @@ func stateRP(lexer *Lexer) statefn {
peeked := lexer.peek()
if peeked == rune(',') {
return stateComma
return stateRPComma
} else if peeked == rune(')') {
return stateRP
} else {
@ -193,6 +194,24 @@ func stateFrameOrProfile(lexer *Lexer) statefn {
return stateRP
}
func stateRPComma(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_COMMA)
if unicode.IsNumber(lexer.peek()) {
return stateLimit
} else {
return stateArgs
}
}
func stateLimit(lexer *Lexer) statefn {
lexer.acceptNumber()
lexer.emit(TYPE_LIMIT)
// if next is comma
lexer.peek()
return stateRP
}
func stateComma(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_COMMA)

20
query/lexer_test.go
View file

@ -162,5 +162,25 @@ func TestLexer(t *testing.T) {
So(tokens6[7].Text, ShouldEqual, ")")
So(tokens6[7].Type, ShouldEqual, TYPE_RP)
tokens7 := Lex("top-n(get(10), 8)")
So(len(tokens6), ShouldEqual, 8)
So(tokens7[0].Text, ShouldEqual, "top-n")
So(tokens7[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens7[1].Text, ShouldEqual, "(")
So(tokens7[1].Type, ShouldEqual, TYPE_LP)
So(tokens7[2].Text, ShouldEqual, "get")
So(tokens7[2].Type, ShouldEqual, TYPE_FUNC)
So(tokens7[3].Text, ShouldEqual, "(")
So(tokens7[3].Type, ShouldEqual, TYPE_LP)
So(tokens7[4].Text, ShouldEqual, "10")
So(tokens7[4].Type, ShouldEqual, TYPE_ID)
So(tokens7[5].Text, ShouldEqual, ")")
So(tokens7[5].Type, ShouldEqual, TYPE_RP)
So(tokens7[6].Text, ShouldEqual, ",")
So(tokens7[6].Type, ShouldEqual, TYPE_COMMA)
So(tokens7[7].Text, ShouldEqual, "8")
So(tokens7[7].Type, ShouldEqual, TYPE_LIMIT)
So(tokens7[8].Text, ShouldEqual, ")")
So(tokens7[8].Type, ShouldEqual, TYPE_RP)
})
}

12
query/parser.go
View file

@ -12,7 +12,7 @@ var InvalidQueryError = errors.New("Invalid query format.")
type QueryParser struct{}
func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) {
func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64, int) {
// BITMAP
if tokens[0].Type == TYPE_ID {
// TODO: look for frame type in the tokens list
@ -40,10 +40,11 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) {
}
}
bm := db.Bitmap{bitmap_id, frame_type}
return []QueryInput{&bm}, uint64(profile_id)
return []QueryInput{&bm}, uint64(profile_id), 0
}
// LIST OF QUERIES
n := int(10) // default LIMIT to 10
qi := []QueryInput{}
open_parens := -1 // >=0 means i'm inside the search for end paren
start := 0
@ -63,9 +64,12 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) {
} else {
open_parens--
}
} else if tokens[i].Type == TYPE_LIMIT {
x, _ := strconv.ParseInt(tokens[i].Text, 10, 32)
n = int(x)
}
}
return qi, 0
return qi, 0, n
}
func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
@ -91,7 +95,7 @@ func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
} else if tokens[i].Type == TYPE_RP {
if open_parens == 0 {
if i == len(tokens)-1 {
q.Inputs, q.ProfileId = qp.walkInputs(tokens[2:i])
q.Inputs, q.ProfileId, q.N = qp.walkInputs(tokens[2:i])
}
} else {
open_parens--

22
query/planner.go
View file

@ -94,6 +94,7 @@ func (qt *CountQueryTree) getLocation(d *db.Database) *db.Location {
type TopNQueryStep struct {
*BaseQueryStep
Input *uuid.UUID
N int
}
type TopNQueryResult struct {
@ -179,6 +180,7 @@ func (qt *IntersectQueryTree) getLocation(d *db.Database) *db.Location {
type CatQueryStep struct {
*BaseQueryStep
Inputs []*uuid.UUID
N int
}
type CatQueryResult struct {
@ -372,7 +374,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
}
// Produces flattened QueryPlan from QueryTree input
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location) *QueryPlan {
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location, n int) *QueryPlan {
plan := QueryPlan{}
if composite, ok := qt.(*CompositeQueryTree); ok {
inputs := make([]QueryInput, len(composite.subqueries))
@ -380,17 +382,17 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
for index, subq := range composite.subqueries {
sub_id := uuid.RandomUUID()
step.inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database))
subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if cat, ok := qt.(*CatQueryTree); ok {
inputs := make([]*uuid.UUID, len(cat.subqueries))
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs}
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, n}
for index, subq := range cat.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database))
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -400,7 +402,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
for index, subq := range union.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database))
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -410,7 +412,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
for index, subq := range intersect.subqueries {
sub_id := uuid.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database))
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
@ -425,13 +427,13 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
} else if cnt, ok := qt.(*CountQueryTree); ok {
sub_id := uuid.RandomUUID()
step := &CountQueryStep{&BaseQueryStep{id, "count", cnt.getLocation(qp.Database), location}, &sub_id}
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database))
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := uuid.RandomUUID()
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id}
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database))
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, n}
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database), n)
plan = append(plan, *subq_steps...)
plan = append(plan, step)
}
@ -442,5 +444,5 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan {
queryTree := qp.buildTree(query, -1)
//return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value
return qp.flatten(queryTree, query.Id, destination)
return qp.flatten(queryTree, query.Id, destination, query.N)
}

6
query/planner_test.go
View file

@ -16,16 +16,16 @@ func TestQueryPlanner(t *testing.T) {
id1 := uuid.RandomUUID()
bm1 := db.Bitmap{10, "general"}
inputs1 := []QueryInput{&bm1}
query1 := Query{&id1, "get", inputs1, 0}
query1 := Query{&id1, "get", inputs1, 0, 0}
id2 := uuid.RandomUUID()
bm2 := db.Bitmap{20, "general"}
inputs2 := []QueryInput{&bm2}
query2 := Query{&id2, "get", inputs2, 0}
query2 := Query{&id2, "get", inputs2, 0, 0}
id3 := uuid.RandomUUID()
inputs := []QueryInput{&query1, &query2}
query := Query{&id3, "union", inputs, 0}
query := Query{&id3, "union", inputs, 0, 0}
/*
bm1 := db.Bitmap{10, "general"}

3
query/query.go
View file

@ -27,7 +27,8 @@ type Query struct {
Inputs []QueryInput //"strconv"
// Represents a parsed query. Inputs can be Query or Bitmap objects
// Maybe Bitmap and Query objects should have different fields to avoid using interface{}
ProfileId uint64
ProfileId uint64 // used only for set() queries
N int // TODO: I think we should make this a generic map for any attributes related to the query
}
func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan {