From b05ad49ec4325df0f48c8177a11596ac497c7653 Mon Sep 17 00:00:00 2001 From: travisturner Date: Wed, 5 Feb 2014 19:51:37 -0600 Subject: [PATCH 1/2] add support for top-n LIMIT --- core/query.go | 10 ++++------ query/lexer.go | 21 ++++++++++++++++++++- query/lexer_test.go | 20 ++++++++++++++++++++ query/parser.go | 12 ++++++++---- query/planner.go | 22 ++++++++++++---------- query/planner_test.go | 6 +++--- query/query.go | 3 ++- 7 files changed, 69 insertions(+), 25 deletions(-) diff --git a/core/query.go b/core/query.go index 13e922ccf..7fb4bc867 100644 --- a/core/query.go +++ b/core/query.go @@ -42,7 +42,7 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) { case []byte: bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val) } - topn, err := self.Index.TopN(qs.Location.FragmentId, bh, 8) //TODO: get the N from the query (default is 8) + topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N) if err != nil { spew.Dump(err) } @@ -137,10 +137,11 @@ func (self *Service) CatQueryStepHandler(msg *db.Message) { bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val) handles = append(handles, bh) case uint64: + //spew.Dump(val) return_type = "sum" sum += val case []index.Pair: - spew.Dump(val) + //spew.Dump(val) return_type = "pair-list" for _, pair := range val { _, ok := merge_map[pair.Key] @@ -175,7 +176,7 @@ func (self *Service) CatQueryStepHandler(msg *db.Message) { for _, r := range rank_list { pair_list = append(pair_list, *r.Pair) } - result = pair_list[:8] //TODO: get the N from the query (default is 8) + result = pair_list[:qs.N] } else { result = "NONE" } @@ -209,10 +210,7 @@ func (self *Service) GetQueryStepHandler(msg *db.Message) { func (self *Service) SetQueryStepHandler(msg *db.Message) { //spew.Dump("SET QUERYSTEP") qs := msg.Data.(query.SetQueryStep) - //result, err := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId) result, _ := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId) - //spew.Dump("result:", result) - //spew.Dump("err:", err) result_message := db.Message{Data: query.SetQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: result}}} self.Transport.Send(&result_message, qs.Destination.ProcessId) diff --git a/query/lexer.go b/query/lexer.go index 5297bfd2f..d64be71c5 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -16,6 +16,7 @@ const ( TYPE_FRAME = iota TYPE_PROFILE = iota TYPE_COMMA = iota + TYPE_LIMIT = iota ) type Token struct { @@ -158,7 +159,7 @@ func stateRP(lexer *Lexer) statefn { peeked := lexer.peek() if peeked == rune(',') { - return stateComma + return stateRPComma } else if peeked == rune(')') { return stateRP } else { @@ -193,6 +194,24 @@ func stateFrameOrProfile(lexer *Lexer) statefn { return stateRP } +func stateRPComma(lexer *Lexer) statefn { + lexer.pos += 1 + lexer.emit(TYPE_COMMA) + if unicode.IsNumber(lexer.peek()) { + return stateLimit + } else { + return stateArgs + } +} + +func stateLimit(lexer *Lexer) statefn { + lexer.acceptNumber() + lexer.emit(TYPE_LIMIT) + // if next is comma + lexer.peek() + return stateRP +} + func stateComma(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_COMMA) diff --git a/query/lexer_test.go b/query/lexer_test.go index dc52d93cf..3549dc790 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -162,5 +162,25 @@ func TestLexer(t *testing.T) { So(tokens6[7].Text, ShouldEqual, ")") So(tokens6[7].Type, ShouldEqual, TYPE_RP) + tokens7 := Lex("top-n(get(10), 8)") + So(len(tokens6), ShouldEqual, 8) + So(tokens7[0].Text, ShouldEqual, "top-n") + So(tokens7[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens7[1].Text, ShouldEqual, "(") + So(tokens7[1].Type, ShouldEqual, TYPE_LP) + So(tokens7[2].Text, ShouldEqual, "get") + So(tokens7[2].Type, ShouldEqual, TYPE_FUNC) + So(tokens7[3].Text, ShouldEqual, "(") + So(tokens7[3].Type, ShouldEqual, TYPE_LP) + So(tokens7[4].Text, ShouldEqual, "10") + So(tokens7[4].Type, ShouldEqual, TYPE_ID) + So(tokens7[5].Text, ShouldEqual, ")") + So(tokens7[5].Type, ShouldEqual, TYPE_RP) + So(tokens7[6].Text, ShouldEqual, ",") + So(tokens7[6].Type, ShouldEqual, TYPE_COMMA) + So(tokens7[7].Text, ShouldEqual, "8") + So(tokens7[7].Type, ShouldEqual, TYPE_LIMIT) + So(tokens7[8].Text, ShouldEqual, ")") + So(tokens7[8].Type, ShouldEqual, TYPE_RP) }) } diff --git a/query/parser.go b/query/parser.go index 876364f5e..08ceea04a 100644 --- a/query/parser.go +++ b/query/parser.go @@ -12,7 +12,7 @@ var InvalidQueryError = errors.New("Invalid query format.") type QueryParser struct{} -func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) { +func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64, int) { // BITMAP if tokens[0].Type == TYPE_ID { // TODO: look for frame type in the tokens list @@ -40,10 +40,11 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) { } } bm := db.Bitmap{bitmap_id, frame_type} - return []QueryInput{&bm}, uint64(profile_id) + return []QueryInput{&bm}, uint64(profile_id), 0 } // LIST OF QUERIES + n := int(10) // default LIMIT to 10 qi := []QueryInput{} open_parens := -1 // >=0 means i'm inside the search for end paren start := 0 @@ -63,9 +64,12 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) { } else { open_parens-- } + } else if tokens[i].Type == TYPE_LIMIT { + x, _ := strconv.ParseInt(tokens[i].Text, 10, 32) + n = int(x) } } - return qi, 0 + return qi, 0, n } func (qp *QueryParser) walk(tokens []Token) (*Query, error) { @@ -91,7 +95,7 @@ func (qp *QueryParser) walk(tokens []Token) (*Query, error) { } else if tokens[i].Type == TYPE_RP { if open_parens == 0 { if i == len(tokens)-1 { - q.Inputs, q.ProfileId = qp.walkInputs(tokens[2:i]) + q.Inputs, q.ProfileId, q.N = qp.walkInputs(tokens[2:i]) } } else { open_parens-- diff --git a/query/planner.go b/query/planner.go index ac36a12b5..f6f11523e 100644 --- a/query/planner.go +++ b/query/planner.go @@ -94,6 +94,7 @@ func (qt *CountQueryTree) getLocation(d *db.Database) *db.Location { type TopNQueryStep struct { *BaseQueryStep Input *uuid.UUID + N int } type TopNQueryResult struct { @@ -179,6 +180,7 @@ func (qt *IntersectQueryTree) getLocation(d *db.Database) *db.Location { type CatQueryStep struct { *BaseQueryStep Inputs []*uuid.UUID + N int } type CatQueryResult struct { @@ -372,7 +374,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { } // Produces flattened QueryPlan from QueryTree input -func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location) *QueryPlan { +func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location, n int) *QueryPlan { plan := QueryPlan{} if composite, ok := qt.(*CompositeQueryTree); ok { inputs := make([]QueryInput, len(composite.subqueries)) @@ -380,17 +382,17 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati for index, subq := range composite.subqueries { sub_id := uuid.RandomUUID() step.inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database)) + subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) } plan = append(plan, step) } else if cat, ok := qt.(*CatQueryTree); ok { inputs := make([]*uuid.UUID, len(cat.subqueries)) - step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs} + step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, n} for index, subq := range cat.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database)) + subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -400,7 +402,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati for index, subq := range union.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database)) + subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -410,7 +412,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati for index, subq := range intersect.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database)) + subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -425,13 +427,13 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati } else if cnt, ok := qt.(*CountQueryTree); ok { sub_id := uuid.RandomUUID() step := &CountQueryStep{&BaseQueryStep{id, "count", cnt.getLocation(qp.Database), location}, &sub_id} - subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database)) + subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) plan = append(plan, step) } else if topn, ok := qt.(*TopNQueryTree); ok { sub_id := uuid.RandomUUID() - step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id} - subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database)) + step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, n} + subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database), n) plan = append(plan, *subq_steps...) plan = append(plan, step) } @@ -442,5 +444,5 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan { queryTree := qp.buildTree(query, -1) //return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value - return qp.flatten(queryTree, query.Id, destination) + return qp.flatten(queryTree, query.Id, destination, query.N) } diff --git a/query/planner_test.go b/query/planner_test.go index 97a67ba8e..a2cda6627 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -16,16 +16,16 @@ func TestQueryPlanner(t *testing.T) { id1 := uuid.RandomUUID() bm1 := db.Bitmap{10, "general"} inputs1 := []QueryInput{&bm1} - query1 := Query{&id1, "get", inputs1, 0} + query1 := Query{&id1, "get", inputs1, 0, 0} id2 := uuid.RandomUUID() bm2 := db.Bitmap{20, "general"} inputs2 := []QueryInput{&bm2} - query2 := Query{&id2, "get", inputs2, 0} + query2 := Query{&id2, "get", inputs2, 0, 0} id3 := uuid.RandomUUID() inputs := []QueryInput{&query1, &query2} - query := Query{&id3, "union", inputs, 0} + query := Query{&id3, "union", inputs, 0, 0} /* bm1 := db.Bitmap{10, "general"} diff --git a/query/query.go b/query/query.go index e865e430b..f1a7cc642 100644 --- a/query/query.go +++ b/query/query.go @@ -27,7 +27,8 @@ type Query struct { Inputs []QueryInput //"strconv" // Represents a parsed query. Inputs can be Query or Bitmap objects // Maybe Bitmap and Query objects should have different fields to avoid using interface{} - ProfileId uint64 + ProfileId uint64 // used only for set() queries + N int // TODO: I think we should make this a generic map for any attributes related to the query } func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan { From fd2a212d30ddfeb5d780cd0e21c5f26e119fede2 Mon Sep 17 00:00:00 2001 From: travisturner Date: Wed, 5 Feb 2014 20:04:11 -0600 Subject: [PATCH 2/2] remove some spews --- executor/executor.go | 1 - index/brand.go | 1 - 2 files changed, 2 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 92f0651b0..223d555a7 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -122,7 +122,6 @@ func (self *Executor) RunPQL(database_name string, pql string) interface{} { if err != nil { spew.Dump(err) } - spew.Dump(final) final_result[query_list[i].Label] = final } diff --git a/index/brand.go b/index/brand.go index e014ba594..ac87cb140 100644 --- a/index/brand.go +++ b/index/brand.go @@ -175,7 +175,6 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) { func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair { self.rank_counter = 0 - println("RANK") self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP is := new(IntSet) return self.TopNCat(src_bitmap, n, is)