From fc75d3730086ebaceb34c7448a500d6b9e049c23 Mon Sep 17 00:00:00 2001 From: travisturner Date: Mon, 30 Dec 2013 13:31:40 -0600 Subject: [PATCH] add support for Set() queries to lexer/parser/planner --- core/http.go | 2 -- query/lexer.go | 63 +++++++++++++++++++++++++++++-------------- query/lexer_test.go | 36 +++++++++++++++++++++++++ query/parser.go | 17 ++++++++---- query/planner.go | 32 ++++++++++++++++++++-- query/planner_test.go | 6 ++--- query/query.go | 8 +++++- 7 files changed, 131 insertions(+), 33 deletions(-) diff --git a/core/http.go b/core/http.go index d316014bf..d8171b0ef 100644 --- a/core/http.go +++ b/core/http.go @@ -63,11 +63,9 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) { http.Error(w, "Error reading POST data", http.StatusBadRequest) return } - cluster := self.service.Cluster database := cluster.GetOrCreateDatabase("main") pql := string(body) - query.Execute(database, pql) } diff --git a/query/lexer.go b/query/lexer.go index 64cad2b25..5297bfd2f 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -9,12 +9,13 @@ import ( ) const ( - TYPE_FUNC = iota - TYPE_LP = iota - TYPE_RP = iota - TYPE_ID = iota - TYPE_FRAME = iota - TYPE_COMMA = iota + TYPE_FUNC = iota + TYPE_LP = iota + TYPE_RP = iota + TYPE_ID = iota + TYPE_FRAME = iota + TYPE_PROFILE = iota + TYPE_COMMA = iota ) type Token struct { @@ -57,6 +58,15 @@ func (lexer *Lexer) acceptRun(valid string) { lexer.backup() } +func (lexer *Lexer) acceptNumber() { + digits := "0123456789" + lexer.acceptRun(digits) +} +func (lexer *Lexer) acceptText() { + digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")" + lexer.acceptRun(digits) +} + // next returns the next rune in the input. func (lexer *Lexer) next() (runey rune) { if lexer.pos >= len(lexer.text) { @@ -121,11 +131,8 @@ func stateArgs(lexer *Lexer) statefn { } func stateID(lexer *Lexer) statefn { - - digits := "0123456789" - lexer.acceptRun(digits) + lexer.acceptNumber() lexer.emit(TYPE_ID) - // if next is comma peeked := lexer.peek() if peeked == rune(',') { @@ -137,6 +144,14 @@ func stateID(lexer *Lexer) statefn { } } +func stateProfile(lexer *Lexer) statefn { + lexer.peek() + lexer.acceptNumber() + lexer.emit(TYPE_PROFILE) + lexer.peek() + return stateRP +} + func stateRP(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_RP) @@ -154,19 +169,27 @@ func stateRP(lexer *Lexer) statefn { func stateFrameComma(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_COMMA) - return stateFrame + return stateFrameOrProfile } -func stateFrame(lexer *Lexer) statefn { - lexer.peek() - digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")" - lexer.acceptRun(digits) - lexer.emit(TYPE_FRAME) - peeked := lexer.peek() - if peeked == rune(')') { - return stateRP +func stateProfileComma(lexer *Lexer) statefn { + lexer.pos += 1 + lexer.emit(TYPE_COMMA) + return stateProfile +} + +func stateFrameOrProfile(lexer *Lexer) statefn { + if unicode.IsNumber(lexer.peek()) { + lexer.acceptNumber() + lexer.emit(TYPE_PROFILE) + lexer.peek() + } else { + lexer.acceptText() + lexer.emit(TYPE_FRAME) + if lexer.peek() == rune(',') { + return stateProfileComma + } } - // should never get here return stateRP } diff --git a/query/lexer_test.go b/query/lexer_test.go index 21f454ca5..dc52d93cf 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -7,6 +7,7 @@ import ( func TestLexer(t *testing.T) { Convey("Basic lexical analysis", t, func() { + tokens := Lex("get(10)") So(len(tokens), ShouldEqual, 4) So(tokens[0].Text, ShouldEqual, "get") @@ -126,5 +127,40 @@ func TestLexer(t *testing.T) { So(tokens4[12].Type, ShouldEqual, TYPE_RP) So(tokens4[13].Text, ShouldEqual, ")") So(tokens4[13].Type, ShouldEqual, TYPE_RP) + + tokens5 := Lex("set(1, 987)") + So(len(tokens5), ShouldEqual, 6) + So(tokens5[0].Text, ShouldEqual, "set") + So(tokens5[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens5[1].Text, ShouldEqual, "(") + So(tokens5[1].Type, ShouldEqual, TYPE_LP) + So(tokens5[2].Text, ShouldEqual, "1") + So(tokens5[2].Type, ShouldEqual, TYPE_ID) + So(tokens5[3].Text, ShouldEqual, ",") + So(tokens5[3].Type, ShouldEqual, TYPE_COMMA) + So(tokens5[4].Text, ShouldEqual, "987") + So(tokens5[4].Type, ShouldEqual, TYPE_PROFILE) + So(tokens5[5].Text, ShouldEqual, ")") + So(tokens5[5].Type, ShouldEqual, TYPE_RP) + + tokens6 := Lex("set(1, general, 987)") + So(len(tokens6), ShouldEqual, 8) + So(tokens6[0].Text, ShouldEqual, "set") + So(tokens6[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens6[1].Text, ShouldEqual, "(") + So(tokens6[1].Type, ShouldEqual, TYPE_LP) + So(tokens6[2].Text, ShouldEqual, "1") + So(tokens6[2].Type, ShouldEqual, TYPE_ID) + So(tokens6[3].Text, ShouldEqual, ",") + So(tokens6[3].Type, ShouldEqual, TYPE_COMMA) + So(tokens6[4].Text, ShouldEqual, "general") + So(tokens6[4].Type, ShouldEqual, TYPE_FRAME) + So(tokens6[5].Text, ShouldEqual, ",") + So(tokens6[5].Type, ShouldEqual, TYPE_COMMA) + So(tokens6[6].Text, ShouldEqual, "987") + So(tokens6[6].Type, ShouldEqual, TYPE_PROFILE) + So(tokens6[7].Text, ShouldEqual, ")") + So(tokens6[7].Type, ShouldEqual, TYPE_RP) + }) } diff --git a/query/parser.go b/query/parser.go index 7a4686c7d..9a3ade22d 100644 --- a/query/parser.go +++ b/query/parser.go @@ -10,7 +10,7 @@ var InvalidQueryError = errors.New("Invalid query format.") type QueryParser struct{} -func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput { +func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, int) { // BITMAP if tokens[0].Type == TYPE_ID { // TODO: look for frame type in the tokens list @@ -20,11 +20,18 @@ func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput { } // if the next 2 tokens are comma-frame, then we have a frame, else set to a default frame_type := "general" - if len(tokens) == 3 && tokens[2].Type == TYPE_FRAME { + if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME { frame_type = tokens[2].Text } bm := db.Bitmap{bitmap_id, frame_type} - return []QueryInput{&bm} + profile_id := 0 + if len(tokens) > 4 && tokens[4].Type == TYPE_PROFILE { + profile_id, err = strconv.Atoi(tokens[4].Text) + if err != nil { + panic(err) + } + } + return []QueryInput{&bm}, profile_id } // LIST OF QUERIES @@ -49,7 +56,7 @@ func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput { } } } - return qi + return qi, 0 } func (qp *QueryParser) walk(tokens []Token) (*Query, error) { @@ -73,7 +80,7 @@ func (qp *QueryParser) walk(tokens []Token) (*Query, error) { } else if tokens[i].Type == TYPE_RP { if open_parens == 0 { if i == len(tokens)-1 { - q.Inputs = qp.walkInputs(tokens[2:i]) + q.Inputs, q.Profile_id = qp.walkInputs(tokens[2:i]) } } else { open_parens-- diff --git a/query/planner.go b/query/planner.go index 6694bcb11..a73e71221 100644 --- a/query/planner.go +++ b/query/planner.go @@ -17,7 +17,7 @@ type QueryStep struct { return_process *db.Process } -func (q QueryStep) String() string { +func (q QueryStep) StringHOLD() string { return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.return_process) } @@ -43,7 +43,7 @@ type CompositeQueryTree struct { func (qt *CompositeQueryTree) getLocation(d *db.Database) *db.Process { if qt.location == nil { subqueryLength := len(qt.subqueries) - if subqueryLength > 1 { + if subqueryLength > 0 { locationIndex := rand.Intn(subqueryLength) subquery := qt.subqueries[locationIndex] qt.location = subquery.getLocation(d) @@ -58,6 +58,12 @@ type GetQueryTree struct { slice int } +// QueryTree for SET queries +type SetQueryTree struct { + bitmap *db.Bitmap + profile_id int +} + // Uses consistent hashing function to select node containing data for GET operation func (qt *GetQueryTree) getLocation(d *db.Database) *db.Process { slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create) @@ -68,9 +74,27 @@ func (qt *GetQueryTree) getLocation(d *db.Database) *db.Process { return fragment.GetProcess() } +// Uses consistent hashing function to select node containing data for GET operation +func (qt *SetQueryTree) getLocation(d *db.Database) *db.Process { + slice, err := d.GetSliceForProfile(qt.profile_id) + fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap) + if err != nil { + panic(err) + } + return fragment.GetProcess() +} + // Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { var tree QueryTree + + // handle SET operation regardless of the slice + if query.Operation == "set" { + tree = &SetQueryTree{query.Inputs[0].(*db.Bitmap), query.Profile_id} + return tree + } + + // handle the remaining operations, taking slice into consideration if slice == -1 { tree = &CompositeQueryTree{operation: "cat"} numSlices, err := qp.Database.NumSlices() @@ -115,6 +139,10 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Proces step := QueryStep{*id, "get", []QueryInput{get.bitmap, get.slice}, get.getLocation(qp.Database), location} plan := QueryPlan{step} return &plan + } else if set, ok := qt.(*SetQueryTree); ok { + step := QueryStep{*id, "set", []QueryInput{set.bitmap, set.profile_id}, set.getLocation(qp.Database), location} + plan := QueryPlan{step} + return &plan } return &plan } diff --git a/query/planner_test.go b/query/planner_test.go index b97deecc1..e173ae0ed 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -15,14 +15,14 @@ func TestQueryPlanner(t *testing.T) { bm1 := db.Bitmap{10, "general"} inputs1 := []QueryInput{&bm1} - query1 := Query{"get", inputs1} + query1 := Query{"get", inputs1, 0} bm2 := db.Bitmap{20, "general"} inputs2 := []QueryInput{&bm2} - query2 := Query{"get", inputs2} + query2 := Query{"get", inputs2, 0} inputs := []QueryInput{&query1, &query2} - query := Query{"union", inputs} + query := Query{"union", inputs, 0} /* bm1 := db.Bitmap{10, "general"} diff --git a/query/query.go b/query/query.go index aa501c250..6c5d3c102 100644 --- a/query/query.go +++ b/query/query.go @@ -14,7 +14,7 @@ type Query struct { Inputs []QueryInput //"strconv" // Represents a parsed query. Inputs can be Query or Bitmap objects // Maybe Bitmap and Query objects should have different fields to avoid using interface{} - + Profile_id int } func Execute(database *db.Database, pql string) { @@ -22,6 +22,7 @@ func Execute(database *db.Database, pql string) { //spew.Dump(pql) tokens := Lex(pql) //spew.Dump(tokens) + query_parser := QueryParser{} query, err := query_parser.Parse(tokens) if err != nil { @@ -29,6 +30,11 @@ func Execute(database *db.Database, pql string) { } //spew.Dump(query) + // switch on different query types: + //if query.Operation == "set" { + //spew.Dump("SET!!") + //} + query_planner := QueryPlanner{Database: database} id, _ := uuid.NewV4() destination := db.Process{}