From f375ad9de5ced3e561fe948e743efcc6d420eb7a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 4 Mar 2014 17:13:57 -0600 Subject: [PATCH 1/3] WIP: lexer/parser refactor/enhancements. --- core/http.go | 30 +++- db/topology.go | 2 +- query/lexer.go | 221 +++++++++++++++++----------- query/lexer_test.go | 338 ++++++++++++++++++++++--------------------- query/parser.go | 242 ++++++++++++++++++++----------- query/parser_test.go | 79 ++++++++-- query/query.go | 23 ++- 7 files changed, 572 insertions(+), 363 deletions(-) diff --git a/core/http.go b/core/http.go index 41c4cd547..2cbacb6e6 100644 --- a/core/http.go +++ b/core/http.go @@ -166,15 +166,33 @@ func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) { return } encoder := json.NewEncoder(w) - //stats := service.GetStats() - // stats := "" m := &runtime.MemStats{} runtime.ReadMemStats(m) - stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc} - err := encoder.Encode(stats) + //self.Report(fmt.Sprintf("%s.goroutines", prefix), + // float64(runtime.NumGoroutine()), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.allocated", prefix), + // float64(memStats.Alloc), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.mallocs", prefix), + // float64(memStats.Mallocs), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.frees", prefix), + // float64(memStats.Frees), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.gc.total_pause", prefix), + // float64(memStats.PauseTotalNs)/nsInMs, now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.heap", prefix), + // float64(memStats.HeapAlloc), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.stack", prefix), + // float64(memStats.StackInuse), now, context, dimensions) + + //stats := map[string]interface{}{ + // "num_goroutines": runtime.NumGoroutine(), + // "memory_allocated": m.Sys, + // "memory_": m.Alloc + //} + + err := encoder.Encode(m) if err != nil { - log.Fatal("Error encoding stats") + panic("Error encoding stats") } } @@ -220,7 +238,7 @@ func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) processes := self.service.ProcessMap.GetMetadata() err := encoder.Encode(processes) if err != nil { - log.Fatal("Error encoding stats") + panic("Error encoding stats") } } diff --git a/db/topology.go b/db/topology.go index 267ef4124..bf776e045 100644 --- a/db/topology.go +++ b/db/topology.go @@ -312,7 +312,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment //d.mutex.Lock() //defer d.mutex.Unlock() frame, _ := d.getFrame(bitmap.FrameType) - log.Println(frame, slice) + //log.Println(frame, slice) fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Println(err) diff --git a/query/lexer.go b/query/lexer.go index d64be71c5..b6ff9af98 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -2,9 +2,8 @@ package query import ( "errors" - "log" + "fmt" "strings" - "unicode" "unicode/utf8" ) @@ -12,10 +11,18 @@ const ( TYPE_FUNC = iota TYPE_LP = iota TYPE_RP = iota + TYPE_LB = iota + TYPE_RB = iota + TYPE_VALUE = iota + TYPE_KEYWORD = iota + TYPE_EQUALS = iota + TYPE_COMMA = iota + TYPE_ERROR = iota + + // Below types deprecated TYPE_ID = iota TYPE_FRAME = iota TYPE_PROFILE = iota - TYPE_COMMA = iota TYPE_LIMIT = iota ) @@ -40,15 +47,28 @@ func (lexer *Lexer) emit(typ int) { lexer.start = lexer.pos } -func (lexer *Lexer) acceptUntil(chars string) error { +func (lexer *Lexer) acceptUntil(chars string, consume bool) (rune, error) { + start_pos := lexer.pos + for { - if strings.HasPrefix(lexer.text[lexer.pos:], chars) { - return nil + next := lexer.next() + + if next == rune(' ') { + lexer.ignore() } - // if we receive a reserved character that we are not expecting, throw a parse error - lexer.pos += 1 - if lexer.pos > len(lexer.text) { - return errors.New("Parse error, expecting " + string(chars)) + + if next == 0 { + lexer.pos = start_pos + return 0, errors.New("Not found") + } + ch := strings.IndexRune(chars, next) + if ch >= 0 { + if consume { + lexer.backup() + } else { + lexer.pos = start_pos + } + return []rune(chars)[ch], nil } } } @@ -59,15 +79,6 @@ func (lexer *Lexer) acceptRun(valid string) { lexer.backup() } -func (lexer *Lexer) acceptNumber() { - digits := "0123456789" - lexer.acceptRun(digits) -} -func (lexer *Lexer) acceptText() { - digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")" - lexer.acceptRun(digits) -} - // next returns the next rune in the input. func (lexer *Lexer) next() (runey rune) { if lexer.pos >= len(lexer.text) { @@ -104,10 +115,18 @@ func (lexer *Lexer) peek() rune { } } +func stateError(err error) func(lexer *Lexer) statefn { + return func(lexer *Lexer) statefn { + lexer.ch <- Token{err.Error(), TYPE_ERROR} + close(lexer.ch) + return nil + } +} + func stateFunc(lexer *Lexer) statefn { - err := lexer.acceptUntil("(") + _, err := lexer.acceptUntil("(", true) if err != nil { - log.Fatal(err) + return stateError(err) } lexer.emit(TYPE_FUNC) return stateLP @@ -123,34 +142,91 @@ func stateLP(lexer *Lexer) statefn { return stateArgs } +func stateLB(lexer *Lexer) statefn { + lexer.acceptUntil("[", true) + lexer.next() + lexer.emit(TYPE_LB) + for { + r, err := lexer.acceptUntil(",]", true) + if err != nil { + return stateError(errors.New("Unclosed bracket!")) + } + lexer.emit(TYPE_VALUE) + if r == ',' { + lexer.next() + lexer.emit(TYPE_COMMA) + } else { + lexer.next() + lexer.emit(TYPE_RB) + return stateArgs + } + } +} + func stateArgs(lexer *Lexer) statefn { - if unicode.IsNumber(lexer.peek()) { - return stateID - } else { + r, err := lexer.acceptUntil("(),=[", false) + if err != nil { + return stateError(err) + } + switch r { + case '(': return stateFunc + case ')': + return stateValue + case ',': + return stateValue + case '=': + return stateKeyword + case '[': + return stateLB + default: + return stateError(errors.New("Expecting arguments!")) } + return nil } -func stateID(lexer *Lexer) statefn { - lexer.acceptNumber() - lexer.emit(TYPE_ID) - // if next is comma - peeked := lexer.peek() - if peeked == rune(',') { - return stateFrameComma - } else if peeked == rune(')') { +func stateKeyword(lexer *Lexer) statefn { + _, err := lexer.acceptUntil("=", true) + if err != nil { + return stateError(err) + } + lexer.emit(TYPE_KEYWORD) + return stateEquals +} + +func stateEquals(lexer *Lexer) statefn { + e := lexer.next() + if e != '=' { + return stateError(errors.New("Expecting '='!")) + } + lexer.emit(TYPE_EQUALS) + return stateValue +} + +func stateValue(lexer *Lexer) statefn { + r, err := lexer.acceptUntil("(),[", false) + if err != nil { + return stateError(err) + } + switch r { + case '(': + return stateFunc + case ')': + lexer.acceptUntil(")", true) + if lexer.pos > lexer.start { + lexer.emit(TYPE_VALUE) + } return stateRP - } else { - return stateID + case ',': + lexer.acceptUntil(",", true) + lexer.emit(TYPE_VALUE) + return stateComma + case '[': + return stateLB + default: + return stateError(errors.New("Unexpected character!")) } -} - -func stateProfile(lexer *Lexer) statefn { - lexer.peek() - lexer.acceptNumber() - lexer.emit(TYPE_PROFILE) - lexer.peek() - return stateRP + return nil } func stateRP(lexer *Lexer) statefn { @@ -167,49 +243,10 @@ func stateRP(lexer *Lexer) statefn { } } -func stateFrameComma(lexer *Lexer) statefn { - lexer.pos += 1 - lexer.emit(TYPE_COMMA) - return stateFrameOrProfile -} - -func stateProfileComma(lexer *Lexer) statefn { - lexer.pos += 1 - lexer.emit(TYPE_COMMA) - return stateProfile -} - -func stateFrameOrProfile(lexer *Lexer) statefn { - if unicode.IsNumber(lexer.peek()) { - lexer.acceptNumber() - lexer.emit(TYPE_PROFILE) - lexer.peek() - } else { - lexer.acceptText() - lexer.emit(TYPE_FRAME) - if lexer.peek() == rune(',') { - return stateProfileComma - } - } - return stateRP -} - func stateRPComma(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_COMMA) - if unicode.IsNumber(lexer.peek()) { - return stateLimit - } else { - return stateArgs - } -} - -func stateLimit(lexer *Lexer) statefn { - lexer.acceptNumber() - lexer.emit(TYPE_LIMIT) - // if next is comma - lexer.peek() - return stateRP + return stateValue } func stateComma(lexer *Lexer) statefn { @@ -223,8 +260,17 @@ func stateEOF(lexer *Lexer) statefn { return nil } -func (lexer *Lexer) Lex() []Token { - tokens := make([]Token, 0) +func (lexer *Lexer) Lex() (tokens []Token, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("query: %v", r) + } + } + }() + tokens = make([]Token, 0) state := stateFunc go func() { for { @@ -235,12 +281,15 @@ func (lexer *Lexer) Lex() []Token { } }() for t := range lexer.ch { + if t.Type == TYPE_ERROR { + err = errors.New(t.Text) + } tokens = append(tokens, t) } - return tokens + return tokens, err } -func Lex(input string) []Token { +func Lex(input string) ([]Token, error) { lexer := Lexer{input, 0, 0, 0, TYPE_FUNC, make(chan Token)} return lexer.Lex() } diff --git a/query/lexer_test.go b/query/lexer_test.go index 3549dc790..47aa172d6 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -7,180 +7,184 @@ import ( func TestLexer(t *testing.T) { Convey("Basic lexical analysis", t, func() { + var tokens []Token + var err error - tokens := Lex("get(10)") + tokens, err = Lex("get(10)") + So(err, ShouldBeNil) So(len(tokens), ShouldEqual, 4) - So(tokens[0].Text, ShouldEqual, "get") - So(tokens[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens[1].Text, ShouldEqual, "(") - So(tokens[1].Type, ShouldEqual, TYPE_LP) - So(tokens[2].Text, ShouldEqual, "10") - So(tokens[2].Type, ShouldEqual, TYPE_ID) - So(tokens[3].Text, ShouldEqual, ")") - So(tokens[3].Type, ShouldEqual, TYPE_RP) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens2 := Lex("intersect(get(10), get(11), get(12))") - So(len(tokens2), ShouldEqual, 17) - So(tokens2[0].Text, ShouldEqual, "intersect") - So(tokens2[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[1].Text, ShouldEqual, "(") - So(tokens2[1].Type, ShouldEqual, TYPE_LP) - So(tokens2[2].Text, ShouldEqual, "get") - So(tokens2[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[3].Text, ShouldEqual, "(") - So(tokens2[3].Type, ShouldEqual, TYPE_LP) - So(tokens2[4].Text, ShouldEqual, "10") - So(tokens2[4].Type, ShouldEqual, TYPE_ID) - So(tokens2[5].Text, ShouldEqual, ")") - So(tokens2[5].Type, ShouldEqual, TYPE_RP) - So(tokens2[6].Text, ShouldEqual, ",") - So(tokens2[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[7].Text, ShouldEqual, "get") - So(tokens2[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[8].Text, ShouldEqual, "(") - So(tokens2[8].Type, ShouldEqual, TYPE_LP) - So(tokens2[9].Text, ShouldEqual, "11") - So(tokens2[9].Type, ShouldEqual, TYPE_ID) - So(tokens2[10].Text, ShouldEqual, ")") - So(tokens2[10].Type, ShouldEqual, TYPE_RP) - So(tokens2[11].Text, ShouldEqual, ",") - So(tokens2[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[12].Text, ShouldEqual, "get") - So(tokens2[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[13].Text, ShouldEqual, "(") - So(tokens2[13].Type, ShouldEqual, TYPE_LP) - So(tokens2[14].Text, ShouldEqual, "12") - So(tokens2[14].Type, ShouldEqual, TYPE_ID) - So(tokens2[15].Text, ShouldEqual, ")") - So(tokens2[15].Type, ShouldEqual, TYPE_RP) - So(tokens2[16].Text, ShouldEqual, ")") - So(tokens2[16].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("get(id=10)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"id", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens3 := Lex("intersect(get(10), get(11), concat(get(12),get(14)))") - So(len(tokens3), ShouldEqual, 25) - So(tokens3[0].Text, ShouldEqual, "intersect") - So(tokens3[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[1].Text, ShouldEqual, "(") - So(tokens3[1].Type, ShouldEqual, TYPE_LP) - So(tokens3[2].Text, ShouldEqual, "get") - So(tokens3[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[3].Text, ShouldEqual, "(") - So(tokens3[3].Type, ShouldEqual, TYPE_LP) - So(tokens3[4].Text, ShouldEqual, "10") - So(tokens3[4].Type, ShouldEqual, TYPE_ID) - So(tokens3[5].Text, ShouldEqual, ")") - So(tokens3[5].Type, ShouldEqual, TYPE_RP) - So(tokens3[6].Text, ShouldEqual, ",") - So(tokens3[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[7].Text, ShouldEqual, "get") - So(tokens3[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[8].Text, ShouldEqual, "(") - So(tokens3[8].Type, ShouldEqual, TYPE_LP) - So(tokens3[9].Text, ShouldEqual, "11") - So(tokens3[9].Type, ShouldEqual, TYPE_ID) - So(tokens3[10].Text, ShouldEqual, ")") - So(tokens3[10].Type, ShouldEqual, TYPE_RP) - So(tokens3[11].Text, ShouldEqual, ",") - So(tokens3[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[12].Text, ShouldEqual, "concat") - So(tokens3[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[13].Text, ShouldEqual, "(") - So(tokens3[13].Type, ShouldEqual, TYPE_LP) - So(tokens3[14].Text, ShouldEqual, "get") - So(tokens3[14].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[15].Text, ShouldEqual, "(") - So(tokens3[15].Type, ShouldEqual, TYPE_LP) - So(tokens3[16].Text, ShouldEqual, "12") - So(tokens3[16].Type, ShouldEqual, TYPE_ID) - So(tokens3[17].Text, ShouldEqual, ")") - So(tokens3[17].Type, ShouldEqual, TYPE_RP) - So(tokens3[18].Text, ShouldEqual, ",") - So(tokens3[18].Type, ShouldEqual, TYPE_COMMA) + tokens, err = Lex("get(id=10, frame=brand)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"id", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"10", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"frame", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"brand", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens4 := Lex("concat(get(1, brand),get(2))") - So(len(tokens4), ShouldEqual, 14) - So(tokens4[0].Text, ShouldEqual, "concat") - So(tokens4[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[1].Text, ShouldEqual, "(") - So(tokens4[1].Type, ShouldEqual, TYPE_LP) - So(tokens4[2].Text, ShouldEqual, "get") - So(tokens4[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[3].Text, ShouldEqual, "(") - So(tokens4[3].Type, ShouldEqual, TYPE_LP) - So(tokens4[4].Text, ShouldEqual, "1") - So(tokens4[4].Type, ShouldEqual, TYPE_ID) - So(tokens4[5].Text, ShouldEqual, ",") - So(tokens4[5].Type, ShouldEqual, TYPE_COMMA) - So(tokens4[6].Text, ShouldEqual, "brand") - So(tokens4[6].Type, ShouldEqual, TYPE_FRAME) - So(tokens4[7].Text, ShouldEqual, ")") - So(tokens4[7].Type, ShouldEqual, TYPE_RP) - So(tokens4[8].Text, ShouldEqual, ",") - So(tokens4[8].Type, ShouldEqual, TYPE_COMMA) - So(tokens4[9].Text, ShouldEqual, "get") - So(tokens4[9].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[10].Text, ShouldEqual, "(") - So(tokens4[10].Type, ShouldEqual, TYPE_LP) - So(tokens4[11].Text, ShouldEqual, "2") - So(tokens4[11].Type, ShouldEqual, TYPE_ID) - So(tokens4[12].Text, ShouldEqual, ")") - So(tokens4[12].Type, ShouldEqual, TYPE_RP) - So(tokens4[13].Text, ShouldEqual, ")") - So(tokens4[13].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("union(get(10))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"union", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens5 := Lex("set(1, 987)") - So(len(tokens5), ShouldEqual, 6) - So(tokens5[0].Text, ShouldEqual, "set") - So(tokens5[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens5[1].Text, ShouldEqual, "(") - So(tokens5[1].Type, ShouldEqual, TYPE_LP) - So(tokens5[2].Text, ShouldEqual, "1") - So(tokens5[2].Type, ShouldEqual, TYPE_ID) - So(tokens5[3].Text, ShouldEqual, ",") - So(tokens5[3].Type, ShouldEqual, TYPE_COMMA) - So(tokens5[4].Text, ShouldEqual, "987") - So(tokens5[4].Type, ShouldEqual, TYPE_PROFILE) - So(tokens5[5].Text, ShouldEqual, ")") - So(tokens5[5].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("intersect(get(10), get(11), get(12))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"intersect", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"11", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"12", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens6 := Lex("set(1, general, 987)") - So(len(tokens6), ShouldEqual, 8) - So(tokens6[0].Text, ShouldEqual, "set") - So(tokens6[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens6[1].Text, ShouldEqual, "(") - So(tokens6[1].Type, ShouldEqual, TYPE_LP) - So(tokens6[2].Text, ShouldEqual, "1") - So(tokens6[2].Type, ShouldEqual, TYPE_ID) - So(tokens6[3].Text, ShouldEqual, ",") - So(tokens6[3].Type, ShouldEqual, TYPE_COMMA) - So(tokens6[4].Text, ShouldEqual, "general") - So(tokens6[4].Type, ShouldEqual, TYPE_FRAME) - So(tokens6[5].Text, ShouldEqual, ",") - So(tokens6[5].Type, ShouldEqual, TYPE_COMMA) - So(tokens6[6].Text, ShouldEqual, "987") - So(tokens6[6].Type, ShouldEqual, TYPE_PROFILE) - So(tokens6[7].Text, ShouldEqual, ")") - So(tokens6[7].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("intersect(get(10), get(11), concat(get(12),get(14)))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"intersect", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"11", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"concat", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"12", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"14", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens7 := Lex("top-n(get(10), 8)") - So(len(tokens6), ShouldEqual, 8) - So(tokens7[0].Text, ShouldEqual, "top-n") - So(tokens7[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens7[1].Text, ShouldEqual, "(") - So(tokens7[1].Type, ShouldEqual, TYPE_LP) - So(tokens7[2].Text, ShouldEqual, "get") - So(tokens7[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens7[3].Text, ShouldEqual, "(") - So(tokens7[3].Type, ShouldEqual, TYPE_LP) - So(tokens7[4].Text, ShouldEqual, "10") - So(tokens7[4].Type, ShouldEqual, TYPE_ID) - So(tokens7[5].Text, ShouldEqual, ")") - So(tokens7[5].Type, ShouldEqual, TYPE_RP) - So(tokens7[6].Text, ShouldEqual, ",") - So(tokens7[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens7[7].Text, ShouldEqual, "8") - So(tokens7[7].Type, ShouldEqual, TYPE_LIMIT) - So(tokens7[8].Text, ShouldEqual, ")") - So(tokens7[8].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("concat(get(1, brand),get(2))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"concat", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"brand", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"2", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("set(1, 987)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"set", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"987", TYPE_VALUE}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("set(1, general, 987)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"set", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"general", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"987", TYPE_VALUE}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("top-n(get(10), 8)") + So(tokens, ShouldResemble, []Token{ + Token{"top-n", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"10", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{",", TYPE_COMMA}, + Token{"8", TYPE_VALUE}, + Token{")", TYPE_RP}, + }) + + tokens, err = Lex("top-n(get(10, general), [1,2,3])") + So(tokens, ShouldResemble, []Token{ + Token{"top-n", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"10", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"general", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{",", TYPE_COMMA}, + Token{"[", TYPE_LB}, + Token{"1", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"2", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"3", TYPE_VALUE}, + Token{"]", TYPE_RB}, + Token{")", TYPE_RP}, + }) }) } diff --git a/query/parser.go b/query/parser.go index 08ceea04a..98a6411db 100644 --- a/query/parser.go +++ b/query/parser.go @@ -2,109 +2,179 @@ package query import ( "errors" - "pilosa/db" + "fmt" "strconv" - "tux21b.org/v1/gocql/uuid" + + "github.com/davecgh/go-spew/spew" ) var InvalidQueryError = errors.New("Invalid query format.") -type QueryParser struct{} - -func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64, int) { - // BITMAP - if tokens[0].Type == TYPE_ID { - // TODO: look for frame type in the tokens list - b, err := strconv.ParseUint(tokens[0].Text, 10, 64) - bitmap_id := uint64(b) - if err != nil { - panic(err) - } - // if the next 2 tokens are comma-frame, then we have a frame, else set to a default - frame_type := "general" - profile_id := uint64(0) - - if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE { - frame_type = tokens[2].Text - profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64) - if err != nil { - panic(err) - } - } else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME { - frame_type = tokens[2].Text - } else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE { - profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64) - if err != nil { - panic(err) - } - } - bm := db.Bitmap{bitmap_id, frame_type} - return []QueryInput{&bm}, uint64(profile_id), 0 - } - - // LIST OF QUERIES - n := int(10) // default LIMIT to 10 - qi := []QueryInput{} - open_parens := -1 // >=0 means i'm inside the search for end paren - start := 0 - for i := 0; i < len(tokens); i++ { - if tokens[i].Type == TYPE_FUNC && open_parens == -1 { - start = i - } else if tokens[i].Type == TYPE_LP { - open_parens++ - } else if tokens[i].Type == TYPE_RP { - if open_parens == 0 { - q, err := qp.walk(tokens[start : i+1]) - if err != nil { - panic(err) - } - qi = append(qi, q) - open_parens = -1 - } else { - open_parens-- - } - } else if tokens[i].Type == TYPE_LIMIT { - x, _ := strconv.ParseInt(tokens[i].Text, 10, 32) - n = int(x) - } - } - return qi, 0, n +type QueryParser struct { + tokens []Token + pos int } -func (qp *QueryParser) walk(tokens []Token) (*Query, error) { - - if tokens[0].Type != TYPE_FUNC { - panic("BAD!") - } - if tokens[1].Type != TYPE_LP { - panic("BAD!") +func (self *QueryParser) next() *Token { + self.pos += 1 + if self.pos > len(self.tokens) { + return nil } + return &self.tokens[self.pos-1] +} + +func (self *QueryParser) peek() *Token { + token := self.next() + self.backup() + return token +} + +func (self *QueryParser) backup() { + self.pos -= 1 +} + +func (self *QueryParser) Parse() (query *Query, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("query: %v", r) + } + } + }() + var token *Token - q := new(Query) id := uuid.RandomUUID() - q.Id = &id - q.Operation = tokens[0].Text + query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})} - // scan from open to close paren - open_parens := 0 - for i := 2; i < len(tokens); i++ { - // 1 must be "(" - if tokens[i].Type == TYPE_LP { - open_parens++ - } else if tokens[i].Type == TYPE_RP { - if open_parens == 0 { - if i == len(tokens)-1 { - q.Inputs, q.ProfileId, q.N = qp.walkInputs(tokens[2:i]) + token = self.next() + if token.Type != TYPE_FUNC { + return nil, fmt.Errorf("Expected function, found token %v.", token) + } + + query.Operation = token.Text + + token = self.next() + if token.Type != TYPE_LP { + return nil, fmt.Errorf("Expected '(', found token %v.", token) + } + +ArgLoop: + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed parentheses!") + } + + switch token.Type { + case TYPE_FUNC: + self.backup() + subquery, err := self.Parse() + if err != nil { + return nil, err + } + query.Subqueries = append(query.Subqueries, *subquery) + case TYPE_VALUE: + switch query.Operation { + case "get": + switch len(query.Args) { + case 0: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["id"] = i + case 1: + query.Args["frame"] = token.Text + default: + return nil, fmt.Errorf("Unexpected argument! (%v)", token) + } + case "set": + switch len(query.Args) { + case 0: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["id"] = i + case 1: + query.Args["frame"] = token.Text + case 2: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["profile_id"] = i + default: + return nil, fmt.Errorf("Unexpected argument! (%v)", token) + } + + default: + spew.Dump("UNPROCESSED VALUE", token) + } + continue + case TYPE_COMMA: + continue + case TYPE_RP: + break ArgLoop + case TYPE_KEYWORD: + var value interface{} + keyword := token.Text + token = self.next() + if token == nil || token.Type != TYPE_EQUALS { + return nil, fmt.Errorf("Expecting equals sign!") + } + token = self.next() + if token == nil || token.Type != TYPE_VALUE { + return nil, fmt.Errorf("Expecting value!") + } + if keyword == "id" { + value, err = strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) } } else { - open_parens-- + value = token.Text } + query.Args[keyword] = value + case TYPE_LB: + query.Args["ids"] = make([]uint64, 0) + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed list!") + } + switch token.Type { + case TYPE_COMMA: + break + case TYPE_VALUE: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["ids"] = append(query.Args["ids"].([]uint64), i) + case TYPE_RB: + continue ArgLoop + default: + return nil, fmt.Errorf("Unexpected token! (%v)", token) + } + } + default: + spew.Dump("unexpected", token) + panic(token) } } - return q, nil + + if query.Operation == "get" && query.Args["frame"] == nil { + query.Args["frame"] = "general" + } + + return query, nil } -func (qp *QueryParser) Parse(tokens []Token) (*Query, error) { - return qp.walk(tokens) +func Parse(tokens []Token) (*Query, error) { + parser := QueryParser{tokens, 0} + return parser.Parse() } diff --git a/query/parser_test.go b/query/parser_test.go index bed194513..b6e9cec76 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -2,20 +2,79 @@ package query import ( "testing" - - "github.com/davecgh/go-spew/spew" . "github.com/smartystreets/goconvey/convey" ) func TestQueryParser(t *testing.T) { - Convey("Basic query parse", t, func() { + Convey("Basic parse - get()", t, func() { + tokens, err := Lex("get(10)") + So(err, ShouldBeNil) - tokens := Lex("union(get(10,general), get(11,brand), get(12))") - qp := QueryParser{} - q, err := qp.Parse(tokens) - if err != nil { - panic(err) - } - spew.Dump(q) + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + }) + Convey("Basic parse - set()", t, func() { + tokens, err := Lex("set(10, general, 20)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "set") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "profile_id": uint64(20)}) + }) + Convey("Basic nested query parse", t, func() { + tokens, err := Lex("union(get(10,general), get(11,brand), get(12))") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "union") + So(len(query.Subqueries), ShouldEqual, 3) + + So(query.Subqueries[0].Operation, ShouldEqual, "get") + So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + So(query.Subqueries[1].Operation, ShouldEqual, "get") + So(query.Subqueries[1].Args, ShouldResemble, map[string]interface{}{"id": uint64(11), "frame": "brand"}) + So(query.Subqueries[2].Operation, ShouldEqual, "get") + So(query.Subqueries[2].Args, ShouldResemble, map[string]interface{}{"id": uint64(12), "frame": "general"}) + }) + Convey("Keyword args", t, func() { + tokens, err := Lex("get(id=10)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + }) + Convey("Keyword args - multiple", t, func() { + tokens, err := Lex("get(id=10, frame=brands)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "brands"}) + }) + Convey("Lists", t, func() { + tokens, err := Lex("top-n(get(10, general), [1,2,3])") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "top-n") + So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}}) + + So(len(query.Subqueries), ShouldEqual, 1) + So(query.Subqueries[0].Operation, ShouldEqual, "get") + So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) }) } diff --git a/query/query.go b/query/query.go index f1a7cc642..4483288ae 100644 --- a/query/query.go +++ b/query/query.go @@ -22,9 +22,13 @@ type PqlListItem struct { } type Query struct { - Id *uuid.UUID - Operation string - Inputs []QueryInput //"strconv" + Id *uuid.UUID + Operation string + Args map[string]interface{} + Subqueries []Query + + // deprecated: + Inputs []QueryInput //"strconv" // Represents a parsed query. Inputs can be Query or Bitmap objects // Maybe Bitmap and Query objects should have different fields to avoid using interface{} ProfileId uint64 // used only for set() queries @@ -32,18 +36,23 @@ type Query struct { } func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan { - tokens := Lex(pql) + tokens, err := Lex(pql) + if err != nil { + panic(err) + } return QueryPlanForTokens(database, tokens, destination) } func QueryForPQL(pql string) *Query { - tokens := Lex(pql) + tokens, err := Lex(pql) + if err != nil { + panic(err) + } return QueryForTokens(tokens) } func QueryForTokens(tokens []Token) *Query { - query_parser := QueryParser{} - query, err := query_parser.Parse(tokens) + query, err := Parse(tokens) if err != nil { panic(err) } From c858da3fa7a9468b400e26a050ad796f99d0430c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 18 Mar 2014 18:40:40 -0500 Subject: [PATCH 2/3] Query planner - make work with new args from query parser, remove "N" variable --- core/query.go | 2 +- query/lexer.go | 4 +- query/parser.go | 11 +++ query/parser_test.go | 4 +- query/planner.go | 109 ++++++++-------------- query/planner_test.go | 207 +++++++++++++++++++++++++++++++++--------- query/query.go | 9 +- 7 files changed, 217 insertions(+), 129 deletions(-) diff --git a/core/query.go b/core/query.go index 55a461778..31b979005 100644 --- a/core/query.go +++ b/core/query.go @@ -45,7 +45,7 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) { case []byte: bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val) } - topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N, categoryleaves) + topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, categoryleaves) if err != nil { spew.Dump(err) } diff --git a/query/lexer.go b/query/lexer.go index b6ff9af98..6c2ac09c4 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -43,7 +43,9 @@ type Lexer struct { } func (lexer *Lexer) emit(typ int) { - lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ} + if lexer.start < lexer.pos { + lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ} + } lexer.start = lexer.pos } diff --git a/query/parser.go b/query/parser.go index 98a6411db..c49ecce8e 100644 --- a/query/parser.go +++ b/query/parser.go @@ -111,6 +111,12 @@ ArgLoop: return nil, fmt.Errorf("Unexpected argument! (%v)", token) } + case "top-n": + i, err := strconv.Atoi(token.Text) + if err != nil { + return nil, fmt.Errorf("Expecting integer! (%v)", err) + } + query.Args["n"] = i default: spew.Dump("UNPROCESSED VALUE", token) } @@ -135,6 +141,11 @@ ArgLoop: if err != nil { return nil, fmt.Errorf("Expecting integer id! (%v)", err) } + } else if keyword == "n" { + value, err = strconv.Atoi(token.Text) + if err != nil { + return nil, fmt.Errorf("Expecting integer! (%v)", err) + } } else { value = token.Text } diff --git a/query/parser_test.go b/query/parser_test.go index b6e9cec76..4d136a37f 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -64,14 +64,14 @@ func TestQueryParser(t *testing.T) { So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "brands"}) }) Convey("Lists", t, func() { - tokens, err := Lex("top-n(get(10, general), [1,2,3])") + tokens, err := Lex("top-n(get(10, general), [1,2,3], 50)") So(err, ShouldBeNil) query, err := Parse(tokens) So(err, ShouldBeNil) So(query.Operation, ShouldEqual, "top-n") - So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}}) + So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}, "n": 50}) So(len(query.Subqueries), ShouldEqual, 1) So(query.Subqueries[0].Operation, ShouldEqual, "get") diff --git a/query/planner.go b/query/planner.go index f6f11523e..392f567ac 100644 --- a/query/planner.go +++ b/query/planner.go @@ -2,26 +2,12 @@ package query import ( "encoding/gob" - "fmt" "math/rand" "pilosa/db" "tux21b.org/v1/gocql/uuid" ) -// A single step in the query plan. -type QueryStep struct { - id *uuid.UUID - operation string - inputs []QueryInput - location *db.Location - destination *db.Location -} - -func (q QueryStep) StringHOLD() string { - return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.destination) -} - type PortableQueryStep interface { GetId() *uuid.UUID GetLocation() *db.Location @@ -105,6 +91,7 @@ type TopNQueryResult struct { type TopNQueryTree struct { subquery QueryTree location *db.Location + N int } // Uses consistent hashing function to select node containing data for GET operation @@ -191,6 +178,7 @@ type CatQueryResult struct { type CatQueryTree struct { subqueries []QueryTree location *db.Location + N int } // Uses consistent hashing function to select node containing data for GET operation @@ -292,44 +280,31 @@ type QueryPlan []interface{} type QueryPlanner struct { Database *db.Database + Query *Query } type QueryTree interface { getLocation(d *db.Database) *db.Location } -// QueryTree for UNION and INTERSECT queries -type CompositeQueryTree struct { - operation string - subqueries []QueryTree - location *db.Location -} - -// Randomly select location from subqueries (so subqueries roll up into composite queries while minimizing inter-node data traffic) -func (qt *CompositeQueryTree) getLocation(d *db.Database) *db.Location { - if qt.location == nil { - subqueryLength := len(qt.subqueries) - if subqueryLength > 0 { - locationIndex := rand.Intn(subqueryLength) - subquery := qt.subqueries[locationIndex] - qt.location = subquery.getLocation(d) - } - } - return qt.location -} - // Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { var tree QueryTree // handle SET operation regardless of the slice if query.Operation == "set" { - tree = &SetQueryTree{query.Inputs[0].(*db.Bitmap), query.ProfileId} + + tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, query.Args["profile_id"].(uint64)} return tree } // handle the remaining operations, taking slice into consideration if slice == -1 { - tree = &CatQueryTree{} + var n int + n_, ok := query.Args["n"] + if ok { + n = n_.(int) + } + tree = &CatQueryTree{N: n} numSlices, err := qp.Database.NumSlices() if err != nil { panic(err) @@ -342,57 +317,48 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { } } else { if query.Operation == "get" { - tree = &GetQueryTree{query.Inputs[0].(*db.Bitmap), slice} + tree = &GetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, slice} return tree } else if query.Operation == "count" { - subquery := qp.buildTree(query.Inputs[0].(*Query), slice) + subquery := qp.buildTree(&query.Subqueries[0], slice) tree = &CountQueryTree{subquery: subquery} } else if query.Operation == "top-n" { - subquery := qp.buildTree(query.Inputs[0].(*Query), slice) - tree = &TopNQueryTree{subquery: subquery} + var n int + n_, ok := query.Args["n"] + if ok { + n = n_.(int) + } + subquery := qp.buildTree(&query.Subqueries[0], slice) + tree = &TopNQueryTree{subquery: subquery, N: n} } else if query.Operation == "union" { - subqueries := make([]QueryTree, len(query.Inputs)) - for i, input := range query.Inputs { - subqueries[i] = qp.buildTree(input.(*Query), slice) + subqueries := make([]QueryTree, len(query.Subqueries)) + for i, query := range query.Subqueries { + subqueries[i] = qp.buildTree(&query, slice) } tree = &UnionQueryTree{subqueries: subqueries} } else if query.Operation == "intersect" { - subqueries := make([]QueryTree, len(query.Inputs)) - for i, input := range query.Inputs { - subqueries[i] = qp.buildTree(input.(*Query), slice) + subqueries := make([]QueryTree, len(query.Subqueries)) + for i, query := range query.Subqueries { + subqueries[i] = qp.buildTree(&query, slice) } tree = &IntersectQueryTree{subqueries: subqueries} } else { - subqueries := make([]QueryTree, len(query.Inputs)) - for i, input := range query.Inputs { - subqueries[i] = qp.buildTree(input.(*Query), slice) - } - tree = &CompositeQueryTree{operation: query.Operation, subqueries: subqueries} + panic("invalid operation") } } return tree } // Produces flattened QueryPlan from QueryTree input -func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location, n int) *QueryPlan { +func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location) *QueryPlan { plan := QueryPlan{} - if composite, ok := qt.(*CompositeQueryTree); ok { - inputs := make([]QueryInput, len(composite.subqueries)) - step := QueryStep{id, composite.operation, inputs, composite.getLocation(qp.Database), location} - for index, subq := range composite.subqueries { - sub_id := uuid.RandomUUID() - step.inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database), n) - plan = append(plan, *subq_steps...) - } - plan = append(plan, step) - } else if cat, ok := qt.(*CatQueryTree); ok { + if cat, ok := qt.(*CatQueryTree); ok { inputs := make([]*uuid.UUID, len(cat.subqueries)) - step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, n} + step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, cat.N} for index, subq := range cat.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database), n) + subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database)) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -402,7 +368,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati for index, subq := range union.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database), n) + subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database)) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -412,7 +378,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati for index, subq := range intersect.subqueries { sub_id := uuid.RandomUUID() step.Inputs[index] = &sub_id - subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database), n) + subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database)) plan = append(plan, *subq_steps...) } plan = append(plan, step) @@ -427,13 +393,13 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati } else if cnt, ok := qt.(*CountQueryTree); ok { sub_id := uuid.RandomUUID() step := &CountQueryStep{&BaseQueryStep{id, "count", cnt.getLocation(qp.Database), location}, &sub_id} - subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database), n) + subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database)) plan = append(plan, *subq_steps...) plan = append(plan, step) } else if topn, ok := qt.(*TopNQueryTree); ok { sub_id := uuid.RandomUUID() - step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, n} - subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database), n) + step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.N} + subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database)) plan = append(plan, *subq_steps...) plan = append(plan, step) } @@ -443,6 +409,5 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati // Transforms Query into QueryTree and flattens to QueryPlan object func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan { queryTree := qp.buildTree(query, -1) - //return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value - return qp.flatten(queryTree, query.Id, destination, query.N) + return qp.flatten(queryTree, query.Id, destination) } diff --git a/query/planner_test.go b/query/planner_test.go index a2cda6627..ec448d243 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -4,67 +4,184 @@ import ( "pilosa/db" "pilosa/util" "testing" - - "github.com/davecgh/go-spew/spew" . "github.com/smartystreets/goconvey/convey" "tux21b.org/v1/gocql/uuid" ) +func basic_database() (*db.Database, *db.Fragment) { + // create an empty database + cluster := db.NewCluster() + database := cluster.GetOrCreateDatabase("main") + frame := database.GetOrCreateFrame("general") + + slice1 := database.GetOrCreateSlice(0) + fragment_id1 := util.Id() + fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1) + process_id1 := uuid.RandomUUID() + process1 := db.NewProcess(&process_id1) + process1.SetHost("----192.1.1.0----") + fragment1.SetProcess(process1) + + slice2 := database.GetOrCreateSlice(1) + fragment_id2 := util.Id() + fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2) + process_id2 := uuid.RandomUUID() + process2 := db.NewProcess(&process_id2) + process2.SetHost("----192.1.1.1----") + fragment2.SetProcess(process2) + return database, fragment1 +} + func TestQueryPlanner(t *testing.T) { - Convey("Basic query plan", t, func() { + Convey("Union query plan", t, func() { id1 := uuid.RandomUUID() - bm1 := db.Bitmap{10, "general"} - inputs1 := []QueryInput{&bm1} - query1 := Query{&id1, "get", inputs1, 0, 0} + query1 := Query{Id: &id1, Operation: "get", Args: map[string]interface{}{"id": uint64(10), "frame": "general"}} id2 := uuid.RandomUUID() - bm2 := db.Bitmap{20, "general"} - inputs2 := []QueryInput{&bm2} - query2 := Query{&id2, "get", inputs2, 0, 0} + query2 := Query{Id: &id2, Operation: "get", Args: map[string]interface{}{"id": uint64(20), "frame": "general"}} id3 := uuid.RandomUUID() - inputs := []QueryInput{&query1, &query2} - query := Query{&id3, "union", inputs, 0, 0} - /* + query := Query{Id: &id3, Operation: "union", Subqueries: []Query{query1, query2}} - bm1 := db.Bitmap{10, "general"} - inputs1 := []QueryInput{&bm1} - query1 := Query{"get", inputs1} - query := query1 - */ + database, fragment1 := basic_database() - // create an empty database - cluster := db.NewCluster() - database := cluster.GetOrCreateDatabase("main") - frame := database.GetOrCreateFrame("general") - - slice1 := database.GetOrCreateSlice(0) - fragment_id1 := util.Id() - fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1) - process_id1 := uuid.RandomUUID() - process1 := db.NewProcess(&process_id1) - process1.SetHost("----192.1.1.0----") - fragment1.SetProcess(process1) - - slice2 := database.GetOrCreateSlice(1) - fragment_id2 := util.Id() - fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2) - process_id2 := uuid.RandomUUID() - process2 := db.NewProcess(&process_id2) - process2.SetHost("----192.1.1.1----") - fragment2.SetProcess(process2) - - qplanner := QueryPlanner{Database: database} + qplanner := QueryPlanner{Database: database, Query: &query} destination := fragment1.GetLocation() id := uuid.RandomUUID() - qp := qplanner.Plan(&query, &id, destination) + qp := *qplanner.Plan(&query, &id, destination) - for i, qs := range *qp { - //spew.Dump(i, qs, qs.inputs) - spew.Dump(i, qs) - spew.Dump("**************************************************************") - } + So(len(qp), ShouldEqual, 7) + So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0) + So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0) + So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"}) + So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union") + So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[0].(GetQueryStep).Id, + qp[1].(GetQueryStep).Id, + }) + So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1) + So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1) + So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"}) + So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union") + So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[3].(GetQueryStep).Id, + qp[4].(GetQueryStep).Id, + }) + So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat") + So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[2].(UnionQueryStep).Id, + qp[5].(UnionQueryStep).Id, + }) + }) + Convey("Get query plan - including parsing", t, func() { + + query := QueryForPQL("get(10,general)") + + database, fragment1 := basic_database() + + qplanner := QueryPlanner{Database: database, Query: query} + destination := fragment1.GetLocation() + + id := uuid.RandomUUID() + qp := *qplanner.Plan(query, &id, destination) + + So(len(qp), ShouldEqual, 3) + So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0) + So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[1].(GetQueryStep).Slice, ShouldEqual, 1) + So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[2].(CatQueryStep).Operation, ShouldEqual, "cat") + So(qp[2].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[0].(GetQueryStep).Id, + qp[1].(GetQueryStep).Id, + }) + }) + Convey("Union query plan - including parsing", t, func() { + + query := QueryForPQL("union(get(10, general), get(20, general))") + + database, fragment1 := basic_database() + + qplanner := QueryPlanner{Database: database, Query: query} + destination := fragment1.GetLocation() + + id := uuid.RandomUUID() + qp := *qplanner.Plan(query, &id, destination) + + So(len(qp), ShouldEqual, 7) + So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0) + So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0) + So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"}) + So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union") + So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[0].(GetQueryStep).Id, + qp[1].(GetQueryStep).Id, + }) + So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1) + So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get") + So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1) + So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"}) + So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union") + So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[3].(GetQueryStep).Id, + qp[4].(GetQueryStep).Id, + }) + So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat") + So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{ + qp[2].(UnionQueryStep).Id, + qp[5].(UnionQueryStep).Id, + }) + }) + Convey("Set query plan - including parsing", t, func() { + query := QueryForPQL("set(10, general, 100)") + + database, fragment1 := basic_database() + + qplanner := QueryPlanner{Database: database, Query: query} + destination := fragment1.GetLocation() + + id := uuid.RandomUUID() + qp := *qplanner.Plan(query, &id, destination) + So(len(qp), ShouldEqual, 1) + So(qp[0].(SetQueryStep).Operation, ShouldEqual, "set") + So(qp[0].(SetQueryStep).ProfileId, ShouldEqual, 100) + So(*(qp[0].(SetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + }) + Convey("Top-n query plan - including parsing", t, func() { + query := QueryForPQL("top-n(get(10, general), [1,2,3], 50)") + + database, fragment1 := basic_database() + + qplanner := QueryPlanner{Database: database, Query: query} + destination := fragment1.GetLocation() + + id := uuid.RandomUUID() + qp := *qplanner.Plan(query, &id, destination) + So(len(qp), ShouldEqual, 5) + So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get") + So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[1].(*TopNQueryStep).Operation, ShouldEqual, "top-n") + So(qp[1].(*TopNQueryStep).Input, ShouldEqual, qp[0].(GetQueryStep).Id) + So(qp[1].(*TopNQueryStep).N, ShouldEqual, 50) + So(qp[2].(GetQueryStep).Operation, ShouldEqual, "get") + So(*(qp[2].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"}) + So(qp[3].(*TopNQueryStep).Operation, ShouldEqual, "top-n") + So(qp[3].(*TopNQueryStep).Input, ShouldEqual, qp[2].(GetQueryStep).Id) + So(qp[3].(*TopNQueryStep).N, ShouldEqual, 50) }) } diff --git a/query/query.go b/query/query.go index 4483288ae..bbe738241 100644 --- a/query/query.go +++ b/query/query.go @@ -26,13 +26,6 @@ type Query struct { Operation string Args map[string]interface{} Subqueries []Query - - // deprecated: - Inputs []QueryInput //"strconv" - // Represents a parsed query. Inputs can be Query or Bitmap objects - // Maybe Bitmap and Query objects should have different fields to avoid using interface{} - ProfileId uint64 // used only for set() queries - N int // TODO: I think we should make this a generic map for any attributes related to the query } func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan { @@ -65,7 +58,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L } func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) *QueryPlan { - query_planner := QueryPlanner{Database: database} + query_planner := QueryPlanner{Database: database, Query: query} id := uuid.RandomUUID() query_plan := query_planner.Plan(query, &id, destination) return query_plan From c744183cec2a9f1ba6ef50a63796d81972871af1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Mar 2014 16:42:36 -0500 Subject: [PATCH 3/3] Update nexter - add statsd and logfile support. --- commands/pilosa-nexter/nexter.go | 28 ++++++++++++++++++++++++---- deps.json | 6 ++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index f2821c40e..b0bd84efd 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -6,9 +6,11 @@ import ( "fmt" "log" "net/http" + "os" "strconv" "strings" + "github.com/cactus/go-statsd-client/statsd" "github.com/coreos/go-etcd/etcd" ) @@ -26,11 +28,15 @@ func (self *stringslice) Set(value string) error { var port uint var blocksize uint64 var etcd_nodes stringslice +var stats string +var logfile string func init() { flag.UintVar(&port, "port", 9000, "Port to run HTTP server on") flag.Uint64Var(&blocksize, "blocksize", 64, "Block size") flag.Var(&etcd_nodes, "etcd", "Etcd server") + flag.String(stats, "statsd", "Statsd server") + flag.String(logfile, "log", "Log file name") flag.Parse() } @@ -48,6 +54,7 @@ type DelReq struct { type Nexter struct { reqchan chan Req done chan bool + stats *statsd.Client } func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { @@ -85,6 +92,7 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { if err != nil { continue } + self.stats.Gauge(path, int64(end), 1.0) for c := start; c < end; c += 1 { ch <- c } @@ -132,15 +140,27 @@ func (self *Nexter) Stop() { self.done <- true } -func NewNexter() *Nexter { - nexter := &Nexter{make(chan Req), make(chan bool)} +func NewNexter() (*Nexter, error) { + stats, err := statsd.New(stats, "") + if err != nil { + return nil, err + } + nexter := &Nexter{make(chan Req), make(chan bool), stats} go nexter.loop() - return nexter + return nexter, nil } func main() { + logf, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Println("Error opening file: %v", err) + } + log.SetOutput(logf) log.Println("Starting Nexter...") - nexter := NewNexter() + nexter, err := NewNexter() + if err != nil { + log.Fatal(err) + } http.HandleFunc("/nexter/", func(w http.ResponseWriter, r *http.Request) { url := r.URL.Path splits := strings.Split(url, "/") diff --git a/deps.json b/deps.json index db940d04e..a0cf0e7bc 100644 --- a/deps.json +++ b/deps.json @@ -73,5 +73,11 @@ "repo": "github.com/gorilla/websocket", "version": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c", "type": "git" + }, + "github.com/cactus/go-statsd-client/statsd": { + "repo": "github.com/cactus/go-statsd-client", + "version": "912f30c35e9cdf51f50bae24f071e227cca152fb", + "type": "git" } + }