diff --git a/core/http.go b/core/http.go index 41c4cd547..2cbacb6e6 100644 --- a/core/http.go +++ b/core/http.go @@ -166,15 +166,33 @@ func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) { return } encoder := json.NewEncoder(w) - //stats := service.GetStats() - // stats := "" m := &runtime.MemStats{} runtime.ReadMemStats(m) - stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc} - err := encoder.Encode(stats) + //self.Report(fmt.Sprintf("%s.goroutines", prefix), + // float64(runtime.NumGoroutine()), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.allocated", prefix), + // float64(memStats.Alloc), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.mallocs", prefix), + // float64(memStats.Mallocs), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.frees", prefix), + // float64(memStats.Frees), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.gc.total_pause", prefix), + // float64(memStats.PauseTotalNs)/nsInMs, now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.heap", prefix), + // float64(memStats.HeapAlloc), now, context, dimensions) + //self.Report(fmt.Sprintf("%s.memory.stack", prefix), + // float64(memStats.StackInuse), now, context, dimensions) + + //stats := map[string]interface{}{ + // "num_goroutines": runtime.NumGoroutine(), + // "memory_allocated": m.Sys, + // "memory_": m.Alloc + //} + + err := encoder.Encode(m) if err != nil { - log.Fatal("Error encoding stats") + panic("Error encoding stats") } } @@ -220,7 +238,7 @@ func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) processes := self.service.ProcessMap.GetMetadata() err := encoder.Encode(processes) if err != nil { - log.Fatal("Error encoding stats") + panic("Error encoding stats") } } diff --git a/db/topology.go b/db/topology.go index 267ef4124..bf776e045 100644 --- a/db/topology.go +++ b/db/topology.go @@ -312,7 +312,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment //d.mutex.Lock() //defer d.mutex.Unlock() frame, _ := d.getFrame(bitmap.FrameType) - log.Println(frame, slice) + //log.Println(frame, slice) fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Println(err) diff --git a/query/lexer.go b/query/lexer.go index d64be71c5..b6ff9af98 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -2,9 +2,8 @@ package query import ( "errors" - "log" + "fmt" "strings" - "unicode" "unicode/utf8" ) @@ -12,10 +11,18 @@ const ( TYPE_FUNC = iota TYPE_LP = iota TYPE_RP = iota + TYPE_LB = iota + TYPE_RB = iota + TYPE_VALUE = iota + TYPE_KEYWORD = iota + TYPE_EQUALS = iota + TYPE_COMMA = iota + TYPE_ERROR = iota + + // Below types deprecated TYPE_ID = iota TYPE_FRAME = iota TYPE_PROFILE = iota - TYPE_COMMA = iota TYPE_LIMIT = iota ) @@ -40,15 +47,28 @@ func (lexer *Lexer) emit(typ int) { lexer.start = lexer.pos } -func (lexer *Lexer) acceptUntil(chars string) error { +func (lexer *Lexer) acceptUntil(chars string, consume bool) (rune, error) { + start_pos := lexer.pos + for { - if strings.HasPrefix(lexer.text[lexer.pos:], chars) { - return nil + next := lexer.next() + + if next == rune(' ') { + lexer.ignore() } - // if we receive a reserved character that we are not expecting, throw a parse error - lexer.pos += 1 - if lexer.pos > len(lexer.text) { - return errors.New("Parse error, expecting " + string(chars)) + + if next == 0 { + lexer.pos = start_pos + return 0, errors.New("Not found") + } + ch := strings.IndexRune(chars, next) + if ch >= 0 { + if consume { + lexer.backup() + } else { + lexer.pos = start_pos + } + return []rune(chars)[ch], nil } } } @@ -59,15 +79,6 @@ func (lexer *Lexer) acceptRun(valid string) { lexer.backup() } -func (lexer *Lexer) acceptNumber() { - digits := "0123456789" - lexer.acceptRun(digits) -} -func (lexer *Lexer) acceptText() { - digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")" - lexer.acceptRun(digits) -} - // next returns the next rune in the input. func (lexer *Lexer) next() (runey rune) { if lexer.pos >= len(lexer.text) { @@ -104,10 +115,18 @@ func (lexer *Lexer) peek() rune { } } +func stateError(err error) func(lexer *Lexer) statefn { + return func(lexer *Lexer) statefn { + lexer.ch <- Token{err.Error(), TYPE_ERROR} + close(lexer.ch) + return nil + } +} + func stateFunc(lexer *Lexer) statefn { - err := lexer.acceptUntil("(") + _, err := lexer.acceptUntil("(", true) if err != nil { - log.Fatal(err) + return stateError(err) } lexer.emit(TYPE_FUNC) return stateLP @@ -123,34 +142,91 @@ func stateLP(lexer *Lexer) statefn { return stateArgs } +func stateLB(lexer *Lexer) statefn { + lexer.acceptUntil("[", true) + lexer.next() + lexer.emit(TYPE_LB) + for { + r, err := lexer.acceptUntil(",]", true) + if err != nil { + return stateError(errors.New("Unclosed bracket!")) + } + lexer.emit(TYPE_VALUE) + if r == ',' { + lexer.next() + lexer.emit(TYPE_COMMA) + } else { + lexer.next() + lexer.emit(TYPE_RB) + return stateArgs + } + } +} + func stateArgs(lexer *Lexer) statefn { - if unicode.IsNumber(lexer.peek()) { - return stateID - } else { + r, err := lexer.acceptUntil("(),=[", false) + if err != nil { + return stateError(err) + } + switch r { + case '(': return stateFunc + case ')': + return stateValue + case ',': + return stateValue + case '=': + return stateKeyword + case '[': + return stateLB + default: + return stateError(errors.New("Expecting arguments!")) } + return nil } -func stateID(lexer *Lexer) statefn { - lexer.acceptNumber() - lexer.emit(TYPE_ID) - // if next is comma - peeked := lexer.peek() - if peeked == rune(',') { - return stateFrameComma - } else if peeked == rune(')') { +func stateKeyword(lexer *Lexer) statefn { + _, err := lexer.acceptUntil("=", true) + if err != nil { + return stateError(err) + } + lexer.emit(TYPE_KEYWORD) + return stateEquals +} + +func stateEquals(lexer *Lexer) statefn { + e := lexer.next() + if e != '=' { + return stateError(errors.New("Expecting '='!")) + } + lexer.emit(TYPE_EQUALS) + return stateValue +} + +func stateValue(lexer *Lexer) statefn { + r, err := lexer.acceptUntil("(),[", false) + if err != nil { + return stateError(err) + } + switch r { + case '(': + return stateFunc + case ')': + lexer.acceptUntil(")", true) + if lexer.pos > lexer.start { + lexer.emit(TYPE_VALUE) + } return stateRP - } else { - return stateID + case ',': + lexer.acceptUntil(",", true) + lexer.emit(TYPE_VALUE) + return stateComma + case '[': + return stateLB + default: + return stateError(errors.New("Unexpected character!")) } -} - -func stateProfile(lexer *Lexer) statefn { - lexer.peek() - lexer.acceptNumber() - lexer.emit(TYPE_PROFILE) - lexer.peek() - return stateRP + return nil } func stateRP(lexer *Lexer) statefn { @@ -167,49 +243,10 @@ func stateRP(lexer *Lexer) statefn { } } -func stateFrameComma(lexer *Lexer) statefn { - lexer.pos += 1 - lexer.emit(TYPE_COMMA) - return stateFrameOrProfile -} - -func stateProfileComma(lexer *Lexer) statefn { - lexer.pos += 1 - lexer.emit(TYPE_COMMA) - return stateProfile -} - -func stateFrameOrProfile(lexer *Lexer) statefn { - if unicode.IsNumber(lexer.peek()) { - lexer.acceptNumber() - lexer.emit(TYPE_PROFILE) - lexer.peek() - } else { - lexer.acceptText() - lexer.emit(TYPE_FRAME) - if lexer.peek() == rune(',') { - return stateProfileComma - } - } - return stateRP -} - func stateRPComma(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_COMMA) - if unicode.IsNumber(lexer.peek()) { - return stateLimit - } else { - return stateArgs - } -} - -func stateLimit(lexer *Lexer) statefn { - lexer.acceptNumber() - lexer.emit(TYPE_LIMIT) - // if next is comma - lexer.peek() - return stateRP + return stateValue } func stateComma(lexer *Lexer) statefn { @@ -223,8 +260,17 @@ func stateEOF(lexer *Lexer) statefn { return nil } -func (lexer *Lexer) Lex() []Token { - tokens := make([]Token, 0) +func (lexer *Lexer) Lex() (tokens []Token, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("query: %v", r) + } + } + }() + tokens = make([]Token, 0) state := stateFunc go func() { for { @@ -235,12 +281,15 @@ func (lexer *Lexer) Lex() []Token { } }() for t := range lexer.ch { + if t.Type == TYPE_ERROR { + err = errors.New(t.Text) + } tokens = append(tokens, t) } - return tokens + return tokens, err } -func Lex(input string) []Token { +func Lex(input string) ([]Token, error) { lexer := Lexer{input, 0, 0, 0, TYPE_FUNC, make(chan Token)} return lexer.Lex() } diff --git a/query/lexer_test.go b/query/lexer_test.go index 3549dc790..47aa172d6 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -7,180 +7,184 @@ import ( func TestLexer(t *testing.T) { Convey("Basic lexical analysis", t, func() { + var tokens []Token + var err error - tokens := Lex("get(10)") + tokens, err = Lex("get(10)") + So(err, ShouldBeNil) So(len(tokens), ShouldEqual, 4) - So(tokens[0].Text, ShouldEqual, "get") - So(tokens[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens[1].Text, ShouldEqual, "(") - So(tokens[1].Type, ShouldEqual, TYPE_LP) - So(tokens[2].Text, ShouldEqual, "10") - So(tokens[2].Type, ShouldEqual, TYPE_ID) - So(tokens[3].Text, ShouldEqual, ")") - So(tokens[3].Type, ShouldEqual, TYPE_RP) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens2 := Lex("intersect(get(10), get(11), get(12))") - So(len(tokens2), ShouldEqual, 17) - So(tokens2[0].Text, ShouldEqual, "intersect") - So(tokens2[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[1].Text, ShouldEqual, "(") - So(tokens2[1].Type, ShouldEqual, TYPE_LP) - So(tokens2[2].Text, ShouldEqual, "get") - So(tokens2[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[3].Text, ShouldEqual, "(") - So(tokens2[3].Type, ShouldEqual, TYPE_LP) - So(tokens2[4].Text, ShouldEqual, "10") - So(tokens2[4].Type, ShouldEqual, TYPE_ID) - So(tokens2[5].Text, ShouldEqual, ")") - So(tokens2[5].Type, ShouldEqual, TYPE_RP) - So(tokens2[6].Text, ShouldEqual, ",") - So(tokens2[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[7].Text, ShouldEqual, "get") - So(tokens2[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[8].Text, ShouldEqual, "(") - So(tokens2[8].Type, ShouldEqual, TYPE_LP) - So(tokens2[9].Text, ShouldEqual, "11") - So(tokens2[9].Type, ShouldEqual, TYPE_ID) - So(tokens2[10].Text, ShouldEqual, ")") - So(tokens2[10].Type, ShouldEqual, TYPE_RP) - So(tokens2[11].Text, ShouldEqual, ",") - So(tokens2[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[12].Text, ShouldEqual, "get") - So(tokens2[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[13].Text, ShouldEqual, "(") - So(tokens2[13].Type, ShouldEqual, TYPE_LP) - So(tokens2[14].Text, ShouldEqual, "12") - So(tokens2[14].Type, ShouldEqual, TYPE_ID) - So(tokens2[15].Text, ShouldEqual, ")") - So(tokens2[15].Type, ShouldEqual, TYPE_RP) - So(tokens2[16].Text, ShouldEqual, ")") - So(tokens2[16].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("get(id=10)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"id", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens3 := Lex("intersect(get(10), get(11), concat(get(12),get(14)))") - So(len(tokens3), ShouldEqual, 25) - So(tokens3[0].Text, ShouldEqual, "intersect") - So(tokens3[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[1].Text, ShouldEqual, "(") - So(tokens3[1].Type, ShouldEqual, TYPE_LP) - So(tokens3[2].Text, ShouldEqual, "get") - So(tokens3[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[3].Text, ShouldEqual, "(") - So(tokens3[3].Type, ShouldEqual, TYPE_LP) - So(tokens3[4].Text, ShouldEqual, "10") - So(tokens3[4].Type, ShouldEqual, TYPE_ID) - So(tokens3[5].Text, ShouldEqual, ")") - So(tokens3[5].Type, ShouldEqual, TYPE_RP) - So(tokens3[6].Text, ShouldEqual, ",") - So(tokens3[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[7].Text, ShouldEqual, "get") - So(tokens3[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[8].Text, ShouldEqual, "(") - So(tokens3[8].Type, ShouldEqual, TYPE_LP) - So(tokens3[9].Text, ShouldEqual, "11") - So(tokens3[9].Type, ShouldEqual, TYPE_ID) - So(tokens3[10].Text, ShouldEqual, ")") - So(tokens3[10].Type, ShouldEqual, TYPE_RP) - So(tokens3[11].Text, ShouldEqual, ",") - So(tokens3[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[12].Text, ShouldEqual, "concat") - So(tokens3[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[13].Text, ShouldEqual, "(") - So(tokens3[13].Type, ShouldEqual, TYPE_LP) - So(tokens3[14].Text, ShouldEqual, "get") - So(tokens3[14].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[15].Text, ShouldEqual, "(") - So(tokens3[15].Type, ShouldEqual, TYPE_LP) - So(tokens3[16].Text, ShouldEqual, "12") - So(tokens3[16].Type, ShouldEqual, TYPE_ID) - So(tokens3[17].Text, ShouldEqual, ")") - So(tokens3[17].Type, ShouldEqual, TYPE_RP) - So(tokens3[18].Text, ShouldEqual, ",") - So(tokens3[18].Type, ShouldEqual, TYPE_COMMA) + tokens, err = Lex("get(id=10, frame=brand)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"id", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"10", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"frame", TYPE_KEYWORD}, + {"=", TYPE_EQUALS}, + {"brand", TYPE_VALUE}, + {")", TYPE_RP}, + }) - tokens4 := Lex("concat(get(1, brand),get(2))") - So(len(tokens4), ShouldEqual, 14) - So(tokens4[0].Text, ShouldEqual, "concat") - So(tokens4[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[1].Text, ShouldEqual, "(") - So(tokens4[1].Type, ShouldEqual, TYPE_LP) - So(tokens4[2].Text, ShouldEqual, "get") - So(tokens4[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[3].Text, ShouldEqual, "(") - So(tokens4[3].Type, ShouldEqual, TYPE_LP) - So(tokens4[4].Text, ShouldEqual, "1") - So(tokens4[4].Type, ShouldEqual, TYPE_ID) - So(tokens4[5].Text, ShouldEqual, ",") - So(tokens4[5].Type, ShouldEqual, TYPE_COMMA) - So(tokens4[6].Text, ShouldEqual, "brand") - So(tokens4[6].Type, ShouldEqual, TYPE_FRAME) - So(tokens4[7].Text, ShouldEqual, ")") - So(tokens4[7].Type, ShouldEqual, TYPE_RP) - So(tokens4[8].Text, ShouldEqual, ",") - So(tokens4[8].Type, ShouldEqual, TYPE_COMMA) - So(tokens4[9].Text, ShouldEqual, "get") - So(tokens4[9].Type, ShouldEqual, TYPE_FUNC) - So(tokens4[10].Text, ShouldEqual, "(") - So(tokens4[10].Type, ShouldEqual, TYPE_LP) - So(tokens4[11].Text, ShouldEqual, "2") - So(tokens4[11].Type, ShouldEqual, TYPE_ID) - So(tokens4[12].Text, ShouldEqual, ")") - So(tokens4[12].Type, ShouldEqual, TYPE_RP) - So(tokens4[13].Text, ShouldEqual, ")") - So(tokens4[13].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("union(get(10))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"union", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens5 := Lex("set(1, 987)") - So(len(tokens5), ShouldEqual, 6) - So(tokens5[0].Text, ShouldEqual, "set") - So(tokens5[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens5[1].Text, ShouldEqual, "(") - So(tokens5[1].Type, ShouldEqual, TYPE_LP) - So(tokens5[2].Text, ShouldEqual, "1") - So(tokens5[2].Type, ShouldEqual, TYPE_ID) - So(tokens5[3].Text, ShouldEqual, ",") - So(tokens5[3].Type, ShouldEqual, TYPE_COMMA) - So(tokens5[4].Text, ShouldEqual, "987") - So(tokens5[4].Type, ShouldEqual, TYPE_PROFILE) - So(tokens5[5].Text, ShouldEqual, ")") - So(tokens5[5].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("intersect(get(10), get(11), get(12))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"intersect", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"11", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"12", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens6 := Lex("set(1, general, 987)") - So(len(tokens6), ShouldEqual, 8) - So(tokens6[0].Text, ShouldEqual, "set") - So(tokens6[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens6[1].Text, ShouldEqual, "(") - So(tokens6[1].Type, ShouldEqual, TYPE_LP) - So(tokens6[2].Text, ShouldEqual, "1") - So(tokens6[2].Type, ShouldEqual, TYPE_ID) - So(tokens6[3].Text, ShouldEqual, ",") - So(tokens6[3].Type, ShouldEqual, TYPE_COMMA) - So(tokens6[4].Text, ShouldEqual, "general") - So(tokens6[4].Type, ShouldEqual, TYPE_FRAME) - So(tokens6[5].Text, ShouldEqual, ",") - So(tokens6[5].Type, ShouldEqual, TYPE_COMMA) - So(tokens6[6].Text, ShouldEqual, "987") - So(tokens6[6].Type, ShouldEqual, TYPE_PROFILE) - So(tokens6[7].Text, ShouldEqual, ")") - So(tokens6[7].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("intersect(get(10), get(11), concat(get(12),get(14)))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"intersect", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"10", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"11", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"concat", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"12", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"14", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) - tokens7 := Lex("top-n(get(10), 8)") - So(len(tokens6), ShouldEqual, 8) - So(tokens7[0].Text, ShouldEqual, "top-n") - So(tokens7[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens7[1].Text, ShouldEqual, "(") - So(tokens7[1].Type, ShouldEqual, TYPE_LP) - So(tokens7[2].Text, ShouldEqual, "get") - So(tokens7[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens7[3].Text, ShouldEqual, "(") - So(tokens7[3].Type, ShouldEqual, TYPE_LP) - So(tokens7[4].Text, ShouldEqual, "10") - So(tokens7[4].Type, ShouldEqual, TYPE_ID) - So(tokens7[5].Text, ShouldEqual, ")") - So(tokens7[5].Type, ShouldEqual, TYPE_RP) - So(tokens7[6].Text, ShouldEqual, ",") - So(tokens7[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens7[7].Text, ShouldEqual, "8") - So(tokens7[7].Type, ShouldEqual, TYPE_LIMIT) - So(tokens7[8].Text, ShouldEqual, ")") - So(tokens7[8].Type, ShouldEqual, TYPE_RP) + tokens, err = Lex("concat(get(1, brand),get(2))") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"concat", TYPE_FUNC}, + {"(", TYPE_LP}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"brand", TYPE_VALUE}, + {")", TYPE_RP}, + {",", TYPE_COMMA}, + {"get", TYPE_FUNC}, + {"(", TYPE_LP}, + {"2", TYPE_VALUE}, + {")", TYPE_RP}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("set(1, 987)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"set", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"987", TYPE_VALUE}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("set(1, general, 987)") + So(err, ShouldBeNil) + So(tokens, ShouldResemble, []Token{ + {"set", TYPE_FUNC}, + {"(", TYPE_LP}, + {"1", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"general", TYPE_VALUE}, + {",", TYPE_COMMA}, + {"987", TYPE_VALUE}, + {")", TYPE_RP}, + }) + + tokens, err = Lex("top-n(get(10), 8)") + So(tokens, ShouldResemble, []Token{ + Token{"top-n", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"10", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{",", TYPE_COMMA}, + Token{"8", TYPE_VALUE}, + Token{")", TYPE_RP}, + }) + + tokens, err = Lex("top-n(get(10, general), [1,2,3])") + So(tokens, ShouldResemble, []Token{ + Token{"top-n", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"10", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"general", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{",", TYPE_COMMA}, + Token{"[", TYPE_LB}, + Token{"1", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"2", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"3", TYPE_VALUE}, + Token{"]", TYPE_RB}, + Token{")", TYPE_RP}, + }) }) } diff --git a/query/parser.go b/query/parser.go index 08ceea04a..98a6411db 100644 --- a/query/parser.go +++ b/query/parser.go @@ -2,109 +2,179 @@ package query import ( "errors" - "pilosa/db" + "fmt" "strconv" - "tux21b.org/v1/gocql/uuid" + + "github.com/davecgh/go-spew/spew" ) var InvalidQueryError = errors.New("Invalid query format.") -type QueryParser struct{} - -func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64, int) { - // BITMAP - if tokens[0].Type == TYPE_ID { - // TODO: look for frame type in the tokens list - b, err := strconv.ParseUint(tokens[0].Text, 10, 64) - bitmap_id := uint64(b) - if err != nil { - panic(err) - } - // if the next 2 tokens are comma-frame, then we have a frame, else set to a default - frame_type := "general" - profile_id := uint64(0) - - if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE { - frame_type = tokens[2].Text - profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64) - if err != nil { - panic(err) - } - } else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME { - frame_type = tokens[2].Text - } else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE { - profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64) - if err != nil { - panic(err) - } - } - bm := db.Bitmap{bitmap_id, frame_type} - return []QueryInput{&bm}, uint64(profile_id), 0 - } - - // LIST OF QUERIES - n := int(10) // default LIMIT to 10 - qi := []QueryInput{} - open_parens := -1 // >=0 means i'm inside the search for end paren - start := 0 - for i := 0; i < len(tokens); i++ { - if tokens[i].Type == TYPE_FUNC && open_parens == -1 { - start = i - } else if tokens[i].Type == TYPE_LP { - open_parens++ - } else if tokens[i].Type == TYPE_RP { - if open_parens == 0 { - q, err := qp.walk(tokens[start : i+1]) - if err != nil { - panic(err) - } - qi = append(qi, q) - open_parens = -1 - } else { - open_parens-- - } - } else if tokens[i].Type == TYPE_LIMIT { - x, _ := strconv.ParseInt(tokens[i].Text, 10, 32) - n = int(x) - } - } - return qi, 0, n +type QueryParser struct { + tokens []Token + pos int } -func (qp *QueryParser) walk(tokens []Token) (*Query, error) { - - if tokens[0].Type != TYPE_FUNC { - panic("BAD!") - } - if tokens[1].Type != TYPE_LP { - panic("BAD!") +func (self *QueryParser) next() *Token { + self.pos += 1 + if self.pos > len(self.tokens) { + return nil } + return &self.tokens[self.pos-1] +} + +func (self *QueryParser) peek() *Token { + token := self.next() + self.backup() + return token +} + +func (self *QueryParser) backup() { + self.pos -= 1 +} + +func (self *QueryParser) Parse() (query *Query, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("query: %v", r) + } + } + }() + var token *Token - q := new(Query) id := uuid.RandomUUID() - q.Id = &id - q.Operation = tokens[0].Text + query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})} - // scan from open to close paren - open_parens := 0 - for i := 2; i < len(tokens); i++ { - // 1 must be "(" - if tokens[i].Type == TYPE_LP { - open_parens++ - } else if tokens[i].Type == TYPE_RP { - if open_parens == 0 { - if i == len(tokens)-1 { - q.Inputs, q.ProfileId, q.N = qp.walkInputs(tokens[2:i]) + token = self.next() + if token.Type != TYPE_FUNC { + return nil, fmt.Errorf("Expected function, found token %v.", token) + } + + query.Operation = token.Text + + token = self.next() + if token.Type != TYPE_LP { + return nil, fmt.Errorf("Expected '(', found token %v.", token) + } + +ArgLoop: + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed parentheses!") + } + + switch token.Type { + case TYPE_FUNC: + self.backup() + subquery, err := self.Parse() + if err != nil { + return nil, err + } + query.Subqueries = append(query.Subqueries, *subquery) + case TYPE_VALUE: + switch query.Operation { + case "get": + switch len(query.Args) { + case 0: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["id"] = i + case 1: + query.Args["frame"] = token.Text + default: + return nil, fmt.Errorf("Unexpected argument! (%v)", token) + } + case "set": + switch len(query.Args) { + case 0: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["id"] = i + case 1: + query.Args["frame"] = token.Text + case 2: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["profile_id"] = i + default: + return nil, fmt.Errorf("Unexpected argument! (%v)", token) + } + + default: + spew.Dump("UNPROCESSED VALUE", token) + } + continue + case TYPE_COMMA: + continue + case TYPE_RP: + break ArgLoop + case TYPE_KEYWORD: + var value interface{} + keyword := token.Text + token = self.next() + if token == nil || token.Type != TYPE_EQUALS { + return nil, fmt.Errorf("Expecting equals sign!") + } + token = self.next() + if token == nil || token.Type != TYPE_VALUE { + return nil, fmt.Errorf("Expecting value!") + } + if keyword == "id" { + value, err = strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) } } else { - open_parens-- + value = token.Text } + query.Args[keyword] = value + case TYPE_LB: + query.Args["ids"] = make([]uint64, 0) + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed list!") + } + switch token.Type { + case TYPE_COMMA: + break + case TYPE_VALUE: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["ids"] = append(query.Args["ids"].([]uint64), i) + case TYPE_RB: + continue ArgLoop + default: + return nil, fmt.Errorf("Unexpected token! (%v)", token) + } + } + default: + spew.Dump("unexpected", token) + panic(token) } } - return q, nil + + if query.Operation == "get" && query.Args["frame"] == nil { + query.Args["frame"] = "general" + } + + return query, nil } -func (qp *QueryParser) Parse(tokens []Token) (*Query, error) { - return qp.walk(tokens) +func Parse(tokens []Token) (*Query, error) { + parser := QueryParser{tokens, 0} + return parser.Parse() } diff --git a/query/parser_test.go b/query/parser_test.go index bed194513..b6e9cec76 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -2,20 +2,79 @@ package query import ( "testing" - - "github.com/davecgh/go-spew/spew" . "github.com/smartystreets/goconvey/convey" ) func TestQueryParser(t *testing.T) { - Convey("Basic query parse", t, func() { + Convey("Basic parse - get()", t, func() { + tokens, err := Lex("get(10)") + So(err, ShouldBeNil) - tokens := Lex("union(get(10,general), get(11,brand), get(12))") - qp := QueryParser{} - q, err := qp.Parse(tokens) - if err != nil { - panic(err) - } - spew.Dump(q) + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + }) + Convey("Basic parse - set()", t, func() { + tokens, err := Lex("set(10, general, 20)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "set") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "profile_id": uint64(20)}) + }) + Convey("Basic nested query parse", t, func() { + tokens, err := Lex("union(get(10,general), get(11,brand), get(12))") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "union") + So(len(query.Subqueries), ShouldEqual, 3) + + So(query.Subqueries[0].Operation, ShouldEqual, "get") + So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + So(query.Subqueries[1].Operation, ShouldEqual, "get") + So(query.Subqueries[1].Args, ShouldResemble, map[string]interface{}{"id": uint64(11), "frame": "brand"}) + So(query.Subqueries[2].Operation, ShouldEqual, "get") + So(query.Subqueries[2].Args, ShouldResemble, map[string]interface{}{"id": uint64(12), "frame": "general"}) + }) + Convey("Keyword args", t, func() { + tokens, err := Lex("get(id=10)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) + }) + Convey("Keyword args - multiple", t, func() { + tokens, err := Lex("get(id=10, frame=brands)") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "get") + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "brands"}) + }) + Convey("Lists", t, func() { + tokens, err := Lex("top-n(get(10, general), [1,2,3])") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + So(query.Operation, ShouldEqual, "top-n") + So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}}) + + So(len(query.Subqueries), ShouldEqual, 1) + So(query.Subqueries[0].Operation, ShouldEqual, "get") + So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) }) } diff --git a/query/query.go b/query/query.go index f1a7cc642..4483288ae 100644 --- a/query/query.go +++ b/query/query.go @@ -22,9 +22,13 @@ type PqlListItem struct { } type Query struct { - Id *uuid.UUID - Operation string - Inputs []QueryInput //"strconv" + Id *uuid.UUID + Operation string + Args map[string]interface{} + Subqueries []Query + + // deprecated: + Inputs []QueryInput //"strconv" // Represents a parsed query. Inputs can be Query or Bitmap objects // Maybe Bitmap and Query objects should have different fields to avoid using interface{} ProfileId uint64 // used only for set() queries @@ -32,18 +36,23 @@ type Query struct { } func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan { - tokens := Lex(pql) + tokens, err := Lex(pql) + if err != nil { + panic(err) + } return QueryPlanForTokens(database, tokens, destination) } func QueryForPQL(pql string) *Query { - tokens := Lex(pql) + tokens, err := Lex(pql) + if err != nil { + panic(err) + } return QueryForTokens(tokens) } func QueryForTokens(tokens []Token) *Query { - query_parser := QueryParser{} - query, err := query_parser.Parse(tokens) + query, err := Parse(tokens) if err != nil { panic(err) }