mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-09 22:51:02 +00:00
Merge branch 'query-refactor' into tlt
Conflicts: commands/pilosa-nexter/nexter.go query/parser.go query/planner_test.go
This commit is contained in:
travisturner
commit
ad2186a44c
11 changed files with 809 additions and 488 deletions
30
core/http.go
30
core/http.go
|
|
@ -173,15 +173,33 @@ func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
encoder := json.NewEncoder(w)
|
||||
//stats := service.GetStats()
|
||||
// stats := ""
|
||||
m := &runtime.MemStats{}
|
||||
runtime.ReadMemStats(m)
|
||||
stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc}
|
||||
|
||||
err := encoder.Encode(stats)
|
||||
//self.Report(fmt.Sprintf("%s.goroutines", prefix),
|
||||
// float64(runtime.NumGoroutine()), now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.allocated", prefix),
|
||||
// float64(memStats.Alloc), now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.mallocs", prefix),
|
||||
// float64(memStats.Mallocs), now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.frees", prefix),
|
||||
// float64(memStats.Frees), now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.gc.total_pause", prefix),
|
||||
// float64(memStats.PauseTotalNs)/nsInMs, now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.heap", prefix),
|
||||
// float64(memStats.HeapAlloc), now, context, dimensions)
|
||||
//self.Report(fmt.Sprintf("%s.memory.stack", prefix),
|
||||
// float64(memStats.StackInuse), now, context, dimensions)
|
||||
|
||||
//stats := map[string]interface{}{
|
||||
// "num_goroutines": runtime.NumGoroutine(),
|
||||
// "memory_allocated": m.Sys,
|
||||
// "memory_": m.Alloc
|
||||
//}
|
||||
|
||||
err := encoder.Encode(m)
|
||||
if err != nil {
|
||||
log.Fatal("Error encoding stats")
|
||||
panic("Error encoding stats")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -227,7 +245,7 @@ func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request)
|
|||
processes := self.service.ProcessMap.GetMetadata()
|
||||
err := encoder.Encode(processes)
|
||||
if err != nil {
|
||||
log.Fatal("Error encoding stats")
|
||||
panic("Error encoding stats")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) {
|
|||
case []byte:
|
||||
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
|
||||
}
|
||||
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N, categoryleaves)
|
||||
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, categoryleaves)
|
||||
if err != nil {
|
||||
spew.Dump(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -312,7 +312,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment
|
|||
//d.mutex.Lock()
|
||||
//defer d.mutex.Unlock()
|
||||
frame, _ := d.getFrame(bitmap.FrameType)
|
||||
log.Println(frame, slice)
|
||||
//log.Println(frame, slice)
|
||||
fsi, err := d.GetFrameSliceIntersect(frame, slice)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
|
|
|
|||
|
|
@ -83,5 +83,11 @@
|
|||
"repo": "github.com/gorilla/websocket",
|
||||
"version": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c",
|
||||
"type": "git"
|
||||
},
|
||||
"github.com/cactus/go-statsd-client/statsd": {
|
||||
"repo": "github.com/cactus/go-statsd-client",
|
||||
"version": "912f30c35e9cdf51f50bae24f071e227cca152fb",
|
||||
"type": "git"
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
225
query/lexer.go
225
query/lexer.go
|
|
@ -2,9 +2,8 @@ package query
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
|
|
@ -12,10 +11,18 @@ const (
|
|||
TYPE_FUNC = iota
|
||||
TYPE_LP = iota
|
||||
TYPE_RP = iota
|
||||
TYPE_LB = iota
|
||||
TYPE_RB = iota
|
||||
TYPE_VALUE = iota
|
||||
TYPE_KEYWORD = iota
|
||||
TYPE_EQUALS = iota
|
||||
TYPE_COMMA = iota
|
||||
TYPE_ERROR = iota
|
||||
|
||||
// Below types deprecated
|
||||
TYPE_ID = iota
|
||||
TYPE_FRAME = iota
|
||||
TYPE_PROFILE = iota
|
||||
TYPE_COMMA = iota
|
||||
TYPE_LIMIT = iota
|
||||
)
|
||||
|
||||
|
|
@ -36,19 +43,34 @@ type Lexer struct {
|
|||
}
|
||||
|
||||
func (lexer *Lexer) emit(typ int) {
|
||||
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
|
||||
if lexer.start < lexer.pos {
|
||||
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
|
||||
}
|
||||
lexer.start = lexer.pos
|
||||
}
|
||||
|
||||
func (lexer *Lexer) acceptUntil(chars string) error {
|
||||
func (lexer *Lexer) acceptUntil(chars string, consume bool) (rune, error) {
|
||||
start_pos := lexer.pos
|
||||
|
||||
for {
|
||||
if strings.HasPrefix(lexer.text[lexer.pos:], chars) {
|
||||
return nil
|
||||
next := lexer.next()
|
||||
|
||||
if next == rune(' ') {
|
||||
lexer.ignore()
|
||||
}
|
||||
// if we receive a reserved character that we are not expecting, throw a parse error
|
||||
lexer.pos += 1
|
||||
if lexer.pos > len(lexer.text) {
|
||||
return errors.New("Parse error, expecting " + string(chars))
|
||||
|
||||
if next == 0 {
|
||||
lexer.pos = start_pos
|
||||
return 0, errors.New("Not found")
|
||||
}
|
||||
ch := strings.IndexRune(chars, next)
|
||||
if ch >= 0 {
|
||||
if consume {
|
||||
lexer.backup()
|
||||
} else {
|
||||
lexer.pos = start_pos
|
||||
}
|
||||
return []rune(chars)[ch], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -59,15 +81,6 @@ func (lexer *Lexer) acceptRun(valid string) {
|
|||
lexer.backup()
|
||||
}
|
||||
|
||||
func (lexer *Lexer) acceptNumber() {
|
||||
digits := "0123456789"
|
||||
lexer.acceptRun(digits)
|
||||
}
|
||||
func (lexer *Lexer) acceptText() {
|
||||
digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")"
|
||||
lexer.acceptRun(digits)
|
||||
}
|
||||
|
||||
// next returns the next rune in the input.
|
||||
func (lexer *Lexer) next() (runey rune) {
|
||||
if lexer.pos >= len(lexer.text) {
|
||||
|
|
@ -104,10 +117,18 @@ func (lexer *Lexer) peek() rune {
|
|||
}
|
||||
}
|
||||
|
||||
func stateError(err error) func(lexer *Lexer) statefn {
|
||||
return func(lexer *Lexer) statefn {
|
||||
lexer.ch <- Token{err.Error(), TYPE_ERROR}
|
||||
close(lexer.ch)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func stateFunc(lexer *Lexer) statefn {
|
||||
err := lexer.acceptUntil("(")
|
||||
_, err := lexer.acceptUntil("(", true)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return stateError(err)
|
||||
}
|
||||
lexer.emit(TYPE_FUNC)
|
||||
return stateLP
|
||||
|
|
@ -123,34 +144,91 @@ func stateLP(lexer *Lexer) statefn {
|
|||
return stateArgs
|
||||
}
|
||||
|
||||
func stateLB(lexer *Lexer) statefn {
|
||||
lexer.acceptUntil("[", true)
|
||||
lexer.next()
|
||||
lexer.emit(TYPE_LB)
|
||||
for {
|
||||
r, err := lexer.acceptUntil(",]", true)
|
||||
if err != nil {
|
||||
return stateError(errors.New("Unclosed bracket!"))
|
||||
}
|
||||
lexer.emit(TYPE_VALUE)
|
||||
if r == ',' {
|
||||
lexer.next()
|
||||
lexer.emit(TYPE_COMMA)
|
||||
} else {
|
||||
lexer.next()
|
||||
lexer.emit(TYPE_RB)
|
||||
return stateArgs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stateArgs(lexer *Lexer) statefn {
|
||||
if unicode.IsNumber(lexer.peek()) {
|
||||
return stateID
|
||||
} else {
|
||||
r, err := lexer.acceptUntil("(),=[", false)
|
||||
if err != nil {
|
||||
return stateError(err)
|
||||
}
|
||||
switch r {
|
||||
case '(':
|
||||
return stateFunc
|
||||
case ')':
|
||||
return stateValue
|
||||
case ',':
|
||||
return stateValue
|
||||
case '=':
|
||||
return stateKeyword
|
||||
case '[':
|
||||
return stateLB
|
||||
default:
|
||||
return stateError(errors.New("Expecting arguments!"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stateID(lexer *Lexer) statefn {
|
||||
lexer.acceptNumber()
|
||||
lexer.emit(TYPE_ID)
|
||||
// if next is comma
|
||||
peeked := lexer.peek()
|
||||
if peeked == rune(',') {
|
||||
return stateFrameComma
|
||||
} else if peeked == rune(')') {
|
||||
func stateKeyword(lexer *Lexer) statefn {
|
||||
_, err := lexer.acceptUntil("=", true)
|
||||
if err != nil {
|
||||
return stateError(err)
|
||||
}
|
||||
lexer.emit(TYPE_KEYWORD)
|
||||
return stateEquals
|
||||
}
|
||||
|
||||
func stateEquals(lexer *Lexer) statefn {
|
||||
e := lexer.next()
|
||||
if e != '=' {
|
||||
return stateError(errors.New("Expecting '='!"))
|
||||
}
|
||||
lexer.emit(TYPE_EQUALS)
|
||||
return stateValue
|
||||
}
|
||||
|
||||
func stateValue(lexer *Lexer) statefn {
|
||||
r, err := lexer.acceptUntil("(),[", false)
|
||||
if err != nil {
|
||||
return stateError(err)
|
||||
}
|
||||
switch r {
|
||||
case '(':
|
||||
return stateFunc
|
||||
case ')':
|
||||
lexer.acceptUntil(")", true)
|
||||
if lexer.pos > lexer.start {
|
||||
lexer.emit(TYPE_VALUE)
|
||||
}
|
||||
return stateRP
|
||||
} else {
|
||||
return stateID
|
||||
case ',':
|
||||
lexer.acceptUntil(",", true)
|
||||
lexer.emit(TYPE_VALUE)
|
||||
return stateComma
|
||||
case '[':
|
||||
return stateLB
|
||||
default:
|
||||
return stateError(errors.New("Unexpected character!"))
|
||||
}
|
||||
}
|
||||
|
||||
func stateProfile(lexer *Lexer) statefn {
|
||||
lexer.peek()
|
||||
lexer.acceptNumber()
|
||||
lexer.emit(TYPE_PROFILE)
|
||||
lexer.peek()
|
||||
return stateRP
|
||||
return nil
|
||||
}
|
||||
|
||||
func stateRP(lexer *Lexer) statefn {
|
||||
|
|
@ -167,49 +245,10 @@ func stateRP(lexer *Lexer) statefn {
|
|||
}
|
||||
}
|
||||
|
||||
func stateFrameComma(lexer *Lexer) statefn {
|
||||
lexer.pos += 1
|
||||
lexer.emit(TYPE_COMMA)
|
||||
return stateFrameOrProfile
|
||||
}
|
||||
|
||||
func stateProfileComma(lexer *Lexer) statefn {
|
||||
lexer.pos += 1
|
||||
lexer.emit(TYPE_COMMA)
|
||||
return stateProfile
|
||||
}
|
||||
|
||||
func stateFrameOrProfile(lexer *Lexer) statefn {
|
||||
if unicode.IsNumber(lexer.peek()) {
|
||||
lexer.acceptNumber()
|
||||
lexer.emit(TYPE_PROFILE)
|
||||
lexer.peek()
|
||||
} else {
|
||||
lexer.acceptText()
|
||||
lexer.emit(TYPE_FRAME)
|
||||
if lexer.peek() == rune(',') {
|
||||
return stateProfileComma
|
||||
}
|
||||
}
|
||||
return stateRP
|
||||
}
|
||||
|
||||
func stateRPComma(lexer *Lexer) statefn {
|
||||
lexer.pos += 1
|
||||
lexer.emit(TYPE_COMMA)
|
||||
if unicode.IsNumber(lexer.peek()) {
|
||||
return stateLimit
|
||||
} else {
|
||||
return stateArgs
|
||||
}
|
||||
}
|
||||
|
||||
func stateLimit(lexer *Lexer) statefn {
|
||||
lexer.acceptNumber()
|
||||
lexer.emit(TYPE_LIMIT)
|
||||
// if next is comma
|
||||
lexer.peek()
|
||||
return stateRP
|
||||
return stateValue
|
||||
}
|
||||
|
||||
func stateComma(lexer *Lexer) statefn {
|
||||
|
|
@ -223,8 +262,17 @@ func stateEOF(lexer *Lexer) statefn {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (lexer *Lexer) Lex() []Token {
|
||||
tokens := make([]Token, 0)
|
||||
func (lexer *Lexer) Lex() (tokens []Token, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
var ok bool
|
||||
err, ok = r.(error)
|
||||
if !ok {
|
||||
err = fmt.Errorf("query: %v", r)
|
||||
}
|
||||
}
|
||||
}()
|
||||
tokens = make([]Token, 0)
|
||||
state := stateFunc
|
||||
go func() {
|
||||
for {
|
||||
|
|
@ -235,12 +283,15 @@ func (lexer *Lexer) Lex() []Token {
|
|||
}
|
||||
}()
|
||||
for t := range lexer.ch {
|
||||
if t.Type == TYPE_ERROR {
|
||||
err = errors.New(t.Text)
|
||||
}
|
||||
tokens = append(tokens, t)
|
||||
}
|
||||
return tokens
|
||||
return tokens, err
|
||||
}
|
||||
|
||||
func Lex(input string) []Token {
|
||||
func Lex(input string) ([]Token, error) {
|
||||
lexer := Lexer{input, 0, 0, 0, TYPE_FUNC, make(chan Token)}
|
||||
return lexer.Lex()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,180 +7,184 @@ import (
|
|||
|
||||
func TestLexer(t *testing.T) {
|
||||
Convey("Basic lexical analysis", t, func() {
|
||||
var tokens []Token
|
||||
var err error
|
||||
|
||||
tokens := Lex("get(10)")
|
||||
tokens, err = Lex("get(10)")
|
||||
So(err, ShouldBeNil)
|
||||
So(len(tokens), ShouldEqual, 4)
|
||||
So(tokens[0].Text, ShouldEqual, "get")
|
||||
So(tokens[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens[1].Text, ShouldEqual, "(")
|
||||
So(tokens[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens[2].Text, ShouldEqual, "10")
|
||||
So(tokens[2].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens[3].Text, ShouldEqual, ")")
|
||||
So(tokens[3].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"10", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens2 := Lex("intersect(get(10), get(11), get(12))")
|
||||
So(len(tokens2), ShouldEqual, 17)
|
||||
So(tokens2[0].Text, ShouldEqual, "intersect")
|
||||
So(tokens2[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens2[1].Text, ShouldEqual, "(")
|
||||
So(tokens2[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens2[2].Text, ShouldEqual, "get")
|
||||
So(tokens2[2].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens2[3].Text, ShouldEqual, "(")
|
||||
So(tokens2[3].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens2[4].Text, ShouldEqual, "10")
|
||||
So(tokens2[4].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens2[5].Text, ShouldEqual, ")")
|
||||
So(tokens2[5].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens2[6].Text, ShouldEqual, ",")
|
||||
So(tokens2[6].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens2[7].Text, ShouldEqual, "get")
|
||||
So(tokens2[7].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens2[8].Text, ShouldEqual, "(")
|
||||
So(tokens2[8].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens2[9].Text, ShouldEqual, "11")
|
||||
So(tokens2[9].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens2[10].Text, ShouldEqual, ")")
|
||||
So(tokens2[10].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens2[11].Text, ShouldEqual, ",")
|
||||
So(tokens2[11].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens2[12].Text, ShouldEqual, "get")
|
||||
So(tokens2[12].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens2[13].Text, ShouldEqual, "(")
|
||||
So(tokens2[13].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens2[14].Text, ShouldEqual, "12")
|
||||
So(tokens2[14].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens2[15].Text, ShouldEqual, ")")
|
||||
So(tokens2[15].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens2[16].Text, ShouldEqual, ")")
|
||||
So(tokens2[16].Type, ShouldEqual, TYPE_RP)
|
||||
tokens, err = Lex("get(id=10)")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"id", TYPE_KEYWORD},
|
||||
{"=", TYPE_EQUALS},
|
||||
{"10", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens3 := Lex("intersect(get(10), get(11), concat(get(12),get(14)))")
|
||||
So(len(tokens3), ShouldEqual, 25)
|
||||
So(tokens3[0].Text, ShouldEqual, "intersect")
|
||||
So(tokens3[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens3[1].Text, ShouldEqual, "(")
|
||||
So(tokens3[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens3[2].Text, ShouldEqual, "get")
|
||||
So(tokens3[2].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens3[3].Text, ShouldEqual, "(")
|
||||
So(tokens3[3].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens3[4].Text, ShouldEqual, "10")
|
||||
So(tokens3[4].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens3[5].Text, ShouldEqual, ")")
|
||||
So(tokens3[5].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens3[6].Text, ShouldEqual, ",")
|
||||
So(tokens3[6].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens3[7].Text, ShouldEqual, "get")
|
||||
So(tokens3[7].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens3[8].Text, ShouldEqual, "(")
|
||||
So(tokens3[8].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens3[9].Text, ShouldEqual, "11")
|
||||
So(tokens3[9].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens3[10].Text, ShouldEqual, ")")
|
||||
So(tokens3[10].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens3[11].Text, ShouldEqual, ",")
|
||||
So(tokens3[11].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens3[12].Text, ShouldEqual, "concat")
|
||||
So(tokens3[12].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens3[13].Text, ShouldEqual, "(")
|
||||
So(tokens3[13].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens3[14].Text, ShouldEqual, "get")
|
||||
So(tokens3[14].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens3[15].Text, ShouldEqual, "(")
|
||||
So(tokens3[15].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens3[16].Text, ShouldEqual, "12")
|
||||
So(tokens3[16].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens3[17].Text, ShouldEqual, ")")
|
||||
So(tokens3[17].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens3[18].Text, ShouldEqual, ",")
|
||||
So(tokens3[18].Type, ShouldEqual, TYPE_COMMA)
|
||||
tokens, err = Lex("get(id=10, frame=brand)")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"id", TYPE_KEYWORD},
|
||||
{"=", TYPE_EQUALS},
|
||||
{"10", TYPE_VALUE},
|
||||
{",", TYPE_COMMA},
|
||||
{"frame", TYPE_KEYWORD},
|
||||
{"=", TYPE_EQUALS},
|
||||
{"brand", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens4 := Lex("concat(get(1, brand),get(2))")
|
||||
So(len(tokens4), ShouldEqual, 14)
|
||||
So(tokens4[0].Text, ShouldEqual, "concat")
|
||||
So(tokens4[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens4[1].Text, ShouldEqual, "(")
|
||||
So(tokens4[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens4[2].Text, ShouldEqual, "get")
|
||||
So(tokens4[2].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens4[3].Text, ShouldEqual, "(")
|
||||
So(tokens4[3].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens4[4].Text, ShouldEqual, "1")
|
||||
So(tokens4[4].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens4[5].Text, ShouldEqual, ",")
|
||||
So(tokens4[5].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens4[6].Text, ShouldEqual, "brand")
|
||||
So(tokens4[6].Type, ShouldEqual, TYPE_FRAME)
|
||||
So(tokens4[7].Text, ShouldEqual, ")")
|
||||
So(tokens4[7].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens4[8].Text, ShouldEqual, ",")
|
||||
So(tokens4[8].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens4[9].Text, ShouldEqual, "get")
|
||||
So(tokens4[9].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens4[10].Text, ShouldEqual, "(")
|
||||
So(tokens4[10].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens4[11].Text, ShouldEqual, "2")
|
||||
So(tokens4[11].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens4[12].Text, ShouldEqual, ")")
|
||||
So(tokens4[12].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens4[13].Text, ShouldEqual, ")")
|
||||
So(tokens4[13].Type, ShouldEqual, TYPE_RP)
|
||||
tokens, err = Lex("union(get(10))")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"union", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"10", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens5 := Lex("set(1, 987)")
|
||||
So(len(tokens5), ShouldEqual, 6)
|
||||
So(tokens5[0].Text, ShouldEqual, "set")
|
||||
So(tokens5[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens5[1].Text, ShouldEqual, "(")
|
||||
So(tokens5[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens5[2].Text, ShouldEqual, "1")
|
||||
So(tokens5[2].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens5[3].Text, ShouldEqual, ",")
|
||||
So(tokens5[3].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens5[4].Text, ShouldEqual, "987")
|
||||
So(tokens5[4].Type, ShouldEqual, TYPE_PROFILE)
|
||||
So(tokens5[5].Text, ShouldEqual, ")")
|
||||
So(tokens5[5].Type, ShouldEqual, TYPE_RP)
|
||||
tokens, err = Lex("intersect(get(10), get(11), get(12))")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"intersect", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"10", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"11", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"12", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens6 := Lex("set(1, general, 987)")
|
||||
So(len(tokens6), ShouldEqual, 8)
|
||||
So(tokens6[0].Text, ShouldEqual, "set")
|
||||
So(tokens6[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens6[1].Text, ShouldEqual, "(")
|
||||
So(tokens6[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens6[2].Text, ShouldEqual, "1")
|
||||
So(tokens6[2].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens6[3].Text, ShouldEqual, ",")
|
||||
So(tokens6[3].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens6[4].Text, ShouldEqual, "general")
|
||||
So(tokens6[4].Type, ShouldEqual, TYPE_FRAME)
|
||||
So(tokens6[5].Text, ShouldEqual, ",")
|
||||
So(tokens6[5].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens6[6].Text, ShouldEqual, "987")
|
||||
So(tokens6[6].Type, ShouldEqual, TYPE_PROFILE)
|
||||
So(tokens6[7].Text, ShouldEqual, ")")
|
||||
So(tokens6[7].Type, ShouldEqual, TYPE_RP)
|
||||
tokens, err = Lex("intersect(get(10), get(11), concat(get(12),get(14)))")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"intersect", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"10", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"11", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"concat", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"12", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"14", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{")", TYPE_RP},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens7 := Lex("top-n(get(10), 8)")
|
||||
So(len(tokens6), ShouldEqual, 8)
|
||||
So(tokens7[0].Text, ShouldEqual, "top-n")
|
||||
So(tokens7[0].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens7[1].Text, ShouldEqual, "(")
|
||||
So(tokens7[1].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens7[2].Text, ShouldEqual, "get")
|
||||
So(tokens7[2].Type, ShouldEqual, TYPE_FUNC)
|
||||
So(tokens7[3].Text, ShouldEqual, "(")
|
||||
So(tokens7[3].Type, ShouldEqual, TYPE_LP)
|
||||
So(tokens7[4].Text, ShouldEqual, "10")
|
||||
So(tokens7[4].Type, ShouldEqual, TYPE_ID)
|
||||
So(tokens7[5].Text, ShouldEqual, ")")
|
||||
So(tokens7[5].Type, ShouldEqual, TYPE_RP)
|
||||
So(tokens7[6].Text, ShouldEqual, ",")
|
||||
So(tokens7[6].Type, ShouldEqual, TYPE_COMMA)
|
||||
So(tokens7[7].Text, ShouldEqual, "8")
|
||||
So(tokens7[7].Type, ShouldEqual, TYPE_LIMIT)
|
||||
So(tokens7[8].Text, ShouldEqual, ")")
|
||||
So(tokens7[8].Type, ShouldEqual, TYPE_RP)
|
||||
tokens, err = Lex("concat(get(1, brand),get(2))")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"concat", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"1", TYPE_VALUE},
|
||||
{",", TYPE_COMMA},
|
||||
{"brand", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{",", TYPE_COMMA},
|
||||
{"get", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"2", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens, err = Lex("set(1, 987)")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"set", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"1", TYPE_VALUE},
|
||||
{",", TYPE_COMMA},
|
||||
{"987", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens, err = Lex("set(1, general, 987)")
|
||||
So(err, ShouldBeNil)
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
{"set", TYPE_FUNC},
|
||||
{"(", TYPE_LP},
|
||||
{"1", TYPE_VALUE},
|
||||
{",", TYPE_COMMA},
|
||||
{"general", TYPE_VALUE},
|
||||
{",", TYPE_COMMA},
|
||||
{"987", TYPE_VALUE},
|
||||
{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens, err = Lex("top-n(get(10), 8)")
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
Token{"top-n", TYPE_FUNC},
|
||||
Token{"(", TYPE_LP},
|
||||
Token{"get", TYPE_FUNC},
|
||||
Token{"(", TYPE_LP},
|
||||
Token{"10", TYPE_VALUE},
|
||||
Token{")", TYPE_RP},
|
||||
Token{",", TYPE_COMMA},
|
||||
Token{"8", TYPE_VALUE},
|
||||
Token{")", TYPE_RP},
|
||||
})
|
||||
|
||||
tokens, err = Lex("top-n(get(10, general), [1,2,3])")
|
||||
So(tokens, ShouldResemble, []Token{
|
||||
Token{"top-n", TYPE_FUNC},
|
||||
Token{"(", TYPE_LP},
|
||||
Token{"get", TYPE_FUNC},
|
||||
Token{"(", TYPE_LP},
|
||||
Token{"10", TYPE_VALUE},
|
||||
Token{",", TYPE_COMMA},
|
||||
Token{"general", TYPE_VALUE},
|
||||
Token{")", TYPE_RP},
|
||||
Token{",", TYPE_COMMA},
|
||||
Token{"[", TYPE_LB},
|
||||
Token{"1", TYPE_VALUE},
|
||||
Token{",", TYPE_COMMA},
|
||||
Token{"2", TYPE_VALUE},
|
||||
Token{",", TYPE_COMMA},
|
||||
Token{"3", TYPE_VALUE},
|
||||
Token{"]", TYPE_RB},
|
||||
Token{")", TYPE_RP},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
273
query/parser.go
273
query/parser.go
|
|
@ -2,111 +2,210 @@ package query
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"pilosa/db"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"tux21b.org/v1/gocql/uuid"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
)
|
||||
|
||||
var InvalidQueryError = errors.New("Invalid query format.")
|
||||
|
||||
type QueryParser struct{}
|
||||
|
||||
func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64, int) {
|
||||
// BITMAP
|
||||
if tokens[0].Type == TYPE_ID {
|
||||
// TODO: look for frame type in the tokens list
|
||||
b, err := strconv.ParseUint(tokens[0].Text, 10, 64)
|
||||
bitmap_id := uint64(b)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// if the next 2 tokens are comma-frame, then we have a frame, else set to a default
|
||||
frame_type := "general"
|
||||
profile_id := uint64(0)
|
||||
|
||||
if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE {
|
||||
frame_type = tokens[2].Text
|
||||
profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
} else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME {
|
||||
frame_type = tokens[2].Text
|
||||
} else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE {
|
||||
profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
var filter int //TRAVIS the is for the category
|
||||
filter = 0
|
||||
bm := db.Bitmap{bitmap_id, frame_type, filter}
|
||||
return []QueryInput{&bm}, uint64(profile_id), 0
|
||||
}
|
||||
|
||||
// LIST OF QUERIES
|
||||
n := int(10) // default LIMIT to 10
|
||||
qi := []QueryInput{}
|
||||
open_parens := -1 // >=0 means i'm inside the search for end paren
|
||||
start := 0
|
||||
for i := 0; i < len(tokens); i++ {
|
||||
if tokens[i].Type == TYPE_FUNC && open_parens == -1 {
|
||||
start = i
|
||||
} else if tokens[i].Type == TYPE_LP {
|
||||
open_parens++
|
||||
} else if tokens[i].Type == TYPE_RP {
|
||||
if open_parens == 0 {
|
||||
q, err := qp.walk(tokens[start : i+1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
qi = append(qi, q)
|
||||
open_parens = -1
|
||||
} else {
|
||||
open_parens--
|
||||
}
|
||||
} else if tokens[i].Type == TYPE_LIMIT {
|
||||
x, _ := strconv.ParseInt(tokens[i].Text, 10, 32)
|
||||
n = int(x)
|
||||
}
|
||||
}
|
||||
return qi, 0, n
|
||||
type QueryParser struct {
|
||||
tokens []Token
|
||||
pos int
|
||||
}
|
||||
|
||||
func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
|
||||
|
||||
if tokens[0].Type != TYPE_FUNC {
|
||||
panic("BAD!")
|
||||
/* MASTER
|
||||
if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE {
|
||||
frame_type = tokens[2].Text
|
||||
profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if tokens[1].Type != TYPE_LP {
|
||||
panic("BAD!")
|
||||
} else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME {
|
||||
frame_type = tokens[2].Text
|
||||
} else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE {
|
||||
profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
var filter int //TRAVIS the is for the category
|
||||
filter = 0
|
||||
bm := db.Bitmap{bitmap_id, frame_type, filter}
|
||||
return []QueryInput{&bm}, uint64(profile_id), 0
|
||||
*/
|
||||
func (self *QueryParser) next() *Token {
|
||||
self.pos += 1
|
||||
if self.pos > len(self.tokens) {
|
||||
return nil
|
||||
}
|
||||
return &self.tokens[self.pos-1]
|
||||
}
|
||||
|
||||
func (self *QueryParser) peek() *Token {
|
||||
token := self.next()
|
||||
self.backup()
|
||||
return token
|
||||
}
|
||||
|
||||
func (self *QueryParser) backup() {
|
||||
self.pos -= 1
|
||||
}
|
||||
|
||||
func (self *QueryParser) Parse() (query *Query, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
var ok bool
|
||||
err, ok = r.(error)
|
||||
if !ok {
|
||||
err = fmt.Errorf("query: %v", r)
|
||||
}
|
||||
}
|
||||
}()
|
||||
var token *Token
|
||||
|
||||
q := new(Query)
|
||||
id := uuid.RandomUUID()
|
||||
q.Id = &id
|
||||
q.Operation = tokens[0].Text
|
||||
query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})}
|
||||
|
||||
// scan from open to close paren
|
||||
open_parens := 0
|
||||
for i := 2; i < len(tokens); i++ {
|
||||
// 1 must be "("
|
||||
if tokens[i].Type == TYPE_LP {
|
||||
open_parens++
|
||||
} else if tokens[i].Type == TYPE_RP {
|
||||
if open_parens == 0 {
|
||||
if i == len(tokens)-1 {
|
||||
q.Inputs, q.ProfileId, q.N = qp.walkInputs(tokens[2:i])
|
||||
token = self.next()
|
||||
if token.Type != TYPE_FUNC {
|
||||
return nil, fmt.Errorf("Expected function, found token %v.", token)
|
||||
}
|
||||
|
||||
query.Operation = token.Text
|
||||
|
||||
token = self.next()
|
||||
if token.Type != TYPE_LP {
|
||||
return nil, fmt.Errorf("Expected '(', found token %v.", token)
|
||||
}
|
||||
|
||||
ArgLoop:
|
||||
for {
|
||||
token = self.next()
|
||||
if token == nil {
|
||||
return nil, fmt.Errorf("Unclosed parentheses!")
|
||||
}
|
||||
|
||||
switch token.Type {
|
||||
case TYPE_FUNC:
|
||||
self.backup()
|
||||
subquery, err := self.Parse()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query.Subqueries = append(query.Subqueries, *subquery)
|
||||
case TYPE_VALUE:
|
||||
switch query.Operation {
|
||||
case "get":
|
||||
switch len(query.Args) {
|
||||
case 0:
|
||||
i, err := strconv.ParseUint(token.Text, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
|
||||
}
|
||||
query.Args["id"] = i
|
||||
case 1:
|
||||
query.Args["frame"] = token.Text
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
|
||||
}
|
||||
case "set":
|
||||
switch len(query.Args) {
|
||||
case 0:
|
||||
i, err := strconv.ParseUint(token.Text, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
|
||||
}
|
||||
query.Args["id"] = i
|
||||
case 1:
|
||||
query.Args["frame"] = token.Text
|
||||
case 2:
|
||||
i, err := strconv.ParseUint(token.Text, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
|
||||
}
|
||||
query.Args["profile_id"] = i
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
|
||||
}
|
||||
|
||||
case "top-n":
|
||||
i, err := strconv.Atoi(token.Text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer! (%v)", err)
|
||||
}
|
||||
query.Args["n"] = i
|
||||
default:
|
||||
spew.Dump("UNPROCESSED VALUE", token)
|
||||
}
|
||||
continue
|
||||
case TYPE_COMMA:
|
||||
continue
|
||||
case TYPE_RP:
|
||||
break ArgLoop
|
||||
case TYPE_KEYWORD:
|
||||
var value interface{}
|
||||
keyword := token.Text
|
||||
token = self.next()
|
||||
if token == nil || token.Type != TYPE_EQUALS {
|
||||
return nil, fmt.Errorf("Expecting equals sign!")
|
||||
}
|
||||
token = self.next()
|
||||
if token == nil || token.Type != TYPE_VALUE {
|
||||
return nil, fmt.Errorf("Expecting value!")
|
||||
}
|
||||
if keyword == "id" {
|
||||
value, err = strconv.ParseUint(token.Text, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
|
||||
}
|
||||
} else if keyword == "n" {
|
||||
value, err = strconv.Atoi(token.Text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer! (%v)", err)
|
||||
}
|
||||
} else {
|
||||
open_parens--
|
||||
value = token.Text
|
||||
}
|
||||
query.Args[keyword] = value
|
||||
case TYPE_LB:
|
||||
query.Args["ids"] = make([]uint64, 0)
|
||||
for {
|
||||
token = self.next()
|
||||
if token == nil {
|
||||
return nil, fmt.Errorf("Unclosed list!")
|
||||
}
|
||||
switch token.Type {
|
||||
case TYPE_COMMA:
|
||||
break
|
||||
case TYPE_VALUE:
|
||||
i, err := strconv.ParseUint(token.Text, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
|
||||
}
|
||||
query.Args["ids"] = append(query.Args["ids"].([]uint64), i)
|
||||
case TYPE_RB:
|
||||
continue ArgLoop
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected token! (%v)", token)
|
||||
}
|
||||
}
|
||||
default:
|
||||
spew.Dump("unexpected", token)
|
||||
panic(token)
|
||||
}
|
||||
}
|
||||
return q, nil
|
||||
|
||||
if query.Operation == "get" && query.Args["frame"] == nil {
|
||||
query.Args["frame"] = "general"
|
||||
}
|
||||
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func (qp *QueryParser) Parse(tokens []Token) (*Query, error) {
|
||||
return qp.walk(tokens)
|
||||
func Parse(tokens []Token) (*Query, error) {
|
||||
parser := QueryParser{tokens, 0}
|
||||
return parser.Parse()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,20 +2,79 @@ package query
|
|||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestQueryParser(t *testing.T) {
|
||||
Convey("Basic query parse", t, func() {
|
||||
Convey("Basic parse - get()", t, func() {
|
||||
tokens, err := Lex("get(10)")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
tokens := Lex("union(get(10,general), get(11,brand), get(12))")
|
||||
qp := QueryParser{}
|
||||
q, err := qp.Parse(tokens)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
spew.Dump(q)
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "get")
|
||||
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"})
|
||||
})
|
||||
Convey("Basic parse - set()", t, func() {
|
||||
tokens, err := Lex("set(10, general, 20)")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "set")
|
||||
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "profile_id": uint64(20)})
|
||||
})
|
||||
Convey("Basic nested query parse", t, func() {
|
||||
tokens, err := Lex("union(get(10,general), get(11,brand), get(12))")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "union")
|
||||
So(len(query.Subqueries), ShouldEqual, 3)
|
||||
|
||||
So(query.Subqueries[0].Operation, ShouldEqual, "get")
|
||||
So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"})
|
||||
So(query.Subqueries[1].Operation, ShouldEqual, "get")
|
||||
So(query.Subqueries[1].Args, ShouldResemble, map[string]interface{}{"id": uint64(11), "frame": "brand"})
|
||||
So(query.Subqueries[2].Operation, ShouldEqual, "get")
|
||||
So(query.Subqueries[2].Args, ShouldResemble, map[string]interface{}{"id": uint64(12), "frame": "general"})
|
||||
})
|
||||
Convey("Keyword args", t, func() {
|
||||
tokens, err := Lex("get(id=10)")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "get")
|
||||
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"})
|
||||
})
|
||||
Convey("Keyword args - multiple", t, func() {
|
||||
tokens, err := Lex("get(id=10, frame=brands)")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "get")
|
||||
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "brands"})
|
||||
})
|
||||
Convey("Lists", t, func() {
|
||||
tokens, err := Lex("top-n(get(10, general), [1,2,3], 50)")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
query, err := Parse(tokens)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(query.Operation, ShouldEqual, "top-n")
|
||||
So(query.Args, ShouldResemble, map[string]interface{}{"ids": []uint64{1, 2, 3}, "n": 50})
|
||||
|
||||
So(len(query.Subqueries), ShouldEqual, 1)
|
||||
So(query.Subqueries[0].Operation, ShouldEqual, "get")
|
||||
So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
109
query/planner.go
109
query/planner.go
|
|
@ -2,26 +2,12 @@ package query
|
|||
|
||||
import (
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"pilosa/db"
|
||||
|
||||
"tux21b.org/v1/gocql/uuid"
|
||||
)
|
||||
|
||||
// A single step in the query plan.
|
||||
type QueryStep struct {
|
||||
id *uuid.UUID
|
||||
operation string
|
||||
inputs []QueryInput
|
||||
location *db.Location
|
||||
destination *db.Location
|
||||
}
|
||||
|
||||
func (q QueryStep) StringHOLD() string {
|
||||
return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.destination)
|
||||
}
|
||||
|
||||
type PortableQueryStep interface {
|
||||
GetId() *uuid.UUID
|
||||
GetLocation() *db.Location
|
||||
|
|
@ -105,6 +91,7 @@ type TopNQueryResult struct {
|
|||
type TopNQueryTree struct {
|
||||
subquery QueryTree
|
||||
location *db.Location
|
||||
N int
|
||||
}
|
||||
|
||||
// Uses consistent hashing function to select node containing data for GET operation
|
||||
|
|
@ -191,6 +178,7 @@ type CatQueryResult struct {
|
|||
type CatQueryTree struct {
|
||||
subqueries []QueryTree
|
||||
location *db.Location
|
||||
N int
|
||||
}
|
||||
|
||||
// Uses consistent hashing function to select node containing data for GET operation
|
||||
|
|
@ -292,44 +280,31 @@ type QueryPlan []interface{}
|
|||
|
||||
type QueryPlanner struct {
|
||||
Database *db.Database
|
||||
Query *Query
|
||||
}
|
||||
type QueryTree interface {
|
||||
getLocation(d *db.Database) *db.Location
|
||||
}
|
||||
|
||||
// QueryTree for UNION and INTERSECT queries
|
||||
type CompositeQueryTree struct {
|
||||
operation string
|
||||
subqueries []QueryTree
|
||||
location *db.Location
|
||||
}
|
||||
|
||||
// Randomly select location from subqueries (so subqueries roll up into composite queries while minimizing inter-node data traffic)
|
||||
func (qt *CompositeQueryTree) getLocation(d *db.Database) *db.Location {
|
||||
if qt.location == nil {
|
||||
subqueryLength := len(qt.subqueries)
|
||||
if subqueryLength > 0 {
|
||||
locationIndex := rand.Intn(subqueryLength)
|
||||
subquery := qt.subqueries[locationIndex]
|
||||
qt.location = subquery.getLocation(d)
|
||||
}
|
||||
}
|
||||
return qt.location
|
||||
}
|
||||
|
||||
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
|
||||
func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
|
||||
var tree QueryTree
|
||||
|
||||
// handle SET operation regardless of the slice
|
||||
if query.Operation == "set" {
|
||||
tree = &SetQueryTree{query.Inputs[0].(*db.Bitmap), query.ProfileId}
|
||||
|
||||
tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, query.Args["profile_id"].(uint64)}
|
||||
return tree
|
||||
}
|
||||
|
||||
// handle the remaining operations, taking slice into consideration
|
||||
if slice == -1 {
|
||||
tree = &CatQueryTree{}
|
||||
var n int
|
||||
n_, ok := query.Args["n"]
|
||||
if ok {
|
||||
n = n_.(int)
|
||||
}
|
||||
tree = &CatQueryTree{N: n}
|
||||
numSlices, err := qp.Database.NumSlices()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -342,57 +317,48 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
|
|||
}
|
||||
} else {
|
||||
if query.Operation == "get" {
|
||||
tree = &GetQueryTree{query.Inputs[0].(*db.Bitmap), slice}
|
||||
tree = &GetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string)}, slice}
|
||||
return tree
|
||||
} else if query.Operation == "count" {
|
||||
subquery := qp.buildTree(query.Inputs[0].(*Query), slice)
|
||||
subquery := qp.buildTree(&query.Subqueries[0], slice)
|
||||
tree = &CountQueryTree{subquery: subquery}
|
||||
} else if query.Operation == "top-n" {
|
||||
subquery := qp.buildTree(query.Inputs[0].(*Query), slice)
|
||||
tree = &TopNQueryTree{subquery: subquery}
|
||||
var n int
|
||||
n_, ok := query.Args["n"]
|
||||
if ok {
|
||||
n = n_.(int)
|
||||
}
|
||||
subquery := qp.buildTree(&query.Subqueries[0], slice)
|
||||
tree = &TopNQueryTree{subquery: subquery, N: n}
|
||||
} else if query.Operation == "union" {
|
||||
subqueries := make([]QueryTree, len(query.Inputs))
|
||||
for i, input := range query.Inputs {
|
||||
subqueries[i] = qp.buildTree(input.(*Query), slice)
|
||||
subqueries := make([]QueryTree, len(query.Subqueries))
|
||||
for i, query := range query.Subqueries {
|
||||
subqueries[i] = qp.buildTree(&query, slice)
|
||||
}
|
||||
tree = &UnionQueryTree{subqueries: subqueries}
|
||||
} else if query.Operation == "intersect" {
|
||||
subqueries := make([]QueryTree, len(query.Inputs))
|
||||
for i, input := range query.Inputs {
|
||||
subqueries[i] = qp.buildTree(input.(*Query), slice)
|
||||
subqueries := make([]QueryTree, len(query.Subqueries))
|
||||
for i, query := range query.Subqueries {
|
||||
subqueries[i] = qp.buildTree(&query, slice)
|
||||
}
|
||||
tree = &IntersectQueryTree{subqueries: subqueries}
|
||||
} else {
|
||||
subqueries := make([]QueryTree, len(query.Inputs))
|
||||
for i, input := range query.Inputs {
|
||||
subqueries[i] = qp.buildTree(input.(*Query), slice)
|
||||
}
|
||||
tree = &CompositeQueryTree{operation: query.Operation, subqueries: subqueries}
|
||||
panic("invalid operation")
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// Produces flattened QueryPlan from QueryTree input
|
||||
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location, n int) *QueryPlan {
|
||||
func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Location) *QueryPlan {
|
||||
plan := QueryPlan{}
|
||||
if composite, ok := qt.(*CompositeQueryTree); ok {
|
||||
inputs := make([]QueryInput, len(composite.subqueries))
|
||||
step := QueryStep{id, composite.operation, inputs, composite.getLocation(qp.Database), location}
|
||||
for index, subq := range composite.subqueries {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step.inputs[index] = &sub_id
|
||||
subq_steps := qp.flatten(subq, &sub_id, composite.getLocation(qp.Database), n)
|
||||
plan = append(plan, *subq_steps...)
|
||||
}
|
||||
plan = append(plan, step)
|
||||
} else if cat, ok := qt.(*CatQueryTree); ok {
|
||||
if cat, ok := qt.(*CatQueryTree); ok {
|
||||
inputs := make([]*uuid.UUID, len(cat.subqueries))
|
||||
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, n}
|
||||
step := CatQueryStep{&BaseQueryStep{id, "cat", cat.getLocation(qp.Database), location}, inputs, cat.N}
|
||||
for index, subq := range cat.subqueries {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step.Inputs[index] = &sub_id
|
||||
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database), n)
|
||||
subq_steps := qp.flatten(subq, &sub_id, cat.getLocation(qp.Database))
|
||||
plan = append(plan, *subq_steps...)
|
||||
}
|
||||
plan = append(plan, step)
|
||||
|
|
@ -402,7 +368,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
|
|||
for index, subq := range union.subqueries {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step.Inputs[index] = &sub_id
|
||||
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database), n)
|
||||
subq_steps := qp.flatten(subq, &sub_id, union.getLocation(qp.Database))
|
||||
plan = append(plan, *subq_steps...)
|
||||
}
|
||||
plan = append(plan, step)
|
||||
|
|
@ -412,7 +378,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
|
|||
for index, subq := range intersect.subqueries {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step.Inputs[index] = &sub_id
|
||||
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database), n)
|
||||
subq_steps := qp.flatten(subq, &sub_id, intersect.getLocation(qp.Database))
|
||||
plan = append(plan, *subq_steps...)
|
||||
}
|
||||
plan = append(plan, step)
|
||||
|
|
@ -427,13 +393,13 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
|
|||
} else if cnt, ok := qt.(*CountQueryTree); ok {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step := &CountQueryStep{&BaseQueryStep{id, "count", cnt.getLocation(qp.Database), location}, &sub_id}
|
||||
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database), n)
|
||||
subq_steps := qp.flatten(cnt.subquery, &sub_id, cnt.getLocation(qp.Database))
|
||||
plan = append(plan, *subq_steps...)
|
||||
plan = append(plan, step)
|
||||
} else if topn, ok := qt.(*TopNQueryTree); ok {
|
||||
sub_id := uuid.RandomUUID()
|
||||
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, n}
|
||||
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database), n)
|
||||
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.N}
|
||||
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database))
|
||||
plan = append(plan, *subq_steps...)
|
||||
plan = append(plan, step)
|
||||
}
|
||||
|
|
@ -443,6 +409,5 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
|
|||
// Transforms Query into QueryTree and flattens to QueryPlan object
|
||||
func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan {
|
||||
queryTree := qp.buildTree(query, -1)
|
||||
//return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value
|
||||
return qp.flatten(queryTree, query.Id, destination, query.N)
|
||||
return qp.flatten(queryTree, query.Id, destination)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,67 +4,184 @@ import (
|
|||
"pilosa/db"
|
||||
"pilosa/util"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"tux21b.org/v1/gocql/uuid"
|
||||
)
|
||||
|
||||
func basic_database() (*db.Database, *db.Fragment) {
|
||||
// create an empty database
|
||||
cluster := db.NewCluster()
|
||||
database := cluster.GetOrCreateDatabase("main")
|
||||
frame := database.GetOrCreateFrame("general")
|
||||
|
||||
slice1 := database.GetOrCreateSlice(0)
|
||||
fragment_id1 := util.Id()
|
||||
fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1)
|
||||
process_id1 := uuid.RandomUUID()
|
||||
process1 := db.NewProcess(&process_id1)
|
||||
process1.SetHost("----192.1.1.0----")
|
||||
fragment1.SetProcess(process1)
|
||||
|
||||
slice2 := database.GetOrCreateSlice(1)
|
||||
fragment_id2 := util.Id()
|
||||
fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2)
|
||||
process_id2 := uuid.RandomUUID()
|
||||
process2 := db.NewProcess(&process_id2)
|
||||
process2.SetHost("----192.1.1.1----")
|
||||
fragment2.SetProcess(process2)
|
||||
return database, fragment1
|
||||
}
|
||||
|
||||
func TestQueryPlanner(t *testing.T) {
|
||||
Convey("Basic query plan", t, func() {
|
||||
Convey("Union query plan", t, func() {
|
||||
|
||||
id1 := uuid.RandomUUID()
|
||||
bm1 := db.Bitmap{10, "general", 0}
|
||||
inputs1 := []QueryInput{&bm1}
|
||||
query1 := Query{&id1, "get", inputs1, 0, 0}
|
||||
query1 := Query{Id: &id1, Operation: "get", Args: map[string]interface{}{"id": uint64(10), "frame": "general"}}
|
||||
|
||||
id2 := uuid.RandomUUID()
|
||||
bm2 := db.Bitmap{20, "general", 0}
|
||||
inputs2 := []QueryInput{&bm2}
|
||||
query2 := Query{&id2, "get", inputs2, 0, 0}
|
||||
query2 := Query{Id: &id2, Operation: "get", Args: map[string]interface{}{"id": uint64(20), "frame": "general"}}
|
||||
|
||||
id3 := uuid.RandomUUID()
|
||||
inputs := []QueryInput{&query1, &query2}
|
||||
query := Query{&id3, "union", inputs, 0, 0}
|
||||
/*
|
||||
query := Query{Id: &id3, Operation: "union", Subqueries: []Query{query1, query2}}
|
||||
|
||||
bm1 := db.Bitmap{10, "general"}
|
||||
inputs1 := []QueryInput{&bm1}
|
||||
query1 := Query{"get", inputs1}
|
||||
query := query1
|
||||
*/
|
||||
database, fragment1 := basic_database()
|
||||
|
||||
// create an empty database
|
||||
cluster := db.NewCluster()
|
||||
database := cluster.GetOrCreateDatabase("main")
|
||||
frame := database.GetOrCreateFrame("general")
|
||||
|
||||
slice1 := database.GetOrCreateSlice(0)
|
||||
fragment_id1 := util.Id()
|
||||
fragment1 := database.GetOrCreateFragment(frame, slice1, fragment_id1)
|
||||
process_id1 := uuid.RandomUUID()
|
||||
process1 := db.NewProcess(&process_id1)
|
||||
process1.SetHost("----192.1.1.0----")
|
||||
fragment1.SetProcess(process1)
|
||||
|
||||
slice2 := database.GetOrCreateSlice(1)
|
||||
fragment_id2 := util.Id()
|
||||
fragment2 := database.GetOrCreateFragment(frame, slice2, fragment_id2)
|
||||
process_id2 := uuid.RandomUUID()
|
||||
process2 := db.NewProcess(&process_id2)
|
||||
process2.SetHost("----192.1.1.1----")
|
||||
fragment2.SetProcess(process2)
|
||||
|
||||
qplanner := QueryPlanner{Database: database}
|
||||
qplanner := QueryPlanner{Database: database, Query: &query}
|
||||
destination := fragment1.GetLocation()
|
||||
|
||||
id := uuid.RandomUUID()
|
||||
qp := qplanner.Plan(&query, &id, destination)
|
||||
qp := *qplanner.Plan(&query, &id, destination)
|
||||
|
||||
for i, qs := range *qp {
|
||||
//spew.Dump(i, qs, qs.inputs)
|
||||
spew.Dump(i, qs)
|
||||
spew.Dump("**************************************************************")
|
||||
}
|
||||
So(len(qp), ShouldEqual, 7)
|
||||
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
|
||||
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0)
|
||||
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
|
||||
So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union")
|
||||
So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[0].(GetQueryStep).Id,
|
||||
qp[1].(GetQueryStep).Id,
|
||||
})
|
||||
So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1)
|
||||
So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1)
|
||||
So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
|
||||
So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union")
|
||||
So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[3].(GetQueryStep).Id,
|
||||
qp[4].(GetQueryStep).Id,
|
||||
})
|
||||
So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat")
|
||||
So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[2].(UnionQueryStep).Id,
|
||||
qp[5].(UnionQueryStep).Id,
|
||||
})
|
||||
})
|
||||
Convey("Get query plan - including parsing", t, func() {
|
||||
|
||||
query := QueryForPQL("get(10,general)")
|
||||
|
||||
database, fragment1 := basic_database()
|
||||
|
||||
qplanner := QueryPlanner{Database: database, Query: query}
|
||||
destination := fragment1.GetLocation()
|
||||
|
||||
id := uuid.RandomUUID()
|
||||
qp := *qplanner.Plan(query, &id, destination)
|
||||
|
||||
So(len(qp), ShouldEqual, 3)
|
||||
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
|
||||
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 1)
|
||||
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[2].(CatQueryStep).Operation, ShouldEqual, "cat")
|
||||
So(qp[2].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[0].(GetQueryStep).Id,
|
||||
qp[1].(GetQueryStep).Id,
|
||||
})
|
||||
})
|
||||
Convey("Union query plan - including parsing", t, func() {
|
||||
|
||||
query := QueryForPQL("union(get(10, general), get(20, general))")
|
||||
|
||||
database, fragment1 := basic_database()
|
||||
|
||||
qplanner := QueryPlanner{Database: database, Query: query}
|
||||
destination := fragment1.GetLocation()
|
||||
|
||||
id := uuid.RandomUUID()
|
||||
qp := *qplanner.Plan(query, &id, destination)
|
||||
|
||||
So(len(qp), ShouldEqual, 7)
|
||||
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[0].(GetQueryStep).Slice, ShouldEqual, 0)
|
||||
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[1].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[1].(GetQueryStep).Slice, ShouldEqual, 0)
|
||||
So(*(qp[1].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
|
||||
So(qp[2].(UnionQueryStep).Operation, ShouldEqual, "union")
|
||||
So(qp[2].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[0].(GetQueryStep).Id,
|
||||
qp[1].(GetQueryStep).Id,
|
||||
})
|
||||
So(qp[3].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[3].(GetQueryStep).Slice, ShouldEqual, 1)
|
||||
So(*(qp[3].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[4].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(qp[4].(GetQueryStep).Slice, ShouldEqual, 1)
|
||||
So(*(qp[4].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{20, "general"})
|
||||
So(qp[5].(UnionQueryStep).Operation, ShouldEqual, "union")
|
||||
So(qp[5].(UnionQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[3].(GetQueryStep).Id,
|
||||
qp[4].(GetQueryStep).Id,
|
||||
})
|
||||
So(qp[6].(CatQueryStep).Operation, ShouldEqual, "cat")
|
||||
So(qp[6].(CatQueryStep).Inputs, ShouldResemble, []*uuid.UUID{
|
||||
qp[2].(UnionQueryStep).Id,
|
||||
qp[5].(UnionQueryStep).Id,
|
||||
})
|
||||
})
|
||||
Convey("Set query plan - including parsing", t, func() {
|
||||
query := QueryForPQL("set(10, general, 100)")
|
||||
|
||||
database, fragment1 := basic_database()
|
||||
|
||||
qplanner := QueryPlanner{Database: database, Query: query}
|
||||
destination := fragment1.GetLocation()
|
||||
|
||||
id := uuid.RandomUUID()
|
||||
qp := *qplanner.Plan(query, &id, destination)
|
||||
So(len(qp), ShouldEqual, 1)
|
||||
So(qp[0].(SetQueryStep).Operation, ShouldEqual, "set")
|
||||
So(qp[0].(SetQueryStep).ProfileId, ShouldEqual, 100)
|
||||
So(*(qp[0].(SetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
})
|
||||
Convey("Top-n query plan - including parsing", t, func() {
|
||||
query := QueryForPQL("top-n(get(10, general), [1,2,3], 50)")
|
||||
|
||||
database, fragment1 := basic_database()
|
||||
|
||||
qplanner := QueryPlanner{Database: database, Query: query}
|
||||
destination := fragment1.GetLocation()
|
||||
|
||||
id := uuid.RandomUUID()
|
||||
qp := *qplanner.Plan(query, &id, destination)
|
||||
So(len(qp), ShouldEqual, 5)
|
||||
So(qp[0].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(*(qp[0].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[1].(*TopNQueryStep).Operation, ShouldEqual, "top-n")
|
||||
So(qp[1].(*TopNQueryStep).Input, ShouldEqual, qp[0].(GetQueryStep).Id)
|
||||
So(qp[1].(*TopNQueryStep).N, ShouldEqual, 50)
|
||||
So(qp[2].(GetQueryStep).Operation, ShouldEqual, "get")
|
||||
So(*(qp[2].(GetQueryStep).Bitmap), ShouldResemble, db.Bitmap{10, "general"})
|
||||
So(qp[3].(*TopNQueryStep).Operation, ShouldEqual, "top-n")
|
||||
So(qp[3].(*TopNQueryStep).Input, ShouldEqual, qp[2].(GetQueryStep).Id)
|
||||
So(qp[3].(*TopNQueryStep).N, ShouldEqual, 50)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,28 +22,30 @@ type PqlListItem struct {
|
|||
}
|
||||
|
||||
type Query struct {
|
||||
Id *uuid.UUID
|
||||
Operation string
|
||||
Inputs []QueryInput //"strconv"
|
||||
// Represents a parsed query. Inputs can be Query or Bitmap objects
|
||||
// Maybe Bitmap and Query objects should have different fields to avoid using interface{}
|
||||
ProfileId uint64 // used only for set() queries
|
||||
N int // TODO: I think we should make this a generic map for any attributes related to the query
|
||||
Id *uuid.UUID
|
||||
Operation string
|
||||
Args map[string]interface{}
|
||||
Subqueries []Query
|
||||
}
|
||||
|
||||
func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan {
|
||||
tokens := Lex(pql)
|
||||
tokens, err := Lex(pql)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return QueryPlanForTokens(database, tokens, destination)
|
||||
}
|
||||
|
||||
func QueryForPQL(pql string) *Query {
|
||||
tokens := Lex(pql)
|
||||
tokens, err := Lex(pql)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return QueryForTokens(tokens)
|
||||
}
|
||||
|
||||
func QueryForTokens(tokens []Token) *Query {
|
||||
query_parser := QueryParser{}
|
||||
query, err := query_parser.Parse(tokens)
|
||||
query, err := Parse(tokens)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
@ -56,7 +58,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L
|
|||
}
|
||||
|
||||
func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) *QueryPlan {
|
||||
query_planner := QueryPlanner{Database: database}
|
||||
query_planner := QueryPlanner{Database: database, Query: query}
|
||||
id := uuid.RandomUUID()
|
||||
query_plan := query_planner.Plan(query, &id, destination)
|
||||
return query_plan
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue