diff --git a/executor/executor.go b/executor/executor.go index d6080971d..f6e92daf6 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -137,8 +137,8 @@ func (self *Executor) RunPQL(database_name string, pql string) (interface{}, err } else { //want to refactor this down to just RunPlugin(tokens) plugins_dir := config.GetString("plugins") plugins_file := plugins_dir + "/" + outer_token + ".js" - filter := query.TokensToString(tokens) - query_list := GetMacro(plugins_file, filter).(query.PqlList) + filter, filters := query.TokensToFilterStrings(tokens) + query_list := GetPlugin(plugins_file, filter, filters).(query.PqlList) for i, _ := range query_list { qry, err := query.QueryForPQL(query_list[i].PQL) diff --git a/executor/utils.go b/executor/utils.go index 88290db30..a1fdf0af0 100644 --- a/executor/utils.go +++ b/executor/utils.go @@ -1,14 +1,16 @@ package executor import ( + "bytes" "io/ioutil" "pilosa/query" + "strings" "github.com/davecgh/go-spew/spew" "github.com/robertkrimen/otto" ) -func GetMacro(file_name string, filter string) interface{} { +func GetPlugin(file_name string, filter string, filters []string) interface{} { file_data, err := ioutil.ReadFile(file_name) if err != nil { @@ -16,7 +18,17 @@ func GetMacro(file_name string, filter string) interface{} { } s := string(file_data[:]) - js := "query_list = (" + s + ")('" + filter + "');" + // convert the list of filters in to a string array + var buffer bytes.Buffer + if len(filters) > 0 { + buffer.WriteString("['") + buffer.WriteString(strings.Join(filters, "','")) + buffer.WriteString("']") + } else { + buffer.WriteString("[]") + } + + js := "query_list = (" + s + ")('" + filter + "', " + buffer.String() + ");" Otto := otto.New() Otto.Run(js) diff --git a/query/lexer.go b/query/lexer.go index 6c2ac09c4..c84c90e7a 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "strings" + "unicode" "unicode/utf8" ) @@ -148,25 +149,49 @@ func stateLB(lexer *Lexer) statefn { lexer.acceptUntil("[", true) lexer.next() lexer.emit(TYPE_LB) - for { - r, err := lexer.acceptUntil(",]", true) - if err != nil { - return stateError(errors.New("Unclosed bracket!")) - } - lexer.emit(TYPE_VALUE) - if r == ',' { - lexer.next() - lexer.emit(TYPE_COMMA) - } else { - lexer.next() - lexer.emit(TYPE_RB) - return stateArgs + + peeked := lexer.peek() + if peeked == rune(']') { + lexer.next() + lexer.emit(TYPE_RB) + return stateArgs + } else if unicode.IsDigit(peeked) { + for { + r, err := lexer.acceptUntil(",]", true) + if err != nil { + return stateError(errors.New("Unclosed bracket!")) + } + lexer.emit(TYPE_VALUE) + if r == ',' { + lexer.next() + lexer.emit(TYPE_COMMA) + } else { + lexer.next() + lexer.emit(TYPE_RB) + return stateArgs + } } + } else { + return stateArgs + } +} + +func stateRB(lexer *Lexer) statefn { + lexer.pos += 1 + lexer.emit(TYPE_RB) + + peeked := lexer.peek() + if peeked == rune(',') { + return stateRPComma + } else if peeked == rune(')') { + return stateRP + } else { + return stateEOF } } func stateArgs(lexer *Lexer) statefn { - r, err := lexer.acceptUntil("(),=[", false) + r, err := lexer.acceptUntil("(),=[]", false) if err != nil { return stateError(err) } @@ -181,6 +206,8 @@ func stateArgs(lexer *Lexer) statefn { return stateKeyword case '[': return stateLB + case ']': + return stateRB default: return stateError(errors.New("Expecting arguments!")) } @@ -240,6 +267,8 @@ func stateRP(lexer *Lexer) statefn { return stateRPComma } else if peeked == rune(')') { return stateRP + } else if peeked == rune(']') { + return stateRB } else { return stateEOF } diff --git a/query/lexer_test.go b/query/lexer_test.go index 47aa172d6..2c0118850 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -186,5 +186,27 @@ func TestLexer(t *testing.T) { Token{"]", TYPE_RB}, Token{")", TYPE_RP}, }) + + tokens, err = Lex("plugin(get(10, general), [get(11, general)])") + So(tokens, ShouldResemble, []Token{ + Token{"plugin", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"10", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"general", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{",", TYPE_COMMA}, + Token{"[", TYPE_LB}, + Token{"get", TYPE_FUNC}, + Token{"(", TYPE_LP}, + Token{"11", TYPE_VALUE}, + Token{",", TYPE_COMMA}, + Token{"general", TYPE_VALUE}, + Token{")", TYPE_RP}, + Token{"]", TYPE_RB}, + Token{")", TYPE_RP}, + }) }) } diff --git a/query/parser.go b/query/parser.go index b69bd3c99..b6a3dac22 100644 --- a/query/parser.go +++ b/query/parser.go @@ -249,29 +249,60 @@ ArgLoop: } query.Args[keyword] = value case TYPE_LB: - query.Args["ids"] = make([]uint64, 0) - for { - token = self.next() - if token == nil { - return nil, fmt.Errorf("Unclosed list!") - } - switch token.Type { - case TYPE_COMMA: - break - case TYPE_VALUE: - i, err := strconv.ParseUint(token.Text, 10, 64) - if err != nil { - return nil, fmt.Errorf("Expecting integer id! (%v)", err) + peeked := self.peek() + // we currently support 2 types of values in square brackets: + // ids <- list of integers (TYPE_VALUE) + // filters <- list of queries (TYPE_FUNC) + switch peeked.Type { + case TYPE_VALUE: + query.Args["ids"] = make([]uint64, 0) + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed list!") + } + switch token.Type { + case TYPE_COMMA: + break + case TYPE_VALUE: + i, err := strconv.ParseUint(token.Text, 10, 64) + if err != nil { + return nil, fmt.Errorf("Expecting integer id! (%v)", err) + } + query.Args["ids"] = append(query.Args["ids"].([]uint64), i) + case TYPE_RB: + continue ArgLoop + default: + return nil, fmt.Errorf("Unexpected token! (%v)", token) + } + } + case TYPE_FUNC: + query.Args["filters"] = make([]Query, 0) + for { + token = self.next() + if token == nil { + return nil, fmt.Errorf("Unclosed list!") + } + switch token.Type { + case TYPE_COMMA: + break + case TYPE_FUNC: + self.backup() + filterquery, err := self.Parse() + if err != nil { + return nil, err + } + query.Args["filters"] = append(query.Args["filters"].([]Query), *filterquery) + case TYPE_RB: + continue ArgLoop + default: + return nil, fmt.Errorf("Unexpected token! (%v)", token) } - query.Args["ids"] = append(query.Args["ids"].([]uint64), i) - case TYPE_RB: - continue ArgLoop - default: - return nil, fmt.Errorf("Unexpected token! (%v)", token) } } + case TYPE_RB: + // default: - log.Println(spew.Sdump("unexpected", token)) return nil, errors.New("BAD TOKEN") } diff --git a/query/parser_test.go b/query/parser_test.go index 7b3c23872..dfa48d53f 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -79,6 +79,18 @@ func TestQueryParser(t *testing.T) { So(query.Subqueries[0].Operation, ShouldEqual, "get") So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"}) }) + Convey("Bracketed Lists", t, func() { + tokens, err := Lex("plugin(get(99), [get(10), get(11)])") + //tokens, err := Lex("plugin(get(99), [1,3])") + So(err, ShouldBeNil) + + query, err := Parse(tokens) + So(err, ShouldBeNil) + + spew.Dump("*********************************************") + spew.Dump(query) + spew.Dump("*********************************************") + }) Convey("Lists", t, func() { tokens, err := Lex("wat(50)") _, err = Parse(tokens) diff --git a/query/query.go b/query/query.go index c3ba650c1..8d9971b65 100644 --- a/query/query.go +++ b/query/query.go @@ -69,12 +69,35 @@ func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Loca return query_plan, nil } -func TokensToString(tokens []Token) string { - var str []string +func TokensToFilterStrings(tokens []Token) (string, []string) { + var whole []string + var filter string + var filters []string + var open_parens int + var in_square_brackets bool + var last_slice = 0 + + open_parens = -1 + in_square_brackets = false for i, _ := range tokens { - str = append(str, tokens[i].Text) + whole = append(whole, tokens[i].Text) + if tokens[i].Type == TYPE_FUNC { + last_slice = i + } else if tokens[i].Type == TYPE_LP { + open_parens += 1 + } else if tokens[i].Type == TYPE_RP { + open_parens -= 1 + if open_parens == 0 { + if !in_square_brackets { + filter = strings.Join(whole[2:], "") + } else { + filters = append(filters, strings.Join(whole[last_slice:], "")) + } + last_slice = i + } + } else if open_parens == 0 && tokens[i].Type == TYPE_LB { + in_square_brackets = true + } } - // for now, we're just using this function to pull the filter out of the outer function "outerfunc(filter)" - str = str[2 : len(str)-1] - return strings.Join(str, "") + return filter, filters } diff --git a/query/query_test.go b/query/query_test.go new file mode 100644 index 000000000..38eb53caa --- /dev/null +++ b/query/query_test.go @@ -0,0 +1,37 @@ +package query + +import ( + "testing" + . "github.com/smartystreets/goconvey/convey" +) + +func TestQuery(t *testing.T) { + Convey("Bracketed Lists 1", t, func() { + tokens, err := Lex("plugin(get(88, general), [get(12, general), get(13, general)])") + So(err, ShouldBeNil) + filter, filters := TokensToFilterStrings(tokens) + So(filter, ShouldEqual, "get(88,general)") + So(filters, ShouldResemble, []string{"get(12,general)", "get(13,general)"}) + /* + spew.Dump("*********************************************") + spew.Dump(filter) + spew.Dump(filters) + spew.Dump("*********************************************") + */ + }) + Convey("Bracketed Lists 2", t, func() { + tokens, err := Lex("plugin(intersect(get(88, general, [0]), get(77, b.n)), [get(12, general), get(13, general)])") + So(err, ShouldBeNil) + filter, filters := TokensToFilterStrings(tokens) + So(filter, ShouldEqual, "intersect(get(88,general,[0]),get(77,b.n))") + So(filters, ShouldResemble, []string{"get(12,general)", "get(13,general)"}) + }) + Convey("Bracketed Lists 3", t, func() { + tokens, err := Lex("plugin(intersect(get(88, general, [0]), get(77, b.n)))") + So(err, ShouldBeNil) + filter, filters := TokensToFilterStrings(tokens) + So(filter, ShouldEqual, "intersect(get(88,general,[0]),get(77,b.n))") + var empty []string + So(filters, ShouldResemble, empty) + }) +}