handle a square-bracketed list of queries (for plugins)

executor/executor.go

@@ -137,8 +137,8 @@ func (self *Executor) RunPQL(database_name string, pql string) (interface{}, err
 	} else { //want to refactor this down to just RunPlugin(tokens)
 		plugins_dir := config.GetString("plugins")
 		plugins_file := plugins_dir + "/" + outer_token + ".js"
-		filter := query.TokensToString(tokens)
-		query_list := GetMacro(plugins_file, filter).(query.PqlList)
+		filter, filters := query.TokensToFilterStrings(tokens)
+		query_list := GetPlugin(plugins_file, filter, filters).(query.PqlList)
 
 		for i, _ := range query_list {
 			qry, err := query.QueryForPQL(query_list[i].PQL)

executor/utils.go

@@ -1,14 +1,16 @@
 package executor
 
 import (
+	"bytes"
 	"io/ioutil"
 	"pilosa/query"
+	"strings"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/robertkrimen/otto"
 )
 
-func GetMacro(file_name string, filter string) interface{} {
+func GetPlugin(file_name string, filter string, filters []string) interface{} {
 
 	file_data, err := ioutil.ReadFile(file_name)
 	if err != nil {
@@ -16,7 +18,17 @@ func GetMacro(file_name string, filter string) interface{} {
 	}
 	s := string(file_data[:])
 
-	js := "query_list = (" + s + ")('" + filter + "');"
+	// convert the list of filters in to a string array
+	var buffer bytes.Buffer
+	if len(filters) > 0 {
+		buffer.WriteString("['")
+		buffer.WriteString(strings.Join(filters, "','"))
+		buffer.WriteString("']")
+	} else {
+		buffer.WriteString("[]")
+	}
+
+	js := "query_list = (" + s + ")('" + filter + "', " + buffer.String() + ");"
 
 	Otto := otto.New()
 	Otto.Run(js)

query/lexer.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 )
 
@@ -148,25 +149,49 @@ func stateLB(lexer *Lexer) statefn {
 	lexer.acceptUntil("[", true)
 	lexer.next()
 	lexer.emit(TYPE_LB)
-	for {
-		r, err := lexer.acceptUntil(",]", true)
-		if err != nil {
-			return stateError(errors.New("Unclosed bracket!"))
-		}
-		lexer.emit(TYPE_VALUE)
-		if r == ',' {
-			lexer.next()
-			lexer.emit(TYPE_COMMA)
-		} else {
-			lexer.next()
-			lexer.emit(TYPE_RB)
-			return stateArgs
+
+	peeked := lexer.peek()
+	if peeked == rune(']') {
+		lexer.next()
+		lexer.emit(TYPE_RB)
+		return stateArgs
+	} else if unicode.IsDigit(peeked) {
+		for {
+			r, err := lexer.acceptUntil(",]", true)
+			if err != nil {
+				return stateError(errors.New("Unclosed bracket!"))
+			}
+			lexer.emit(TYPE_VALUE)
+			if r == ',' {
+				lexer.next()
+				lexer.emit(TYPE_COMMA)
+			} else {
+				lexer.next()
+				lexer.emit(TYPE_RB)
+				return stateArgs
+			}
 		}
+	} else {
+		return stateArgs
+	}
+}
+
+func stateRB(lexer *Lexer) statefn {
+	lexer.pos += 1
+	lexer.emit(TYPE_RB)
+
+	peeked := lexer.peek()
+	if peeked == rune(',') {
+		return stateRPComma
+	} else if peeked == rune(')') {
+		return stateRP
+	} else {
+		return stateEOF
 	}
 }
 
 func stateArgs(lexer *Lexer) statefn {
-	r, err := lexer.acceptUntil("(),=[", false)
+	r, err := lexer.acceptUntil("(),=[]", false)
 	if err != nil {
 		return stateError(err)
 	}
@@ -181,6 +206,8 @@ func stateArgs(lexer *Lexer) statefn {
 		return stateKeyword
 	case '[':
 		return stateLB
+	case ']':
+		return stateRB
 	default:
 		return stateError(errors.New("Expecting arguments!"))
 	}
@@ -240,6 +267,8 @@ func stateRP(lexer *Lexer) statefn {
 		return stateRPComma
 	} else if peeked == rune(')') {
 		return stateRP
+	} else if peeked == rune(']') {
+		return stateRB
 	} else {
 		return stateEOF
 	}

query/lexer_test.go

@@ -186,5 +186,27 @@ func TestLexer(t *testing.T) {
 			Token{"]", TYPE_RB},
 			Token{")", TYPE_RP},
 		})
+
+		tokens, err = Lex("plugin(get(10, general), [get(11, general)])")
+		So(tokens, ShouldResemble, []Token{
+			Token{"plugin", TYPE_FUNC},
+			Token{"(", TYPE_LP},
+			Token{"get", TYPE_FUNC},
+			Token{"(", TYPE_LP},
+			Token{"10", TYPE_VALUE},
+			Token{",", TYPE_COMMA},
+			Token{"general", TYPE_VALUE},
+			Token{")", TYPE_RP},
+			Token{",", TYPE_COMMA},
+			Token{"[", TYPE_LB},
+			Token{"get", TYPE_FUNC},
+			Token{"(", TYPE_LP},
+			Token{"11", TYPE_VALUE},
+			Token{",", TYPE_COMMA},
+			Token{"general", TYPE_VALUE},
+			Token{")", TYPE_RP},
+			Token{"]", TYPE_RB},
+			Token{")", TYPE_RP},
+		})
 	})
 }

query/parser.go

@@ -249,29 +249,60 @@ ArgLoop:
 			}
 			query.Args[keyword] = value
 		case TYPE_LB:
-			query.Args["ids"] = make([]uint64, 0)
-			for {
-				token = self.next()
-				if token == nil {
-					return nil, fmt.Errorf("Unclosed list!")
-				}
-				switch token.Type {
-				case TYPE_COMMA:
-					break
-				case TYPE_VALUE:
-					i, err := strconv.ParseUint(token.Text, 10, 64)
-					if err != nil {
-						return nil, fmt.Errorf("Expecting integer id! (%v)", err)
+			peeked := self.peek()
+			// we currently support 2 types of values in square brackets:
+			//    ids      <- list of integers (TYPE_VALUE)
+			//    filters  <- list of queries (TYPE_FUNC)
+			switch peeked.Type {
+			case TYPE_VALUE:
+				query.Args["ids"] = make([]uint64, 0)
+				for {
+					token = self.next()
+					if token == nil {
+						return nil, fmt.Errorf("Unclosed list!")
+					}
+					switch token.Type {
+					case TYPE_COMMA:
+						break
+					case TYPE_VALUE:
+						i, err := strconv.ParseUint(token.Text, 10, 64)
+						if err != nil {
+							return nil, fmt.Errorf("Expecting integer id! (%v)", err)
+						}
+						query.Args["ids"] = append(query.Args["ids"].([]uint64), i)
+					case TYPE_RB:
+						continue ArgLoop
+					default:
+						return nil, fmt.Errorf("Unexpected token! (%v)", token)
+					}
+				}
+			case TYPE_FUNC:
+				query.Args["filters"] = make([]Query, 0)
+				for {
+					token = self.next()
+					if token == nil {
+						return nil, fmt.Errorf("Unclosed list!")
+					}
+					switch token.Type {
+					case TYPE_COMMA:
+						break
+					case TYPE_FUNC:
+						self.backup()
+						filterquery, err := self.Parse()
+						if err != nil {
+							return nil, err
+						}
+						query.Args["filters"] = append(query.Args["filters"].([]Query), *filterquery)
+					case TYPE_RB:
+						continue ArgLoop
+					default:
+						return nil, fmt.Errorf("Unexpected token! (%v)", token)
 					}
-					query.Args["ids"] = append(query.Args["ids"].([]uint64), i)
-				case TYPE_RB:
-					continue ArgLoop
-				default:
-					return nil, fmt.Errorf("Unexpected token! (%v)", token)
 				}
 			}
+		case TYPE_RB:
+			//
 		default:
-
 			log.Println(spew.Sdump("unexpected", token))
 			return nil, errors.New("BAD TOKEN")
 		}

query/parser_test.go

@@ -79,6 +79,18 @@ func TestQueryParser(t *testing.T) {
 		So(query.Subqueries[0].Operation, ShouldEqual, "get")
 		So(query.Subqueries[0].Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general"})
 	})
+	Convey("Bracketed Lists", t, func() {
+		tokens, err := Lex("plugin(get(99), [get(10), get(11)])")
+		//tokens, err := Lex("plugin(get(99), [1,3])")
+		So(err, ShouldBeNil)
+
+		query, err := Parse(tokens)
+		So(err, ShouldBeNil)
+
+		spew.Dump("*********************************************")
+		spew.Dump(query)
+		spew.Dump("*********************************************")
+	})
 	Convey("Lists", t, func() {
 		tokens, err := Lex("wat(50)")
 		_, err = Parse(tokens)

query/query.go

@@ -69,12 +69,35 @@ func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Loca
 	return query_plan, nil
 }
 
-func TokensToString(tokens []Token) string {
-	var str []string
+func TokensToFilterStrings(tokens []Token) (string, []string) {
+	var whole []string
+	var filter string
+	var filters []string
+	var open_parens int
+	var in_square_brackets bool
+	var last_slice = 0
+
+	open_parens = -1
+	in_square_brackets = false
 	for i, _ := range tokens {
-		str = append(str, tokens[i].Text)
+		whole = append(whole, tokens[i].Text)
+		if tokens[i].Type == TYPE_FUNC {
+			last_slice = i
+		} else if tokens[i].Type == TYPE_LP {
+			open_parens += 1
+		} else if tokens[i].Type == TYPE_RP {
+			open_parens -= 1
+			if open_parens == 0 {
+				if !in_square_brackets {
+					filter = strings.Join(whole[2:], "")
+				} else {
+					filters = append(filters, strings.Join(whole[last_slice:], ""))
+				}
+				last_slice = i
+			}
+		} else if open_parens == 0 && tokens[i].Type == TYPE_LB {
+			in_square_brackets = true
+		}
 	}
-	// for now, we're just using this function to pull the filter out of the outer function "outerfunc(filter)"
-	str = str[2 : len(str)-1]
-	return strings.Join(str, "")
+	return filter, filters
 }

query/query_test.go

@@ -0,0 +1,37 @@
+package query
+
+import (
+	"testing"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestQuery(t *testing.T) {
+	Convey("Bracketed Lists 1", t, func() {
+		tokens, err := Lex("plugin(get(88, general), [get(12, general), get(13, general)])")
+		So(err, ShouldBeNil)
+		filter, filters := TokensToFilterStrings(tokens)
+		So(filter, ShouldEqual, "get(88,general)")
+		So(filters, ShouldResemble, []string{"get(12,general)", "get(13,general)"})
+		/*
+			spew.Dump("*********************************************")
+			spew.Dump(filter)
+			spew.Dump(filters)
+			spew.Dump("*********************************************")
+		*/
+	})
+	Convey("Bracketed Lists 2", t, func() {
+		tokens, err := Lex("plugin(intersect(get(88, general, [0]), get(77, b.n)), [get(12, general), get(13, general)])")
+		So(err, ShouldBeNil)
+		filter, filters := TokensToFilterStrings(tokens)
+		So(filter, ShouldEqual, "intersect(get(88,general,[0]),get(77,b.n))")
+		So(filters, ShouldResemble, []string{"get(12,general)", "get(13,general)"})
+	})
+	Convey("Bracketed Lists 3", t, func() {
+		tokens, err := Lex("plugin(intersect(get(88, general, [0]), get(77, b.n)))")
+		So(err, ShouldBeNil)
+		filter, filters := TokensToFilterStrings(tokens)
+		So(filter, ShouldEqual, "intersect(get(88,general,[0]),get(77,b.n))")
+		var empty []string
+		So(filters, ShouldResemble, empty)
+	})
+}