preparation for macro support. clean up the RunQuery logic

core/http.go

@@ -68,7 +68,14 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
 	// TODO: we need to get the database name from the query string (for now, hard-coded)
 	database_name := "main"
 	pql := string(body)
-	self.service.Executor.RunQuery(database_name, pql)
+	results := self.service.Executor.RunPQL(database_name, pql)
+
+	encoder := json.NewEncoder(w)
+	err = encoder.Encode(results)
+	if err != nil {
+		log.Fatal("Error encoding stats")
+	}
+
 }
 
 func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {

core/query.go

@@ -160,5 +160,10 @@ func (self *Service) GetQueryStepHandler(msg *db.Message) {
 func (self *Service) SetQueryStepHandler(msg *db.Message) {
 	spew.Dump("SET QUERYSTEP")
 	qs := msg.Data.(query.SetQueryStep)
-	self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
+	result, err := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
+	spew.Dump("result:", result)
+	spew.Dump("err:", err)
+
+	result_message := db.Message{Data: query.SetQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: result}}}
+	self.Transport.Send(&result_message, qs.Destination.ProcessId)
 }

dispatch/dispatch.go

@@ -5,6 +5,8 @@ import (
 	"pilosa/core"
 	"pilosa/db"
 	"pilosa/query"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
 type Dispatch struct {
@@ -29,6 +31,7 @@ func (self *Dispatch) Run() {
 			pong := db.Message{Data: core.PongRequest{Id: data.Id}}
 			self.service.Transport.Send(&pong, data.Source)
 		case db.HoldResult:
+			spew.Dump("HOLD-SET", data.ResultId())
 			self.service.Hold.Set(data.ResultId(), data.ResultData(), 30)
 		case query.PortableQueryStep:
 			go self.service.Executor.NewJob(message)

executor/executor.go

@@ -2,12 +2,13 @@ package executor
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
+	"pilosa/config"
 	"pilosa/core"
 	"pilosa/db"
 	"pilosa/query"
 	"pilosa/util"
-	"reflect"
 	"tux21b.org/v1/gocql/uuid"
 
 	"github.com/davecgh/go-spew/spew"
@@ -46,8 +47,35 @@ func (self *Executor) NewJob(job *db.Message) {
 	}
 }
 
-func (self *Executor) RunQuery(database_name string, pql string) {
-	database := self.service.Cluster.GetOrCreateDatabase(database_name)
+type stringSlice []string
+
+func (slice stringSlice) pos(value string) int {
+	for p, v := range slice {
+		if v == value {
+			return p
+		}
+	}
+	return -1
+}
+
+func (self *Executor) RunQueryTest(database_name string, pql string) string {
+	return pql
+}
+
+type queryItem struct {
+	label string
+	pql   string
+}
+type queryItemCopy struct {
+	id    *uuid.UUID
+	label string
+}
+type QueryItemResult struct {
+	Label  string
+	Result interface{}
+}
+
+func (self *Executor) runQuery(database *db.Database, qry *query.Query) {
 	process, err := self.service.GetProcess()
 	if err != nil {
 		spew.Dump(err)
@@ -56,33 +84,78 @@ func (self *Executor) RunQuery(database_name string, pql string) {
 	fragment_id := util.SUUID(0)
 	destination := db.Location{&process_id, fragment_id}
 
-	query_plan := query.QueryPlanForPQL(database, pql, &destination)
-	//spew.Dump(query_plan)
-
+	spew.Dump("QUERY.ID:", qry.Id)
+	query_plan := query.QueryPlanForQuery(database, qry, &destination)
 	// loop over the query steps and send to Transport
-	var last_id *uuid.UUID
 	for _, qs := range *query_plan {
 		msg := new(db.Message)
 		msg.Data = qs
 		switch step := qs.(type) {
 		case query.PortableQueryStep:
 			self.service.Transport.Send(msg, step.GetLocation().ProcessId)
-			if reflect.TypeOf(step) != reflect.TypeOf(query.SetQueryStep{}) {
-				last_id = step.GetId()
-			}
 		}
 	}
+}
 
-	// add an entry to my execute map[key] that is waiting for the final result
-	if last_id != nil {
-		final, err := self.service.Hold.Get(last_id, 10)
+func (self *Executor) RunPQL(database_name string, pql string) interface{} {
+	database := self.service.Cluster.GetOrCreateDatabase(database_name)
+
+	// see if the outer query function is a custom query
+	reserved_functions := stringSlice{"get", "set", "union", "intersect", "count"}
+	tokens := query.Lex(pql)
+	outer_token := tokens[0].Text
+
+	if reserved_functions.pos(outer_token) != -1 {
+
+		qry := query.QueryForTokens(tokens)
+		go self.runQuery(database, qry)
+
+		var final interface{}
+		final, err := self.service.Hold.Get(qry.Id, 10)
 		if err != nil {
 			spew.Dump(err)
 		}
-		spew.Dump("*******************************************************")
-		spew.Dump("FINAL", final)
-		spew.Dump("*******************************************************")
+		return final
+
+	} else {
+		macros_dir := config.Get("macros").(string)
+		macros_file := macros_dir + "/" + outer_token + ".js"
+		file_data, err := ioutil.ReadFile(macros_file)
+		if err != nil {
+			spew.Dump(err)
+		}
+		spew.Dump(file_data)
+
+		// CUSTOM QUERY LIST ///////////////
+		var query_list []queryItem
+		query_list = append(query_list, queryItem{"set1", "set(20, 1)"})
+		query_list = append(query_list, queryItem{"set2", "set(20, 1)"})
+		query_list = append(query_list, queryItem{"set3", "set(20, 2)"})
+		query_list = append(query_list, queryItem{"set4", "set(20, 3)"})
+		query_list = append(query_list, queryItem{"set5", "set(20, 4)"})
+		query_list = append(query_list, queryItem{"count", "count(get(20))"})
+		var query_list_copy []*queryItemCopy
+		////////////////////////////////////
+
+		for _, qi := range query_list {
+			qry := query.QueryForPQL(qi.pql)
+			go self.runQuery(database, qry)
+			query_list_copy = append(query_list_copy, &queryItemCopy{qry.Id, qi.label})
+		}
+
+		var final_result []*QueryItemResult
+		for _, qlc := range query_list_copy {
+			final, err := self.service.Hold.Get(qlc.id, 10)
+			if err != nil {
+				spew.Dump(err)
+			}
+			spew.Dump(final)
+			final_result = append(final_result, &QueryItemResult{qlc.label, final})
+		}
+
+		return final_result
 	}
+
 }
 
 func (self *Executor) Run() {

interfaces/core.go

@@ -25,5 +25,5 @@ type Executorer interface {
 	Close()
 	Run()
 	NewJob(*db.Message)
-	RunQuery(string, string)
+	RunPQL(string, string) interface{}
 }

query/parser.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"pilosa/db"
 	"strconv"
+
+	"tux21b.org/v1/gocql/uuid"
 )
 
 var InvalidQueryError = errors.New("Invalid query format.")
@@ -75,6 +77,8 @@ func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
 	}
 
 	q := new(Query)
+	id := uuid.RandomUUID()
+	q.Id = &id
 	q.Operation = tokens[0].Text
 
 	// scan from open to close paren

query/planner.go

@@ -219,6 +219,10 @@ type SetQueryStep struct {
 	ProfileId uint64
 }
 
+type SetQueryResult struct {
+	*BaseQueryResult
+}
+
 // QueryTree for SET queries
 type SetQueryTree struct {
 	bitmap     *db.Bitmap
@@ -238,6 +242,7 @@ func (qt *SetQueryTree) getLocation(d *db.Database) *db.Location {
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
 func init() {
+	gob.Register(SetQueryResult{})
 	gob.Register(GetQueryResult{})
 	gob.Register(CatQueryResult{})
 	gob.Register(UnionQueryResult{})
@@ -401,5 +406,6 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
 // Transforms Query into QueryTree and flattens to QueryPlan object
 func (qp *QueryPlanner) Plan(query *Query, id *uuid.UUID, destination *db.Location) *QueryPlan {
 	queryTree := qp.buildTree(query, -1)
-	return qp.flatten(queryTree, id, destination)
+	//return qp.flatten(queryTree, id, destination) // TODO: remove the "id" parameter, since we are using the query.Id as the value
+	return qp.flatten(queryTree, query.Id, destination)
 }

query/planner_test.go

@@ -13,16 +13,19 @@ import (
 func TestQueryPlanner(t *testing.T) {
 	Convey("Basic query plan", t, func() {
 
+		id1 := uuid.RandomUUID()
 		bm1 := db.Bitmap{10, "general"}
 		inputs1 := []QueryInput{&bm1}
-		query1 := Query{"get", inputs1, 0}
+		query1 := Query{&id1, "get", inputs1, 0}
 
+		id2 := uuid.RandomUUID()
 		bm2 := db.Bitmap{20, "general"}
 		inputs2 := []QueryInput{&bm2}
-		query2 := Query{"get", inputs2, 0}
+		query2 := Query{&id2, "get", inputs2, 0}
 
+		id3 := uuid.RandomUUID()
 		inputs := []QueryInput{&query1, &query2}
-		query := Query{"union", inputs, 0}
+		query := Query{&id3, "union", inputs, 0}
 		/*
 
 			bm1 := db.Bitmap{10, "general"}

query/query.go

@@ -2,6 +2,7 @@ package query
 
 import (
 	"pilosa/db"
+
 	"tux21b.org/v1/gocql/uuid"
 )
 
@@ -12,6 +13,7 @@ type QueryResults struct {
 }
 
 type Query struct {
+	Id        *uuid.UUID
 	Operation string
 	Inputs    []QueryInput //"strconv"
 	// Represents a parsed query. Inputs can be Query or Bitmap objects
@@ -21,13 +23,41 @@ type Query struct {
 
 func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) *QueryPlan {
 	tokens := Lex(pql)
+	return QueryPlanForTokens(database, tokens, destination)
+}
+
+func QueryForPQL(pql string) *Query {
+	tokens := Lex(pql)
+	return QueryForTokens(tokens)
+}
+
+func QueryForTokens(tokens []Token) *Query {
 	query_parser := QueryParser{}
 	query, err := query_parser.Parse(tokens)
 	if err != nil {
 		panic(err)
 	}
+	return query
+}
+
+func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.Location) *QueryPlan {
+	query := QueryForTokens(tokens)
+	return QueryPlanForQuery(database, query, destination)
+	/*
+		//spew.Dump(query)
+		query_planner := QueryPlanner{Database: database}
+		id := uuid.RandomUUID()
+		query_plan := query_planner.Plan(query, &id, destination)
+		//spew.Dump(query_plan)
+		return query_plan
+	*/
+}
+
+func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) *QueryPlan {
+	//spew.Dump(query)
 	query_planner := QueryPlanner{Database: database}
 	id := uuid.RandomUUID()
 	query_plan := query_planner.Plan(query, &id, destination)
+	//spew.Dump(query_plan)
 	return query_plan
 }