scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-09 04:17:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add support for Set() queries to lexer/parser/planner

Browse source
This commit is contained in:
travisturner 2013-12-30 13:31:40 -06:00
parent 0ea1b17ea1
commit fc75d37300
7 changed files with 131 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/http.go
View file

@ -63,11 +63,9 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Error reading POST data", http.StatusBadRequest)
return
}
cluster := self.service.Cluster
database := cluster.GetOrCreateDatabase("main")
pql := string(body)
query.Execute(database, pql)
}

63
query/lexer.go
View file

@ -9,12 +9,13 @@ import (
)
const (
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_ID = iota
TYPE_FRAME = iota
TYPE_COMMA = iota
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_ID = iota
TYPE_FRAME = iota
TYPE_PROFILE = iota
TYPE_COMMA = iota
)
type Token struct {
@ -57,6 +58,15 @@ func (lexer *Lexer) acceptRun(valid string) {
lexer.backup()
}
func (lexer *Lexer) acceptNumber() {
digits := "0123456789"
lexer.acceptRun(digits)
}
func (lexer *Lexer) acceptText() {
digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")"
lexer.acceptRun(digits)
}
// next returns the next rune in the input.
func (lexer *Lexer) next() (runey rune) {
if lexer.pos >= len(lexer.text) {
@ -121,11 +131,8 @@ func stateArgs(lexer *Lexer) statefn {
}
func stateID(lexer *Lexer) statefn {
digits := "0123456789"
lexer.acceptRun(digits)
lexer.acceptNumber()
lexer.emit(TYPE_ID)
// if next is comma
peeked := lexer.peek()
if peeked == rune(',') {
@ -137,6 +144,14 @@ func stateID(lexer *Lexer) statefn {
}
}
func stateProfile(lexer *Lexer) statefn {
lexer.peek()
lexer.acceptNumber()
lexer.emit(TYPE_PROFILE)
lexer.peek()
return stateRP
}
func stateRP(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_RP)
@ -154,19 +169,27 @@ func stateRP(lexer *Lexer) statefn {
func stateFrameComma(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateFrame
return stateFrameOrProfile
}
func stateFrame(lexer *Lexer) statefn {
lexer.peek()
digits := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_." // TODO: make this more flexible. accept anything up to a space or RP: ")"
lexer.acceptRun(digits)
lexer.emit(TYPE_FRAME)
peeked := lexer.peek()
if peeked == rune(')') {
return stateRP
func stateProfileComma(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateProfile
}
func stateFrameOrProfile(lexer *Lexer) statefn {
if unicode.IsNumber(lexer.peek()) {
lexer.acceptNumber()
lexer.emit(TYPE_PROFILE)
lexer.peek()
} else {
lexer.acceptText()
lexer.emit(TYPE_FRAME)
if lexer.peek() == rune(',') {
return stateProfileComma
}
}
// should never get here
return stateRP
}

36
query/lexer_test.go
View file

@ -7,6 +7,7 @@ import (
func TestLexer(t *testing.T) {
Convey("Basic lexical analysis", t, func() {
tokens := Lex("get(10)")
So(len(tokens), ShouldEqual, 4)
So(tokens[0].Text, ShouldEqual, "get")
@ -126,5 +127,40 @@ func TestLexer(t *testing.T) {
So(tokens4[12].Type, ShouldEqual, TYPE_RP)
So(tokens4[13].Text, ShouldEqual, ")")
So(tokens4[13].Type, ShouldEqual, TYPE_RP)
tokens5 := Lex("set(1, 987)")
So(len(tokens5), ShouldEqual, 6)
So(tokens5[0].Text, ShouldEqual, "set")
So(tokens5[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens5[1].Text, ShouldEqual, "(")
So(tokens5[1].Type, ShouldEqual, TYPE_LP)
So(tokens5[2].Text, ShouldEqual, "1")
So(tokens5[2].Type, ShouldEqual, TYPE_ID)
So(tokens5[3].Text, ShouldEqual, ",")
So(tokens5[3].Type, ShouldEqual, TYPE_COMMA)
So(tokens5[4].Text, ShouldEqual, "987")
So(tokens5[4].Type, ShouldEqual, TYPE_PROFILE)
So(tokens5[5].Text, ShouldEqual, ")")
So(tokens5[5].Type, ShouldEqual, TYPE_RP)
tokens6 := Lex("set(1, general, 987)")
So(len(tokens6), ShouldEqual, 8)
So(tokens6[0].Text, ShouldEqual, "set")
So(tokens6[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens6[1].Text, ShouldEqual, "(")
So(tokens6[1].Type, ShouldEqual, TYPE_LP)
So(tokens6[2].Text, ShouldEqual, "1")
So(tokens6[2].Type, ShouldEqual, TYPE_ID)
So(tokens6[3].Text, ShouldEqual, ",")
So(tokens6[3].Type, ShouldEqual, TYPE_COMMA)
So(tokens6[4].Text, ShouldEqual, "general")
So(tokens6[4].Type, ShouldEqual, TYPE_FRAME)
So(tokens6[5].Text, ShouldEqual, ",")
So(tokens6[5].Type, ShouldEqual, TYPE_COMMA)
So(tokens6[6].Text, ShouldEqual, "987")
So(tokens6[6].Type, ShouldEqual, TYPE_PROFILE)
So(tokens6[7].Text, ShouldEqual, ")")
So(tokens6[7].Type, ShouldEqual, TYPE_RP)
})
}

17
query/parser.go
View file

@ -10,7 +10,7 @@ var InvalidQueryError = errors.New("Invalid query format.")
type QueryParser struct{}
func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput {
func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, int) {
// BITMAP
if tokens[0].Type == TYPE_ID {
// TODO: look for frame type in the tokens list
@ -20,11 +20,18 @@ func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput {
}
// if the next 2 tokens are comma-frame, then we have a frame, else set to a default
frame_type := "general"
if len(tokens) == 3 && tokens[2].Type == TYPE_FRAME {
if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME {
frame_type = tokens[2].Text
}
bm := db.Bitmap{bitmap_id, frame_type}
return []QueryInput{&bm}
profile_id := 0
if len(tokens) > 4 && tokens[4].Type == TYPE_PROFILE {
profile_id, err = strconv.Atoi(tokens[4].Text)
if err != nil {
panic(err)
}
}
return []QueryInput{&bm}, profile_id
}
// LIST OF QUERIES
@ -49,7 +56,7 @@ func (qp *QueryParser) walkInputs(tokens []Token) []QueryInput {
}
}
}
return qi
return qi, 0
}
func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
@ -73,7 +80,7 @@ func (qp *QueryParser) walk(tokens []Token) (*Query, error) {
} else if tokens[i].Type == TYPE_RP {
if open_parens == 0 {
if i == len(tokens)-1 {
q.Inputs = qp.walkInputs(tokens[2:i])
q.Inputs, q.Profile_id = qp.walkInputs(tokens[2:i])
}
} else {
open_parens--

32
query/planner.go
View file

@ -17,7 +17,7 @@ type QueryStep struct {
return_process *db.Process
}
func (q QueryStep) String() string {
func (q QueryStep) StringHOLD() string {
return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.return_process)
}
@ -43,7 +43,7 @@ type CompositeQueryTree struct {
func (qt *CompositeQueryTree) getLocation(d *db.Database) *db.Process {
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 1 {
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location = subquery.getLocation(d)
@ -58,6 +58,12 @@ type GetQueryTree struct {
slice int
}
// QueryTree for SET queries
type SetQueryTree struct {
bitmap *db.Bitmap
profile_id int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *GetQueryTree) getLocation(d *db.Database) *db.Process {
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
@ -68,9 +74,27 @@ func (qt *GetQueryTree) getLocation(d *db.Database) *db.Process {
return fragment.GetProcess()
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *SetQueryTree) getLocation(d *db.Database) *db.Process {
slice, err := d.GetSliceForProfile(qt.profile_id)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
panic(err)
}
return fragment.GetProcess()
}
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
var tree QueryTree
// handle SET operation regardless of the slice
if query.Operation == "set" {
tree = &SetQueryTree{query.Inputs[0].(*db.Bitmap), query.Profile_id}
return tree
}
// handle the remaining operations, taking slice into consideration
if slice == -1 {
tree = &CompositeQueryTree{operation: "cat"}
numSlices, err := qp.Database.NumSlices()
@ -115,6 +139,10 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Proces
step := QueryStep{*id, "get", []QueryInput{get.bitmap, get.slice}, get.getLocation(qp.Database), location}
plan := QueryPlan{step}
return &plan
} else if set, ok := qt.(*SetQueryTree); ok {
step := QueryStep{*id, "set", []QueryInput{set.bitmap, set.profile_id}, set.getLocation(qp.Database), location}
plan := QueryPlan{step}
return &plan
}
return &plan
}

6
query/planner_test.go
View file

@ -15,14 +15,14 @@ func TestQueryPlanner(t *testing.T) {
bm1 := db.Bitmap{10, "general"}
inputs1 := []QueryInput{&bm1}
query1 := Query{"get", inputs1}
query1 := Query{"get", inputs1, 0}
bm2 := db.Bitmap{20, "general"}
inputs2 := []QueryInput{&bm2}
query2 := Query{"get", inputs2}
query2 := Query{"get", inputs2, 0}
inputs := []QueryInput{&query1, &query2}
query := Query{"union", inputs}
query := Query{"union", inputs, 0}
/*
bm1 := db.Bitmap{10, "general"}

8
query/query.go
View file

@ -14,7 +14,7 @@ type Query struct {
Inputs []QueryInput //"strconv"
// Represents a parsed query. Inputs can be Query or Bitmap objects
// Maybe Bitmap and Query objects should have different fields to avoid using interface{}
Profile_id int
}
func Execute(database *db.Database, pql string) {
@ -22,6 +22,7 @@ func Execute(database *db.Database, pql string) {
//spew.Dump(pql)
tokens := Lex(pql)
//spew.Dump(tokens)
query_parser := QueryParser{}
query, err := query_parser.Parse(tokens)
if err != nil {
@ -29,6 +30,11 @@ func Execute(database *db.Database, pql string) {
}
//spew.Dump(query)
// switch on different query types:
//if query.Operation == "set" {
//spew.Dump("SET!!")
//}
query_planner := QueryPlanner{Database: database}
id, _ := uuid.NewV4()
destination := db.Process{}