scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

gofmt all the things!

Browse source
This commit is contained in:
Cody Soyland 2013-12-16 11:21:27 -06:00
parent d8051ea1be
commit 781ecd0ce4
24 changed files with 387 additions and 401 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
commands/pilosa-cruncher/cruncher.go
View file

@ -1,8 +1,8 @@
package main
import (
"pilosa/cruncher"
"pilosa/config"
"pilosa/cruncher"
)
func main() {

6
commands/pilosa-nexter/nexter.go
View file

@ -34,7 +34,7 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) {
path := "nexter/" + strconv.Itoa(id)
for {
node, err := client.Get(path, false,false)
node, err := client.Get(path, false, false)
if err != nil {
ee, ok := err.(etcd.EtcdError)
if ok && ee.ErrorCode == 100 { // node does not exist
@ -85,7 +85,7 @@ func (self *Nexter) loop() {
case req := <-self.reqchan:
switch req.(type) {
case IncReq:
countreq := req.(*IncReq)
countreq := req.(*IncReq)
counter, ok := counters[countreq.id]
if !ok {
counter = make(chan uint64)
@ -94,7 +94,7 @@ func (self *Nexter) loop() {
}
go func() { countreq.ret <- <-counter }()
case DelReq:
delreq := req.(*DelReq)
delreq := req.(*DelReq)
delete(counters, delreq.id)
path := "nexter/" + strconv.Itoa(delreq.id)
client.Delete(path, true)

2
config/config.go
View file

@ -1,9 +1,9 @@
package config
import (
"log"
"io/ioutil"
"launchpad.net/goyaml"
"log"
"os"
"sync"
)

4
config/config_test.go
View file

@ -1,9 +1,9 @@
package config
import (
"testing"
"os"
. "github.com/smartystreets/goconvey/convey"
"os"
"testing"
)
func TestConfig(t *testing.T) {

43
core/etcd.go
View file

@ -1,24 +1,24 @@
package core
import (
"github.com/davecgh/go-spew/spew"
"github.com/coreos/go-etcd/etcd"
"pilosa/db"
"log"
"strings"
"github.com/nu7hatch/gouuid"
"strconv"
"errors"
"github.com/coreos/go-etcd/etcd"
"github.com/davecgh/go-spew/spew"
"github.com/nu7hatch/gouuid"
"log"
"pilosa/db"
"strconv"
"strings"
)
type TopologyMapper struct {
service *Service
service *Service
namespace string
}
func (self *TopologyMapper) Run() {
log.Println(self.namespace + "/db")
resp, err := self.service.Etcd.Get(self.namespace + "/db", false, true)
resp, err := self.service.Etcd.Get(self.namespace+"/db", false, true)
if err != nil {
log.Fatal(err)
}
@ -34,7 +34,7 @@ func (self *TopologyMapper) Run() {
go func() {
// TODO: error check and restart watcher
// TODO: use modindex to make sure watch catches everything
_, _ = self.service.Etcd.Watch(self.namespace + "/db", 0, true, receiver, stop)
_, _ = self.service.Etcd.Watch(self.namespace+"/db", 0, true, receiver, stop)
}()
go func() {
for resp = range receiver {
@ -126,9 +126,9 @@ func flatten(node *etcd.Node) []*etcd.Node {
}
type Node struct {
id *uuid.UUID
ip string
port_tcp int
id *uuid.UUID
ip string
port_tcp int
port_http int
}
@ -137,8 +137,8 @@ type ProcessMap struct {
}
type ProcessMapper struct {
etcd *etcd.Client
nodes []Node
etcd *etcd.Client
nodes []Node
receiver chan *etcd.Response
commands chan *ProcessMapperCommand
}
@ -156,7 +156,7 @@ func getKey(input string) string {
return bits[len(bits)-1]
}
func(self *ProcessMapper) getnode(u *uuid.UUID) *Node {
func (self *ProcessMapper) getnode(u *uuid.UUID) *Node {
return new(Node)
}
@ -191,7 +191,6 @@ func (self *ProcessMapper) Run() {
nodes = append(nodes, node)
}
self.nodes = nodes
spew.Dump(self.nodes)
go func() {
@ -211,21 +210,21 @@ func (self *ProcessMapper) Run() {
case "set":
bits := strings.Split(response.Node.Key, "/")
if len(bits) != 4 {
log.Fatal("bug in etcd sync or etcd data")
log.Fatal("bug in etcd sync or etcd data")
}
//router, err := db.NewLocation(response.Node.Value)
u, err := uuid.ParseHex(bits[2])
node := self.getnode(u)
if err != nil {
log.Fatal(err)
log.Fatal(err)
}
switch bits[3] {
case "port_tcp":
node.port_tcp, _ = strconv.Atoi(response.Node.Value)
node.port_tcp, _ = strconv.Atoi(response.Node.Value)
case "port_http":
node.port_http, _ = strconv.Atoi(response.Node.Value)
node.port_http, _ = strconv.Atoi(response.Node.Value)
case "ip":
node.ip = response.Node.Value
node.ip = response.Node.Value
}
spew.Dump(node)
case "delete":

4
core/http.go
View file

@ -1,10 +1,10 @@
package core
import (
"net/http"
"encoding/json"
"log"
"io/ioutil"
"log"
"net/http"
"pilosa/db"
"pilosa/query"
)

44
core/service.go
View file

@ -5,20 +5,20 @@ import (
"log"
"os"
"os/signal"
"syscall"
"pilosa/db"
"pilosa/interfaces"
"syscall"
)
type Service struct {
Stopper
Etcd *etcd.Client
Cluster *db.Cluster
Etcd *etcd.Client
Cluster *db.Cluster
TopologyMapper *TopologyMapper
ProcessMapper *ProcessMapper
ProcessMap *ProcessMap
Transport interfaces.Transporter
Dispatch interfaces.Dispatcher
ProcessMapper *ProcessMapper
ProcessMap *ProcessMap
Transport interfaces.Transporter
Dispatch interfaces.Dispatcher
}
func NewService() *Service {
@ -39,20 +39,20 @@ func (service *Service) GetSignals() (chan os.Signal, chan os.Signal) {
}
func (service *Service) Run() {
log.Println("Running service...")
go service.TopologyMapper.Run()
go service.ProcessMapper.Run()
log.Println("Running service...")
go service.TopologyMapper.Run()
go service.ProcessMapper.Run()
sigterm, sighup := service.GetSignals()
for {
select {
case <- sighup:
log.Println("SIGHUP! Reloading configuration...")
// TODO: reload configuration
case <- sigterm:
log.Println("SIGTERM! Cleaning up...")
service.Stop()
return
}
}
sigterm, sighup := service.GetSignals()
for {
select {
case <-sighup:
log.Println("SIGHUP! Reloading configuration...")
// TODO: reload configuration
case <-sigterm:
log.Println("SIGTERM! Cleaning up...")
service.Stop()
return
}
}
}

2
core/stopper.go
View file

@ -7,7 +7,7 @@ import (
type Stopper struct {
TermChans []chan int
DoneChans []chan int
Mutex sync.RWMutex
Mutex sync.RWMutex
}
func (stopper *Stopper) Stop() {

29
cruncher/cruncher.go
View file

@ -1,18 +1,17 @@
package cruncher
import (
"github.com/davecgh/go-spew/spew"
"pilosa/index"
"github.com/davecgh/go-spew/spew"
"pilosa/core"
"pilosa/transport"
"pilosa/dispatch"
"pilosa/index"
"pilosa/transport"
)
type Cruncher struct {
core.Service
close_chan chan bool
api *index.FragmentContainer
close_chan chan bool
api *index.FragmentContainer
}
func (cruncher *Cruncher) Run(port int) {
@ -20,20 +19,20 @@ func (cruncher *Cruncher) Run(port int) {
spew.Dump(port)
cruncher.api = index.NewFragmentContainer()
/*
bh = api.Get(frag,tileid)
api.SetBit(frag,bh,1)
api.Count(frag,bh)
api.Union(frag,[bh1,bh2])
api.Intersect(frag,[bh1,bh2])
*/
/*
bh = api.Get(frag,tileid)
api.SetBit(frag,bh,1)
api.Count(frag,bh)
api.Union(frag,[bh1,bh2])
api.Intersect(frag,[bh1,bh2])
*/
cruncher.Service.Run()
}
func NewCruncher() *Cruncher {
service := core.NewService()
fragment_container := index.NewFragmentContainer()
fragment_container := index.NewFragmentContainer()
cruncher := Cruncher{*service, make(chan bool), fragment_container}
cruncher.Transport = transport.NewTcpTransport(service)
cruncher.Dispatch = dispatch.NewCruncherDispatch(service)

36
cruncher/cruncher_test.go
View file

@ -1,30 +1,30 @@
package cruncher
import (
"testing"
//"github.com/nu7hatch/gouuid"
"github.com/davecgh/go-spew/spew"
. "github.com/smartystreets/goconvey/convey"
"testing"
//"github.com/nu7hatch/gouuid"
"github.com/davecgh/go-spew/spew"
. "github.com/smartystreets/goconvey/convey"
)
func TestCruncher(t *testing.T) {
Convey("Basic Cruncher Tests", t, func() {
spew.Dump("cruncher test")
Convey("Basic Cruncher Tests", t, func() {
spew.Dump("cruncher test")
/*
cluster := NewCluster()
database := cluster.GetOrCreateDatabase("main")
/*
cluster := NewCluster()
database := cluster.GetOrCreateDatabase("main")
frame := database.GetOrCreateFrame("general")
slice := database.GetOrCreateSlice(0)
frame := database.GetOrCreateFrame("general")
slice := database.GetOrCreateSlice(0)
fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e")
spew.Dump(fragment_id)
database.GetOrCreateFragment(frame, slice, fragment_id)
fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e")
spew.Dump(fragment_id)
database.GetOrCreateFragment(frame, slice, fragment_id)
spew.Dump(database)
spew.Dump("DONE")
*/
spew.Dump(database)
spew.Dump("DONE")
*/
})
})
}

4
db/db.go
View file

@ -1,7 +1,7 @@
package db
type Message struct {
Key string `json:key`
Data interface{} `json:data`
Key string `json:key`
Data interface{} `json:data`
Destination Location
}

151
db/topology.go
View file

@ -3,13 +3,13 @@ package db
import (
"github.com/stathat/consistent"
//"github.com/davecgh/go-spew/spew"
"errors"
"fmt"
"github.com/nu7hatch/gouuid"
"log"
"fmt"
"errors"
"strings"
"strconv"
"sync"
"strings"
"sync"
)
var FrameDoesNotExistError = errors.New("Frame does not exist.")
@ -18,7 +18,7 @@ var FragmentDoesNotExistError = errors.New("Fragment does not exist.")
var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.")
type Location struct {
Ip string
Ip string
Port int
}
@ -38,7 +38,7 @@ func NewLocation(location_string string) (*Location, error) {
}
ip := splitstring[0]
port, err := strconv.Atoi(splitstring[1])
if err != nil{
if err != nil {
return nil, errors.New("Port is not a number!")
}
return &Location{ip, port}, nil
@ -51,15 +51,12 @@ func (location *Location) ToString() string {
// Map of node location to their router
type NodeMap map[Location]Location
/////////// CLUSTERS ////////////////////////////////////////////////////////////////////
// Represents the entire cluster, and a reference to the Node this instance is running on
type Cluster struct {
databases map[string]*Database
mutex sync.Mutex
mutex sync.Mutex
}
func NewCluster() *Cluster {
@ -68,16 +65,15 @@ func NewCluster() *Cluster {
return &cluster
}
/////////// DATABASES ////////////////////////////////////////////////////////////////////
// A database is a collection of all the frames within a given profile space
type Database struct {
Name string
frames []*Frame
slices []*Slice
frame_slice_intersects []*FrameSliceIntersect
mutex sync.Mutex
Name string
frames []*Frame
slices []*Slice
frame_slice_intersects []*FrameSliceIntersect
mutex sync.Mutex
}
// Add a database to a cluster
@ -100,13 +96,13 @@ func (c *Cluster) getDatabase(name string) (*Database, error) {
}
func (c *Cluster) GetOrCreateDatabase(name string) *Database {
c.mutex.Lock()
defer c.mutex.Unlock()
database, err := c.getDatabase(name)
if err == nil {
return database
}
return c.addDatabase(name)
c.mutex.Lock()
defer c.mutex.Unlock()
database, err := c.getDatabase(name)
if err == nil {
return database
}
return c.addDatabase(name)
}
// Count the number of slices in a database
@ -139,29 +135,28 @@ func (d *Database) getFrame(name string) (*Frame, error) {
func (d *Database) addFrame(name string) *Frame {
frame := Frame{name: name}
d.frames = append(d.frames, &frame)
// add intersections
// add intersections
for _, slice := range d.slices {
d.AddFrameSliceIntersect(&frame, slice)
}
d.AddFrameSliceIntersect(&frame, slice)
}
return &frame
}
func (d *Database) GetOrCreateFrame(name string) *Frame {
d.mutex.Lock()
defer d.mutex.Unlock()
frame, err := d.getFrame(name)
if err == nil {
return frame
}
return d.addFrame(name)
d.mutex.Lock()
defer d.mutex.Unlock()
frame, err := d.getFrame(name)
if err == nil {
return frame
}
return d.addFrame(name)
}
///////// SLICES /////////////////////////////////////////////////////////////////////////
// A slice is the vertical combination of every fragment.
type Slice struct {
id int
id int
}
// Get a slice from a database
@ -178,31 +173,30 @@ func (d *Database) getSlice(slice_id int) (*Slice, error) {
func (d *Database) addSlice(slice_id int) *Slice {
slice := Slice{id: slice_id}
d.slices = append(d.slices, &slice)
// add intersections
// add intersections
for _, frame := range d.frames {
d.AddFrameSliceIntersect(frame, &slice)
}
d.AddFrameSliceIntersect(frame, &slice)
}
return &slice
}
func (d *Database) GetOrCreateSlice(slice_id int) *Slice {
d.mutex.Lock()
defer d.mutex.Unlock()
slice, err := d.getSlice(slice_id)
if err == nil {
return slice
}
return d.addSlice(slice_id)
d.mutex.Lock()
defer d.mutex.Unlock()
slice, err := d.getSlice(slice_id)
if err == nil {
return slice
}
return d.addSlice(slice_id)
}
///////// FRAME-SLICE INTERSECT //////////////////////////////////////////////////////////////
type FrameSliceIntersect struct {
frame *Frame
slice *Slice
frame *Frame
slice *Slice
fragments []*Fragment
hashring *consistent.Consistent
hashring *consistent.Consistent
}
func (d *Database) AddFrameSliceIntersect(frame *Frame, slice *Slice) *FrameSliceIntersect {
@ -232,28 +226,26 @@ func (fsi *FrameSliceIntersect) GetFragment(fragment_id *uuid.UUID) (*Fragment,
}
func (fsi *FrameSliceIntersect) AddFragment(fragment *Fragment) {
fsi.fragments = append(fsi.fragments, fragment)
fsi.hashring.Add(fragment.id.String())
fsi.fragments = append(fsi.fragments, fragment)
fsi.hashring.Add(fragment.id.String())
}
///////// FRAGMENTS ////////////////////////////////////////////////////////////////////////
// A fragment is a collection of bitmaps within a slice. The fragment contains a reference to the responsible node for that fragment. The node is in the form ip:port
type Fragment struct {
id *uuid.UUID
process *Process
id *uuid.UUID
process *Process
}
// rename this one
func (d *Database) OldGetFragment(bitmap Bitmap, profile_id int) (*Fragment, error) {
d.mutex.Lock()
defer d.mutex.Unlock()
slice, _ := d.GetSliceForProfile(profile_id)
frame, _ := d.getFrame(bitmap.FrameType)
fsi, err := d.GetFrameSliceIntersect(frame, slice)
frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id))
d.mutex.Lock()
defer d.mutex.Unlock()
slice, _ := d.GetSliceForProfile(profile_id)
frame, _ := d.getFrame(bitmap.FrameType)
fsi, err := d.GetFrameSliceIntersect(frame, slice)
frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id))
frag_id, err := uuid.ParseHex(frag_id_s)
if err != nil {
log.Fatal(err)
@ -261,7 +253,6 @@ func (d *Database) OldGetFragment(bitmap Bitmap, profile_id int) (*Fragment, err
return fsi.GetFragment(frag_id)
}
/*
// NOT IMPLEMENTED
// this would loop through all frame_slice_intersect[], then all fragmments to find a match
@ -269,21 +260,21 @@ func (d *Database) GetFragmentById(fragment_id *uuid.UUID) *Fragment {
}
*/
func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Fatal(err)
}
return fsi.GetFragment(fragment_id)
return fsi.GetFragment(fragment_id)
}
func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) *Fragment {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Fatal(err)
}
fragment := Fragment{id: fragment_id}
fsi.AddFragment(&fragment)
return &fragment
fragment := Fragment{id: fragment_id}
fsi.AddFragment(&fragment)
return &fragment
}
/*
@ -319,30 +310,28 @@ func (d *Database) AddFragmentByProcess(frame *Frame, slice *Slice, process *Pro
*/
func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) *Fragment {
d.mutex.Lock()
defer d.mutex.Unlock()
fragment, err := d.getFragment(frame, slice, fragment_id)
if err == nil {
return fragment
}
return d.addFragment(frame, slice, fragment_id)
d.mutex.Lock()
defer d.mutex.Unlock()
fragment, err := d.getFragment(frame, slice, fragment_id)
if err == nil {
return fragment
}
return d.addFragment(frame, slice, fragment_id)
}
func (f *Fragment) SetProcess(process *Process) {
f.process = process
f.process = process
}
///////////////////////////////////////////////////////////////////////////////////////////////
// Get a slice from a database
func (d *Database) GetSliceForProfile(profile_id int) (*Slice, error) {
slice_id := profile_id / SLICE_WIDTH
return d.getSlice(slice_id)
slice_id := profile_id / SLICE_WIDTH
return d.getSlice(slice_id)
}
type Bitmap struct {
Id int
Id int
FrameType string
}

34
db/topology_test.go
View file

@ -1,29 +1,29 @@
package db
import (
"testing"
"log"
"github.com/nu7hatch/gouuid"
"github.com/davecgh/go-spew/spew"
. "github.com/smartystreets/goconvey/convey"
"github.com/davecgh/go-spew/spew"
"github.com/nu7hatch/gouuid"
. "github.com/smartystreets/goconvey/convey"
"log"
"testing"
)
func TestTopology(t *testing.T) {
Convey("Basic DB structures", t, func() {
log.Println("topology test")
Convey("Basic DB structures", t, func() {
log.Println("topology test")
cluster := NewCluster()
database := cluster.GetOrCreateDatabase("main")
cluster := NewCluster()
database := cluster.GetOrCreateDatabase("main")
frame := database.GetOrCreateFrame("general")
slice := database.GetOrCreateSlice(0)
frame := database.GetOrCreateFrame("general")
slice := database.GetOrCreateSlice(0)
fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e")
spew.Dump(fragment_id)
database.GetOrCreateFragment(frame, slice, fragment_id)
fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e")
spew.Dump(fragment_id)
database.GetOrCreateFragment(frame, slice, fragment_id)
spew.Dump(database)
spew.Dump("DONE")
spew.Dump(database)
spew.Dump("DONE")
})
})
}

2
dispatch/dispatch.go
View file

@ -1,8 +1,8 @@
package dispatch
import (
"pilosa/core"
"log"
"pilosa/core"
)
type CruncherDispatch struct {

2
interfaces/core.go
View file

@ -8,7 +8,7 @@ type Transporter interface {
Init() error
Close()
Send(*db.Message)
Receive() (*db.Message)
Receive() *db.Message
}
type Dispatcher interface {

131
query/lexer.go
View file

@ -2,18 +2,18 @@ package query
import (
"errors"
"strings"
"log"
"unicode"
"unicode/utf8"
"log"
"strings"
"unicode"
"unicode/utf8"
//"github.com/davecgh/go-spew/spew"
)
const (
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_ID = iota
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_ID = iota
TYPE_COMMA = iota
)
@ -25,15 +25,14 @@ type Token struct {
type statefn func(lexer *Lexer) statefn
type Lexer struct {
text string // the string being scanned.
pos int // current position in the input.
width int // width of last rune read from input.
start int // start position of this item.
state int // current state of lexer NEEDED???
ch chan Token // channel of scanned items (Tokens).
text string // the string being scanned.
pos int // current position in the input.
width int // width of last rune read from input.
start int // start position of this item.
state int // current state of lexer NEEDED???
ch chan Token // channel of scanned items (Tokens).
}
func (lexer *Lexer) emit(typ int) {
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
lexer.start = lexer.pos
@ -44,7 +43,7 @@ func (lexer *Lexer) acceptUntil(chars string) error {
if strings.HasPrefix(lexer.text[lexer.pos:], chars) {
return nil
}
// if we receive a reserved character that we are not expecting, throw a parse error
// if we receive a reserved character that we are not expecting, throw a parse error
lexer.pos += 1
if lexer.pos > len(lexer.text) {
return errors.New("Parse error, expecting " + string(chars))
@ -53,45 +52,45 @@ func (lexer *Lexer) acceptUntil(chars string) error {
}
func (lexer *Lexer) acceptRun(valid string) {
for strings.IndexRune(valid, lexer.next()) >= 0 {
}
lexer.backup()
for strings.IndexRune(valid, lexer.next()) >= 0 {
}
lexer.backup()
}
// next returns the next rune in the input.
func (lexer *Lexer) next() (runey rune) {
if lexer.pos >= len(lexer.text) {
lexer.width = 0
return 0
}
runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:])
lexer.pos += lexer.width
return runey
if lexer.pos >= len(lexer.text) {
lexer.width = 0
return 0
}
runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:])
lexer.pos += lexer.width
return runey
}
// ignore skips over the pending input before this point.
func (lexer *Lexer) ignore() {
lexer.start = lexer.pos
lexer.start = lexer.pos
}
// backup steps back one rune.
// Can be called only once per call of next.
func (lexer *Lexer) backup() {
lexer.pos -= lexer.width
lexer.pos -= lexer.width
}
// peek returns but does not consume
// the next rune in the input.
func (lexer *Lexer) peek() rune {
for {
next_rune := lexer.next()
// ignore spaces
if next_rune != rune(' ') {
lexer.backup()
return next_rune
}
lexer.ignore()
}
next_rune := lexer.next()
// ignore spaces
if next_rune != rune(' ') {
lexer.backup()
return next_rune
}
lexer.ignore()
}
}
func stateFunc(lexer *Lexer) statefn {
@ -106,50 +105,50 @@ func stateFunc(lexer *Lexer) statefn {
func stateLP(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_LP)
// handle multiple LPs
if lexer.peek() == rune('(') {
return stateLP
}
// handle multiple LPs
if lexer.peek() == rune('(') {
return stateLP
}
return stateArgs
}
func stateArgs(lexer *Lexer) statefn {
if unicode.IsNumber(lexer.peek()) {
return stateID
} else {
return stateFunc
}
if unicode.IsNumber(lexer.peek()) {
return stateID
} else {
return stateFunc
}
}
func stateID(lexer *Lexer) statefn {
digits := "0123456789"
lexer.acceptRun(digits)
digits := "0123456789"
lexer.acceptRun(digits)
lexer.emit(TYPE_ID)
// if next is comma
peeked := lexer.peek()
if peeked == rune(',') {
return stateComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateID
}
// if next is comma
peeked := lexer.peek()
if peeked == rune(',') {
return stateComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateID
}
}
func stateRP(lexer *Lexer) statefn {
lexer.pos += 1
lexer.emit(TYPE_RP)
peeked := lexer.peek()
if peeked == rune(',') {
return stateComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateEOF
}
peeked := lexer.peek()
if peeked == rune(',') {
return stateComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateEOF
}
}
func stateComma(lexer *Lexer) statefn {
@ -163,10 +162,10 @@ func stateEOF(lexer *Lexer) statefn {
return nil
}
func (lexer *Lexer) Lex() []Token{
func (lexer *Lexer) Lex() []Token {
tokens := make([]Token, 0)
state := stateFunc
go func () {
go func() {
for {
state = state(lexer)
if state == nil {

188
query/lexer_test.go
View file

@ -1,104 +1,104 @@
package query
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestLexer(t *testing.T) {
Convey("Basic lexical analysis", t, func() {
tokens := Lex("get(10)")
So(len(tokens), ShouldEqual, 4)
So(tokens[0].Text, ShouldEqual, "get")
So(tokens[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens[1].Text, ShouldEqual, "(")
So(tokens[1].Type, ShouldEqual, TYPE_LP)
So(tokens[2].Text, ShouldEqual, "10")
So(tokens[2].Type, ShouldEqual, TYPE_ID)
So(tokens[3].Text, ShouldEqual, ")")
So(tokens[3].Type, ShouldEqual, TYPE_RP)
Convey("Basic lexical analysis", t, func() {
tokens := Lex("get(10)")
So(len(tokens), ShouldEqual, 4)
So(tokens[0].Text, ShouldEqual, "get")
So(tokens[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens[1].Text, ShouldEqual, "(")
So(tokens[1].Type, ShouldEqual, TYPE_LP)
So(tokens[2].Text, ShouldEqual, "10")
So(tokens[2].Type, ShouldEqual, TYPE_ID)
So(tokens[3].Text, ShouldEqual, ")")
So(tokens[3].Type, ShouldEqual, TYPE_RP)
tokens2 := Lex("intersect(get(10), get(11), get(12))")
So(len(tokens2), ShouldEqual, 17)
So(tokens2[0].Text, ShouldEqual, "intersect")
So(tokens2[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[1].Text, ShouldEqual, "(")
So(tokens2[1].Type, ShouldEqual, TYPE_LP)
So(tokens2[2].Text, ShouldEqual, "get")
So(tokens2[2].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[3].Text, ShouldEqual, "(")
So(tokens2[3].Type, ShouldEqual, TYPE_LP)
So(tokens2[4].Text, ShouldEqual, "10")
So(tokens2[4].Type, ShouldEqual, TYPE_ID)
So(tokens2[5].Text, ShouldEqual, ")")
So(tokens2[5].Type, ShouldEqual, TYPE_RP)
So(tokens2[6].Text, ShouldEqual, ",")
So(tokens2[6].Type, ShouldEqual, TYPE_COMMA)
So(tokens2[7].Text, ShouldEqual, "get")
So(tokens2[7].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[8].Text, ShouldEqual, "(")
So(tokens2[8].Type, ShouldEqual, TYPE_LP)
So(tokens2[9].Text, ShouldEqual, "11")
So(tokens2[9].Type, ShouldEqual, TYPE_ID)
So(tokens2[10].Text, ShouldEqual, ")")
So(tokens2[10].Type, ShouldEqual, TYPE_RP)
So(tokens2[11].Text, ShouldEqual, ",")
So(tokens2[11].Type, ShouldEqual, TYPE_COMMA)
So(tokens2[12].Text, ShouldEqual, "get")
So(tokens2[12].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[13].Text, ShouldEqual, "(")
So(tokens2[13].Type, ShouldEqual, TYPE_LP)
So(tokens2[14].Text, ShouldEqual, "12")
So(tokens2[14].Type, ShouldEqual, TYPE_ID)
So(tokens2[15].Text, ShouldEqual, ")")
So(tokens2[15].Type, ShouldEqual, TYPE_RP)
So(tokens2[16].Text, ShouldEqual, ")")
So(tokens2[16].Type, ShouldEqual, TYPE_RP)
tokens2 := Lex("intersect(get(10), get(11), get(12))")
So(len(tokens2), ShouldEqual, 17)
So(tokens2[0].Text, ShouldEqual, "intersect")
So(tokens2[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[1].Text, ShouldEqual, "(")
So(tokens2[1].Type, ShouldEqual, TYPE_LP)
So(tokens2[2].Text, ShouldEqual, "get")
So(tokens2[2].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[3].Text, ShouldEqual, "(")
So(tokens2[3].Type, ShouldEqual, TYPE_LP)
So(tokens2[4].Text, ShouldEqual, "10")
So(tokens2[4].Type, ShouldEqual, TYPE_ID)
So(tokens2[5].Text, ShouldEqual, ")")
So(tokens2[5].Type, ShouldEqual, TYPE_RP)
So(tokens2[6].Text, ShouldEqual, ",")
So(tokens2[6].Type, ShouldEqual, TYPE_COMMA)
So(tokens2[7].Text, ShouldEqual, "get")
So(tokens2[7].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[8].Text, ShouldEqual, "(")
So(tokens2[8].Type, ShouldEqual, TYPE_LP)
So(tokens2[9].Text, ShouldEqual, "11")
So(tokens2[9].Type, ShouldEqual, TYPE_ID)
So(tokens2[10].Text, ShouldEqual, ")")
So(tokens2[10].Type, ShouldEqual, TYPE_RP)
So(tokens2[11].Text, ShouldEqual, ",")
So(tokens2[11].Type, ShouldEqual, TYPE_COMMA)
So(tokens2[12].Text, ShouldEqual, "get")
So(tokens2[12].Type, ShouldEqual, TYPE_FUNC)
So(tokens2[13].Text, ShouldEqual, "(")
So(tokens2[13].Type, ShouldEqual, TYPE_LP)
So(tokens2[14].Text, ShouldEqual, "12")
So(tokens2[14].Type, ShouldEqual, TYPE_ID)
So(tokens2[15].Text, ShouldEqual, ")")
So(tokens2[15].Type, ShouldEqual, TYPE_RP)
So(tokens2[16].Text, ShouldEqual, ")")
So(tokens2[16].Type, ShouldEqual, TYPE_RP)
tokens3 := Lex("intersect(get(10), get(11), concat(12,14))")
So(len(tokens3), ShouldEqual, 19)
So(tokens3[0].Text, ShouldEqual, "intersect")
So(tokens3[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[1].Text, ShouldEqual, "(")
So(tokens3[1].Type, ShouldEqual, TYPE_LP)
So(tokens3[2].Text, ShouldEqual, "get")
So(tokens3[2].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[3].Text, ShouldEqual, "(")
So(tokens3[3].Type, ShouldEqual, TYPE_LP)
So(tokens3[4].Text, ShouldEqual, "10")
So(tokens3[4].Type, ShouldEqual, TYPE_ID)
So(tokens3[5].Text, ShouldEqual, ")")
So(tokens3[5].Type, ShouldEqual, TYPE_RP)
So(tokens3[6].Text, ShouldEqual, ",")
So(tokens3[6].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[7].Text, ShouldEqual, "get")
So(tokens3[7].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[8].Text, ShouldEqual, "(")
So(tokens3[8].Type, ShouldEqual, TYPE_LP)
So(tokens3[9].Text, ShouldEqual, "11")
So(tokens3[9].Type, ShouldEqual, TYPE_ID)
So(tokens3[10].Text, ShouldEqual, ")")
So(tokens3[10].Type, ShouldEqual, TYPE_RP)
So(tokens3[11].Text, ShouldEqual, ",")
So(tokens3[11].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[12].Text, ShouldEqual, "concat")
So(tokens3[12].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[13].Text, ShouldEqual, "(")
So(tokens3[13].Type, ShouldEqual, TYPE_LP)
So(tokens3[14].Text, ShouldEqual, "12")
So(tokens3[14].Type, ShouldEqual, TYPE_ID)
So(tokens3[15].Text, ShouldEqual, ",")
So(tokens3[15].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[16].Text, ShouldEqual, "14")
So(tokens3[16].Type, ShouldEqual, TYPE_ID)
So(tokens3[17].Text, ShouldEqual, ")")
So(tokens3[17].Type, ShouldEqual, TYPE_RP)
So(tokens3[18].Text, ShouldEqual, ")")
So(tokens3[18].Type, ShouldEqual, TYPE_RP)
tokens3 := Lex("intersect(get(10), get(11), concat(12,14))")
So(len(tokens3), ShouldEqual, 19)
So(tokens3[0].Text, ShouldEqual, "intersect")
So(tokens3[0].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[1].Text, ShouldEqual, "(")
So(tokens3[1].Type, ShouldEqual, TYPE_LP)
So(tokens3[2].Text, ShouldEqual, "get")
So(tokens3[2].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[3].Text, ShouldEqual, "(")
So(tokens3[3].Type, ShouldEqual, TYPE_LP)
So(tokens3[4].Text, ShouldEqual, "10")
So(tokens3[4].Type, ShouldEqual, TYPE_ID)
So(tokens3[5].Text, ShouldEqual, ")")
So(tokens3[5].Type, ShouldEqual, TYPE_RP)
So(tokens3[6].Text, ShouldEqual, ",")
So(tokens3[6].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[7].Text, ShouldEqual, "get")
So(tokens3[7].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[8].Text, ShouldEqual, "(")
So(tokens3[8].Type, ShouldEqual, TYPE_LP)
So(tokens3[9].Text, ShouldEqual, "11")
So(tokens3[9].Type, ShouldEqual, TYPE_ID)
So(tokens3[10].Text, ShouldEqual, ")")
So(tokens3[10].Type, ShouldEqual, TYPE_RP)
So(tokens3[11].Text, ShouldEqual, ",")
So(tokens3[11].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[12].Text, ShouldEqual, "concat")
So(tokens3[12].Type, ShouldEqual, TYPE_FUNC)
So(tokens3[13].Text, ShouldEqual, "(")
So(tokens3[13].Type, ShouldEqual, TYPE_LP)
So(tokens3[14].Text, ShouldEqual, "12")
So(tokens3[14].Type, ShouldEqual, TYPE_ID)
So(tokens3[15].Text, ShouldEqual, ",")
So(tokens3[15].Type, ShouldEqual, TYPE_COMMA)
So(tokens3[16].Text, ShouldEqual, "14")
So(tokens3[16].Type, ShouldEqual, TYPE_ID)
So(tokens3[17].Text, ShouldEqual, ")")
So(tokens3[17].Type, ShouldEqual, TYPE_RP)
So(tokens3[18].Text, ShouldEqual, ")")
So(tokens3[18].Type, ShouldEqual, TYPE_RP)
tokens4 := Lex("concat(1,2,345,890)")
So(len(tokens4), ShouldEqual, 10)
So(tokens4[6].Text, ShouldEqual, "345")
So(tokens4[6].Type, ShouldEqual, TYPE_ID)
})
tokens4 := Lex("concat(1,2,345,890)")
So(len(tokens4), ShouldEqual, 10)
So(tokens4[6].Text, ShouldEqual, "345")
So(tokens4[6].Type, ShouldEqual, TYPE_ID)
})
}

3
query/parser.go
View file

@ -1,13 +1,12 @@
package query
import (
"encoding/json"
"encoding/json"
"errors"
"pilosa/db"
//"github.com/davecgh/go-spew/spew"
)
var InvalidQueryError = errors.New("Invalid query format.")
type QueryParser struct {

56
query/planner.go
View file

@ -8,12 +8,12 @@ import (
"pilosa/db"
)
// A single step in the query plan.
// A single step in the query plan.
type QueryStep struct {
id uuid.UUID
operation string
inputs []QueryInput
location string
id uuid.UUID
operation string
inputs []QueryInput
location string
destination string
}
@ -21,19 +21,19 @@ func (q QueryStep) String() string {
return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.destination)
}
type QueryInput interface {}
type QueryInput interface{}
// Represents a parsed query. Inputs can be Query or Bitmap objects
type Query struct{
type Query struct {
Operation string
Inputs []QueryInput // Maybe Bitmap and Query objects should have different fields to avoid using interface{}
Inputs []QueryInput // Maybe Bitmap and Query objects should have different fields to avoid using interface{}
}
// This is the output of the query planner. Contains a list of steps which can be performed in parallel
type QueryPlan []QueryStep
type QueryPlanner struct {
Cluster *db.Cluster
Cluster *db.Cluster
Database *db.Database
}
@ -43,9 +43,9 @@ type QueryTree interface {
// QueryTree for UNION, INTER, and CAT queries
type CompositeQueryTree struct {
operation string
operation string
subqueries []QueryTree
location string
location string
}
// Randomly select location from subqueries (so subqueries roll up into composite queries while minimizing inter-node data traffic)
@ -64,34 +64,34 @@ func (qt *CompositeQueryTree) getLocation(d *db.Database) string {
// QueryTree for GET queries
type GetQueryTree struct {
bitmap db.Bitmap
slice int
slice int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *GetQueryTree) getLocation(d *db.Database) string {
/*
frame, err := d.GetFrame(qt.bitmap.FrameType)
if err != nil {
panic(err)
}
slice := frame.Slices[qt.slice]
hashString, err := slice.Hashring.Get(strconv.Itoa(qt.bitmap.Id))
/*
frame, err := d.GetFrame(qt.bitmap.FrameType)
if err != nil {
panic(err)
}
slice := frame.Slices[qt.slice]
hashString, err := slice.Hashring.Get(strconv.Itoa(qt.bitmap.Id))
var sliceIndex int
var fragIndex int
fmt.Sscan(hashString, &fragIndex, &sliceIndex)
fragment := slice.Fragments[fragIndex]
var sliceIndex int
var fragIndex int
fmt.Sscan(hashString, &fragIndex, &sliceIndex)
fragment := slice.Fragments[fragIndex]
return fmt.Sprintf(fragment.Node)
*/
return "Nothing yet"
return fmt.Sprintf(fragment.Node)
*/
return "Nothing yet"
}
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
var tree QueryTree
if slice == -1 {
tree = &CompositeQueryTree{operation:"cat"}
tree = &CompositeQueryTree{operation: "cat"}
numSlices, err := qp.Database.NumSlices()
if err != nil {
panic(err)
@ -110,7 +110,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
for i, input := range query.Inputs {
subqueries[i] = qp.buildTree(input.(*Query), slice)
}
tree = &CompositeQueryTree{operation:query.Operation, subqueries:subqueries}
tree = &CompositeQueryTree{operation: query.Operation, subqueries: subqueries}
}
}
return tree

1
query/planner_example.go
View file

@ -1,4 +1,5 @@
package query
//package main
//
//import (

10
query/planner_test.go
View file

@ -1,13 +1,13 @@
package query
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"log"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestQueryPlanner(t *testing.T) {
Convey("Basic query plan", t, func() {
log.Println("query planner test")
})
Convey("Basic query plan", t, func() {
log.Println("query planner test")
})
}

14
transport/http.go
View file

@ -1,14 +1,14 @@
package transport
import (
"pilosa/db"
"log"
"pilosa/db"
)
type HttpTransport struct {
port int
port int
outbox chan *db.Message
done chan int
done chan int
}
func (trans *HttpTransport) Init() error {
@ -22,10 +22,10 @@ func (trans *HttpTransport) Loop() {
var message *db.Message
for {
select {
case message = <-trans.outbox:
log.Println(message)
case <-trans.done:
return
case message = <-trans.outbox:
log.Println(message)
case <-trans.done:
return
}
}
}

4
transport/tcp.go
View file

@ -1,13 +1,13 @@
package transport
import (
"log"
"pilosa/core"
"pilosa/db"
"log"
)
type TcpTransport struct {
port int
port int
inbox chan *db.Message
}

16
transport/transport_test.go
View file

@ -8,13 +8,13 @@ import (
func TestHttpTransport(t *testing.T) {
Convey("Test HTTP transport", t, func() {
/*
var com Transporter
com = NewHttpTransport(9009)
com.Init()
com.Send("derp", &db.Message{"derp", 42, db.Location{}})
com.Close()
So(1, ShouldEqual, 1)
*/
/*
var com Transporter
com = NewHttpTransport(9009)
com.Init()
com.Send("derp", &db.Message{"derp", 42, db.Location{}})
com.Close()
So(1, ShouldEqual, 1)
*/
})
}