From 781ecd0ce4de416a54f52e1fd79eab0c8a2b1811 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 16 Dec 2013 11:21:27 -0600 Subject: [PATCH] gofmt all the things! --- commands/pilosa-cruncher/cruncher.go | 2 +- commands/pilosa-nexter/nexter.go | 6 +- config/config.go | 2 +- config/config_test.go | 4 +- core/etcd.go | 43 +++--- core/http.go | 4 +- core/service.go | 44 +++---- core/stopper.go | 2 +- cruncher/cruncher.go | 29 ++--- cruncher/cruncher_test.go | 36 ++--- db/db.go | 4 +- db/topology.go | 151 ++++++++++----------- db/topology_test.go | 34 ++--- dispatch/dispatch.go | 2 +- interfaces/core.go | 2 +- query/lexer.go | 131 +++++++++---------- query/lexer_test.go | 188 +++++++++++++-------------- query/parser.go | 3 +- query/planner.go | 56 ++++---- query/planner_example.go | 1 + query/planner_test.go | 10 +- transport/http.go | 14 +- transport/tcp.go | 4 +- transport/transport_test.go | 16 +-- 24 files changed, 387 insertions(+), 401 deletions(-) diff --git a/commands/pilosa-cruncher/cruncher.go b/commands/pilosa-cruncher/cruncher.go index 865431d5b..ea66bc21a 100644 --- a/commands/pilosa-cruncher/cruncher.go +++ b/commands/pilosa-cruncher/cruncher.go @@ -1,8 +1,8 @@ package main import ( - "pilosa/cruncher" "pilosa/config" + "pilosa/cruncher" ) func main() { diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index 52abc33bb..31df74c2c 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -34,7 +34,7 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { path := "nexter/" + strconv.Itoa(id) for { - node, err := client.Get(path, false,false) + node, err := client.Get(path, false, false) if err != nil { ee, ok := err.(etcd.EtcdError) if ok && ee.ErrorCode == 100 { // node does not exist @@ -85,7 +85,7 @@ func (self *Nexter) loop() { case req := <-self.reqchan: switch req.(type) { case IncReq: - countreq := req.(*IncReq) + countreq := req.(*IncReq) counter, ok := counters[countreq.id] if !ok { counter = make(chan uint64) @@ -94,7 +94,7 @@ func (self *Nexter) loop() { } go func() { countreq.ret <- <-counter }() case DelReq: - delreq := req.(*DelReq) + delreq := req.(*DelReq) delete(counters, delreq.id) path := "nexter/" + strconv.Itoa(delreq.id) client.Delete(path, true) diff --git a/config/config.go b/config/config.go index 2d8f4264e..184c04c7c 100644 --- a/config/config.go +++ b/config/config.go @@ -1,9 +1,9 @@ package config import ( - "log" "io/ioutil" "launchpad.net/goyaml" + "log" "os" "sync" ) diff --git a/config/config_test.go b/config/config_test.go index 0e51805ae..635d2e562 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,9 +1,9 @@ package config import ( - "testing" - "os" . "github.com/smartystreets/goconvey/convey" + "os" + "testing" ) func TestConfig(t *testing.T) { diff --git a/core/etcd.go b/core/etcd.go index 38c35eff3..d600e94e7 100644 --- a/core/etcd.go +++ b/core/etcd.go @@ -1,24 +1,24 @@ package core import ( - "github.com/davecgh/go-spew/spew" - "github.com/coreos/go-etcd/etcd" - "pilosa/db" - "log" - "strings" - "github.com/nu7hatch/gouuid" - "strconv" "errors" + "github.com/coreos/go-etcd/etcd" + "github.com/davecgh/go-spew/spew" + "github.com/nu7hatch/gouuid" + "log" + "pilosa/db" + "strconv" + "strings" ) type TopologyMapper struct { - service *Service + service *Service namespace string } func (self *TopologyMapper) Run() { log.Println(self.namespace + "/db") - resp, err := self.service.Etcd.Get(self.namespace + "/db", false, true) + resp, err := self.service.Etcd.Get(self.namespace+"/db", false, true) if err != nil { log.Fatal(err) } @@ -34,7 +34,7 @@ func (self *TopologyMapper) Run() { go func() { // TODO: error check and restart watcher // TODO: use modindex to make sure watch catches everything - _, _ = self.service.Etcd.Watch(self.namespace + "/db", 0, true, receiver, stop) + _, _ = self.service.Etcd.Watch(self.namespace+"/db", 0, true, receiver, stop) }() go func() { for resp = range receiver { @@ -126,9 +126,9 @@ func flatten(node *etcd.Node) []*etcd.Node { } type Node struct { - id *uuid.UUID - ip string - port_tcp int + id *uuid.UUID + ip string + port_tcp int port_http int } @@ -137,8 +137,8 @@ type ProcessMap struct { } type ProcessMapper struct { - etcd *etcd.Client - nodes []Node + etcd *etcd.Client + nodes []Node receiver chan *etcd.Response commands chan *ProcessMapperCommand } @@ -156,7 +156,7 @@ func getKey(input string) string { return bits[len(bits)-1] } -func(self *ProcessMapper) getnode(u *uuid.UUID) *Node { +func (self *ProcessMapper) getnode(u *uuid.UUID) *Node { return new(Node) } @@ -191,7 +191,6 @@ func (self *ProcessMapper) Run() { nodes = append(nodes, node) } - self.nodes = nodes spew.Dump(self.nodes) go func() { @@ -211,21 +210,21 @@ func (self *ProcessMapper) Run() { case "set": bits := strings.Split(response.Node.Key, "/") if len(bits) != 4 { - log.Fatal("bug in etcd sync or etcd data") + log.Fatal("bug in etcd sync or etcd data") } //router, err := db.NewLocation(response.Node.Value) u, err := uuid.ParseHex(bits[2]) node := self.getnode(u) if err != nil { - log.Fatal(err) + log.Fatal(err) } switch bits[3] { case "port_tcp": - node.port_tcp, _ = strconv.Atoi(response.Node.Value) + node.port_tcp, _ = strconv.Atoi(response.Node.Value) case "port_http": - node.port_http, _ = strconv.Atoi(response.Node.Value) + node.port_http, _ = strconv.Atoi(response.Node.Value) case "ip": - node.ip = response.Node.Value + node.ip = response.Node.Value } spew.Dump(node) case "delete": diff --git a/core/http.go b/core/http.go index fdb2e893c..ee5dde8cb 100644 --- a/core/http.go +++ b/core/http.go @@ -1,10 +1,10 @@ package core import ( - "net/http" "encoding/json" - "log" "io/ioutil" + "log" + "net/http" "pilosa/db" "pilosa/query" ) diff --git a/core/service.go b/core/service.go index a298a2ecf..f78e324cf 100644 --- a/core/service.go +++ b/core/service.go @@ -5,20 +5,20 @@ import ( "log" "os" "os/signal" - "syscall" "pilosa/db" "pilosa/interfaces" + "syscall" ) type Service struct { Stopper - Etcd *etcd.Client - Cluster *db.Cluster + Etcd *etcd.Client + Cluster *db.Cluster TopologyMapper *TopologyMapper - ProcessMapper *ProcessMapper - ProcessMap *ProcessMap - Transport interfaces.Transporter - Dispatch interfaces.Dispatcher + ProcessMapper *ProcessMapper + ProcessMap *ProcessMap + Transport interfaces.Transporter + Dispatch interfaces.Dispatcher } func NewService() *Service { @@ -39,20 +39,20 @@ func (service *Service) GetSignals() (chan os.Signal, chan os.Signal) { } func (service *Service) Run() { - log.Println("Running service...") - go service.TopologyMapper.Run() - go service.ProcessMapper.Run() + log.Println("Running service...") + go service.TopologyMapper.Run() + go service.ProcessMapper.Run() - sigterm, sighup := service.GetSignals() - for { - select { - case <- sighup: - log.Println("SIGHUP! Reloading configuration...") - // TODO: reload configuration - case <- sigterm: - log.Println("SIGTERM! Cleaning up...") - service.Stop() - return - } - } + sigterm, sighup := service.GetSignals() + for { + select { + case <-sighup: + log.Println("SIGHUP! Reloading configuration...") + // TODO: reload configuration + case <-sigterm: + log.Println("SIGTERM! Cleaning up...") + service.Stop() + return + } + } } diff --git a/core/stopper.go b/core/stopper.go index 271f605a4..3e3c9fda8 100644 --- a/core/stopper.go +++ b/core/stopper.go @@ -7,7 +7,7 @@ import ( type Stopper struct { TermChans []chan int DoneChans []chan int - Mutex sync.RWMutex + Mutex sync.RWMutex } func (stopper *Stopper) Stop() { diff --git a/cruncher/cruncher.go b/cruncher/cruncher.go index 61bf8ac4b..834f6312b 100644 --- a/cruncher/cruncher.go +++ b/cruncher/cruncher.go @@ -1,18 +1,17 @@ package cruncher import ( - "github.com/davecgh/go-spew/spew" - "pilosa/index" + "github.com/davecgh/go-spew/spew" "pilosa/core" - "pilosa/transport" "pilosa/dispatch" + "pilosa/index" + "pilosa/transport" ) - type Cruncher struct { core.Service - close_chan chan bool - api *index.FragmentContainer + close_chan chan bool + api *index.FragmentContainer } func (cruncher *Cruncher) Run(port int) { @@ -20,20 +19,20 @@ func (cruncher *Cruncher) Run(port int) { spew.Dump(port) cruncher.api = index.NewFragmentContainer() -/* - bh = api.Get(frag,tileid) - api.SetBit(frag,bh,1) - api.Count(frag,bh) - api.Union(frag,[bh1,bh2]) - api.Intersect(frag,[bh1,bh2]) -*/ - + /* + bh = api.Get(frag,tileid) + api.SetBit(frag,bh,1) + api.Count(frag,bh) + api.Union(frag,[bh1,bh2]) + api.Intersect(frag,[bh1,bh2]) + */ + cruncher.Service.Run() } func NewCruncher() *Cruncher { service := core.NewService() - fragment_container := index.NewFragmentContainer() + fragment_container := index.NewFragmentContainer() cruncher := Cruncher{*service, make(chan bool), fragment_container} cruncher.Transport = transport.NewTcpTransport(service) cruncher.Dispatch = dispatch.NewCruncherDispatch(service) diff --git a/cruncher/cruncher_test.go b/cruncher/cruncher_test.go index b38ac3b6e..fd0d52598 100644 --- a/cruncher/cruncher_test.go +++ b/cruncher/cruncher_test.go @@ -1,30 +1,30 @@ package cruncher import ( - "testing" - //"github.com/nu7hatch/gouuid" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "testing" + //"github.com/nu7hatch/gouuid" + "github.com/davecgh/go-spew/spew" + . "github.com/smartystreets/goconvey/convey" ) func TestCruncher(t *testing.T) { - Convey("Basic Cruncher Tests", t, func() { - spew.Dump("cruncher test") + Convey("Basic Cruncher Tests", t, func() { + spew.Dump("cruncher test") - /* - cluster := NewCluster() - database := cluster.GetOrCreateDatabase("main") + /* + cluster := NewCluster() + database := cluster.GetOrCreateDatabase("main") - frame := database.GetOrCreateFrame("general") - slice := database.GetOrCreateSlice(0) + frame := database.GetOrCreateFrame("general") + slice := database.GetOrCreateSlice(0) - fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e") - spew.Dump(fragment_id) - database.GetOrCreateFragment(frame, slice, fragment_id) + fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e") + spew.Dump(fragment_id) + database.GetOrCreateFragment(frame, slice, fragment_id) - spew.Dump(database) - spew.Dump("DONE") - */ + spew.Dump(database) + spew.Dump("DONE") + */ - }) + }) } diff --git a/db/db.go b/db/db.go index 02f0d0e58..f49216985 100644 --- a/db/db.go +++ b/db/db.go @@ -1,7 +1,7 @@ package db type Message struct { - Key string `json:key` - Data interface{} `json:data` + Key string `json:key` + Data interface{} `json:data` Destination Location } diff --git a/db/topology.go b/db/topology.go index b4332bc7d..a6efa87f9 100644 --- a/db/topology.go +++ b/db/topology.go @@ -3,13 +3,13 @@ package db import ( "github.com/stathat/consistent" //"github.com/davecgh/go-spew/spew" + "errors" + "fmt" "github.com/nu7hatch/gouuid" "log" - "fmt" - "errors" - "strings" "strconv" - "sync" + "strings" + "sync" ) var FrameDoesNotExistError = errors.New("Frame does not exist.") @@ -18,7 +18,7 @@ var FragmentDoesNotExistError = errors.New("Fragment does not exist.") var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.") type Location struct { - Ip string + Ip string Port int } @@ -38,7 +38,7 @@ func NewLocation(location_string string) (*Location, error) { } ip := splitstring[0] port, err := strconv.Atoi(splitstring[1]) - if err != nil{ + if err != nil { return nil, errors.New("Port is not a number!") } return &Location{ip, port}, nil @@ -51,15 +51,12 @@ func (location *Location) ToString() string { // Map of node location to their router type NodeMap map[Location]Location - - - /////////// CLUSTERS //////////////////////////////////////////////////////////////////// // Represents the entire cluster, and a reference to the Node this instance is running on type Cluster struct { databases map[string]*Database - mutex sync.Mutex + mutex sync.Mutex } func NewCluster() *Cluster { @@ -68,16 +65,15 @@ func NewCluster() *Cluster { return &cluster } - /////////// DATABASES //////////////////////////////////////////////////////////////////// // A database is a collection of all the frames within a given profile space type Database struct { - Name string - frames []*Frame - slices []*Slice - frame_slice_intersects []*FrameSliceIntersect - mutex sync.Mutex + Name string + frames []*Frame + slices []*Slice + frame_slice_intersects []*FrameSliceIntersect + mutex sync.Mutex } // Add a database to a cluster @@ -100,13 +96,13 @@ func (c *Cluster) getDatabase(name string) (*Database, error) { } func (c *Cluster) GetOrCreateDatabase(name string) *Database { - c.mutex.Lock() - defer c.mutex.Unlock() - database, err := c.getDatabase(name) - if err == nil { - return database - } - return c.addDatabase(name) + c.mutex.Lock() + defer c.mutex.Unlock() + database, err := c.getDatabase(name) + if err == nil { + return database + } + return c.addDatabase(name) } // Count the number of slices in a database @@ -139,29 +135,28 @@ func (d *Database) getFrame(name string) (*Frame, error) { func (d *Database) addFrame(name string) *Frame { frame := Frame{name: name} d.frames = append(d.frames, &frame) - // add intersections + // add intersections for _, slice := range d.slices { - d.AddFrameSliceIntersect(&frame, slice) - } + d.AddFrameSliceIntersect(&frame, slice) + } return &frame } func (d *Database) GetOrCreateFrame(name string) *Frame { - d.mutex.Lock() - defer d.mutex.Unlock() - frame, err := d.getFrame(name) - if err == nil { - return frame - } - return d.addFrame(name) + d.mutex.Lock() + defer d.mutex.Unlock() + frame, err := d.getFrame(name) + if err == nil { + return frame + } + return d.addFrame(name) } - ///////// SLICES ///////////////////////////////////////////////////////////////////////// // A slice is the vertical combination of every fragment. type Slice struct { - id int + id int } // Get a slice from a database @@ -178,31 +173,30 @@ func (d *Database) getSlice(slice_id int) (*Slice, error) { func (d *Database) addSlice(slice_id int) *Slice { slice := Slice{id: slice_id} d.slices = append(d.slices, &slice) - // add intersections + // add intersections for _, frame := range d.frames { - d.AddFrameSliceIntersect(frame, &slice) - } + d.AddFrameSliceIntersect(frame, &slice) + } return &slice } func (d *Database) GetOrCreateSlice(slice_id int) *Slice { - d.mutex.Lock() - defer d.mutex.Unlock() - slice, err := d.getSlice(slice_id) - if err == nil { - return slice - } - return d.addSlice(slice_id) + d.mutex.Lock() + defer d.mutex.Unlock() + slice, err := d.getSlice(slice_id) + if err == nil { + return slice + } + return d.addSlice(slice_id) } - ///////// FRAME-SLICE INTERSECT ////////////////////////////////////////////////////////////// type FrameSliceIntersect struct { - frame *Frame - slice *Slice + frame *Frame + slice *Slice fragments []*Fragment - hashring *consistent.Consistent + hashring *consistent.Consistent } func (d *Database) AddFrameSliceIntersect(frame *Frame, slice *Slice) *FrameSliceIntersect { @@ -232,28 +226,26 @@ func (fsi *FrameSliceIntersect) GetFragment(fragment_id *uuid.UUID) (*Fragment, } func (fsi *FrameSliceIntersect) AddFragment(fragment *Fragment) { - fsi.fragments = append(fsi.fragments, fragment) - fsi.hashring.Add(fragment.id.String()) + fsi.fragments = append(fsi.fragments, fragment) + fsi.hashring.Add(fragment.id.String()) } - - ///////// FRAGMENTS //////////////////////////////////////////////////////////////////////// // A fragment is a collection of bitmaps within a slice. The fragment contains a reference to the responsible node for that fragment. The node is in the form ip:port type Fragment struct { - id *uuid.UUID - process *Process + id *uuid.UUID + process *Process } // rename this one func (d *Database) OldGetFragment(bitmap Bitmap, profile_id int) (*Fragment, error) { - d.mutex.Lock() - defer d.mutex.Unlock() - slice, _ := d.GetSliceForProfile(profile_id) - frame, _ := d.getFrame(bitmap.FrameType) - fsi, err := d.GetFrameSliceIntersect(frame, slice) - frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id)) + d.mutex.Lock() + defer d.mutex.Unlock() + slice, _ := d.GetSliceForProfile(profile_id) + frame, _ := d.getFrame(bitmap.FrameType) + fsi, err := d.GetFrameSliceIntersect(frame, slice) + frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id)) frag_id, err := uuid.ParseHex(frag_id_s) if err != nil { log.Fatal(err) @@ -261,7 +253,6 @@ func (d *Database) OldGetFragment(bitmap Bitmap, profile_id int) (*Fragment, err return fsi.GetFragment(frag_id) } - /* // NOT IMPLEMENTED // this would loop through all frame_slice_intersect[], then all fragmments to find a match @@ -269,21 +260,21 @@ func (d *Database) GetFragmentById(fragment_id *uuid.UUID) *Fragment { } */ func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) (*Fragment, error) { - fsi, err := d.GetFrameSliceIntersect(frame, slice) + fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Fatal(err) } - return fsi.GetFragment(fragment_id) + return fsi.GetFragment(fragment_id) } func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) *Fragment { - fsi, err := d.GetFrameSliceIntersect(frame, slice) + fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Fatal(err) } - fragment := Fragment{id: fragment_id} - fsi.AddFragment(&fragment) - return &fragment + fragment := Fragment{id: fragment_id} + fsi.AddFragment(&fragment) + return &fragment } /* @@ -319,30 +310,28 @@ func (d *Database) AddFragmentByProcess(frame *Frame, slice *Slice, process *Pro */ func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id *uuid.UUID) *Fragment { - d.mutex.Lock() - defer d.mutex.Unlock() - fragment, err := d.getFragment(frame, slice, fragment_id) - if err == nil { - return fragment - } - return d.addFragment(frame, slice, fragment_id) + d.mutex.Lock() + defer d.mutex.Unlock() + fragment, err := d.getFragment(frame, slice, fragment_id) + if err == nil { + return fragment + } + return d.addFragment(frame, slice, fragment_id) } func (f *Fragment) SetProcess(process *Process) { - f.process = process + f.process = process } /////////////////////////////////////////////////////////////////////////////////////////////// - // Get a slice from a database func (d *Database) GetSliceForProfile(profile_id int) (*Slice, error) { - slice_id := profile_id / SLICE_WIDTH - return d.getSlice(slice_id) + slice_id := profile_id / SLICE_WIDTH + return d.getSlice(slice_id) } - type Bitmap struct { - Id int + Id int FrameType string } diff --git a/db/topology_test.go b/db/topology_test.go index 0666ef0e7..762e90448 100644 --- a/db/topology_test.go +++ b/db/topology_test.go @@ -1,29 +1,29 @@ package db import ( - "testing" - "log" - "github.com/nu7hatch/gouuid" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "github.com/davecgh/go-spew/spew" + "github.com/nu7hatch/gouuid" + . "github.com/smartystreets/goconvey/convey" + "log" + "testing" ) func TestTopology(t *testing.T) { - Convey("Basic DB structures", t, func() { - log.Println("topology test") + Convey("Basic DB structures", t, func() { + log.Println("topology test") - cluster := NewCluster() - database := cluster.GetOrCreateDatabase("main") + cluster := NewCluster() + database := cluster.GetOrCreateDatabase("main") - frame := database.GetOrCreateFrame("general") - slice := database.GetOrCreateSlice(0) + frame := database.GetOrCreateFrame("general") + slice := database.GetOrCreateSlice(0) - fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e") - spew.Dump(fragment_id) - database.GetOrCreateFragment(frame, slice, fragment_id) + fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e") + spew.Dump(fragment_id) + database.GetOrCreateFragment(frame, slice, fragment_id) - spew.Dump(database) - spew.Dump("DONE") + spew.Dump(database) + spew.Dump("DONE") - }) + }) } diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 8201fba2f..90ba94b19 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -1,8 +1,8 @@ package dispatch import ( - "pilosa/core" "log" + "pilosa/core" ) type CruncherDispatch struct { diff --git a/interfaces/core.go b/interfaces/core.go index c6acfeb07..e6d991820 100644 --- a/interfaces/core.go +++ b/interfaces/core.go @@ -8,7 +8,7 @@ type Transporter interface { Init() error Close() Send(*db.Message) - Receive() (*db.Message) + Receive() *db.Message } type Dispatcher interface { diff --git a/query/lexer.go b/query/lexer.go index 68762b6ac..aefc7c5cb 100644 --- a/query/lexer.go +++ b/query/lexer.go @@ -2,18 +2,18 @@ package query import ( "errors" - "strings" - "log" - "unicode" - "unicode/utf8" + "log" + "strings" + "unicode" + "unicode/utf8" //"github.com/davecgh/go-spew/spew" ) const ( - TYPE_FUNC = iota - TYPE_LP = iota - TYPE_RP = iota - TYPE_ID = iota + TYPE_FUNC = iota + TYPE_LP = iota + TYPE_RP = iota + TYPE_ID = iota TYPE_COMMA = iota ) @@ -25,15 +25,14 @@ type Token struct { type statefn func(lexer *Lexer) statefn type Lexer struct { - text string // the string being scanned. - pos int // current position in the input. - width int // width of last rune read from input. - start int // start position of this item. - state int // current state of lexer NEEDED??? - ch chan Token // channel of scanned items (Tokens). + text string // the string being scanned. + pos int // current position in the input. + width int // width of last rune read from input. + start int // start position of this item. + state int // current state of lexer NEEDED??? + ch chan Token // channel of scanned items (Tokens). } - func (lexer *Lexer) emit(typ int) { lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ} lexer.start = lexer.pos @@ -44,7 +43,7 @@ func (lexer *Lexer) acceptUntil(chars string) error { if strings.HasPrefix(lexer.text[lexer.pos:], chars) { return nil } - // if we receive a reserved character that we are not expecting, throw a parse error + // if we receive a reserved character that we are not expecting, throw a parse error lexer.pos += 1 if lexer.pos > len(lexer.text) { return errors.New("Parse error, expecting " + string(chars)) @@ -53,45 +52,45 @@ func (lexer *Lexer) acceptUntil(chars string) error { } func (lexer *Lexer) acceptRun(valid string) { - for strings.IndexRune(valid, lexer.next()) >= 0 { - } - lexer.backup() + for strings.IndexRune(valid, lexer.next()) >= 0 { + } + lexer.backup() } // next returns the next rune in the input. func (lexer *Lexer) next() (runey rune) { - if lexer.pos >= len(lexer.text) { - lexer.width = 0 - return 0 - } - runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:]) - lexer.pos += lexer.width - return runey + if lexer.pos >= len(lexer.text) { + lexer.width = 0 + return 0 + } + runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:]) + lexer.pos += lexer.width + return runey } // ignore skips over the pending input before this point. func (lexer *Lexer) ignore() { - lexer.start = lexer.pos + lexer.start = lexer.pos } // backup steps back one rune. // Can be called only once per call of next. func (lexer *Lexer) backup() { - lexer.pos -= lexer.width + lexer.pos -= lexer.width } // peek returns but does not consume // the next rune in the input. func (lexer *Lexer) peek() rune { for { - next_rune := lexer.next() - // ignore spaces - if next_rune != rune(' ') { - lexer.backup() - return next_rune - } - lexer.ignore() - } + next_rune := lexer.next() + // ignore spaces + if next_rune != rune(' ') { + lexer.backup() + return next_rune + } + lexer.ignore() + } } func stateFunc(lexer *Lexer) statefn { @@ -106,50 +105,50 @@ func stateFunc(lexer *Lexer) statefn { func stateLP(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_LP) - // handle multiple LPs - if lexer.peek() == rune('(') { - return stateLP - } + // handle multiple LPs + if lexer.peek() == rune('(') { + return stateLP + } return stateArgs } func stateArgs(lexer *Lexer) statefn { - if unicode.IsNumber(lexer.peek()) { - return stateID - } else { - return stateFunc - } + if unicode.IsNumber(lexer.peek()) { + return stateID + } else { + return stateFunc + } } func stateID(lexer *Lexer) statefn { - digits := "0123456789" - lexer.acceptRun(digits) + digits := "0123456789" + lexer.acceptRun(digits) lexer.emit(TYPE_ID) - // if next is comma - peeked := lexer.peek() - if peeked == rune(',') { - return stateComma - } else if peeked == rune(')') { - return stateRP - } else { - return stateID - } + // if next is comma + peeked := lexer.peek() + if peeked == rune(',') { + return stateComma + } else if peeked == rune(')') { + return stateRP + } else { + return stateID + } } func stateRP(lexer *Lexer) statefn { lexer.pos += 1 lexer.emit(TYPE_RP) - peeked := lexer.peek() - if peeked == rune(',') { - return stateComma - } else if peeked == rune(')') { - return stateRP - } else { - return stateEOF - } + peeked := lexer.peek() + if peeked == rune(',') { + return stateComma + } else if peeked == rune(')') { + return stateRP + } else { + return stateEOF + } } func stateComma(lexer *Lexer) statefn { @@ -163,10 +162,10 @@ func stateEOF(lexer *Lexer) statefn { return nil } -func (lexer *Lexer) Lex() []Token{ +func (lexer *Lexer) Lex() []Token { tokens := make([]Token, 0) state := stateFunc - go func () { + go func() { for { state = state(lexer) if state == nil { diff --git a/query/lexer_test.go b/query/lexer_test.go index f0c632384..890d9a3d5 100644 --- a/query/lexer_test.go +++ b/query/lexer_test.go @@ -1,104 +1,104 @@ package query import ( - "testing" - . "github.com/smartystreets/goconvey/convey" + . "github.com/smartystreets/goconvey/convey" + "testing" ) func TestLexer(t *testing.T) { - Convey("Basic lexical analysis", t, func() { - tokens := Lex("get(10)") - So(len(tokens), ShouldEqual, 4) - So(tokens[0].Text, ShouldEqual, "get") - So(tokens[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens[1].Text, ShouldEqual, "(") - So(tokens[1].Type, ShouldEqual, TYPE_LP) - So(tokens[2].Text, ShouldEqual, "10") - So(tokens[2].Type, ShouldEqual, TYPE_ID) - So(tokens[3].Text, ShouldEqual, ")") - So(tokens[3].Type, ShouldEqual, TYPE_RP) + Convey("Basic lexical analysis", t, func() { + tokens := Lex("get(10)") + So(len(tokens), ShouldEqual, 4) + So(tokens[0].Text, ShouldEqual, "get") + So(tokens[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens[1].Text, ShouldEqual, "(") + So(tokens[1].Type, ShouldEqual, TYPE_LP) + So(tokens[2].Text, ShouldEqual, "10") + So(tokens[2].Type, ShouldEqual, TYPE_ID) + So(tokens[3].Text, ShouldEqual, ")") + So(tokens[3].Type, ShouldEqual, TYPE_RP) - tokens2 := Lex("intersect(get(10), get(11), get(12))") - So(len(tokens2), ShouldEqual, 17) - So(tokens2[0].Text, ShouldEqual, "intersect") - So(tokens2[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[1].Text, ShouldEqual, "(") - So(tokens2[1].Type, ShouldEqual, TYPE_LP) - So(tokens2[2].Text, ShouldEqual, "get") - So(tokens2[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[3].Text, ShouldEqual, "(") - So(tokens2[3].Type, ShouldEqual, TYPE_LP) - So(tokens2[4].Text, ShouldEqual, "10") - So(tokens2[4].Type, ShouldEqual, TYPE_ID) - So(tokens2[5].Text, ShouldEqual, ")") - So(tokens2[5].Type, ShouldEqual, TYPE_RP) - So(tokens2[6].Text, ShouldEqual, ",") - So(tokens2[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[7].Text, ShouldEqual, "get") - So(tokens2[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[8].Text, ShouldEqual, "(") - So(tokens2[8].Type, ShouldEqual, TYPE_LP) - So(tokens2[9].Text, ShouldEqual, "11") - So(tokens2[9].Type, ShouldEqual, TYPE_ID) - So(tokens2[10].Text, ShouldEqual, ")") - So(tokens2[10].Type, ShouldEqual, TYPE_RP) - So(tokens2[11].Text, ShouldEqual, ",") - So(tokens2[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens2[12].Text, ShouldEqual, "get") - So(tokens2[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens2[13].Text, ShouldEqual, "(") - So(tokens2[13].Type, ShouldEqual, TYPE_LP) - So(tokens2[14].Text, ShouldEqual, "12") - So(tokens2[14].Type, ShouldEqual, TYPE_ID) - So(tokens2[15].Text, ShouldEqual, ")") - So(tokens2[15].Type, ShouldEqual, TYPE_RP) - So(tokens2[16].Text, ShouldEqual, ")") - So(tokens2[16].Type, ShouldEqual, TYPE_RP) + tokens2 := Lex("intersect(get(10), get(11), get(12))") + So(len(tokens2), ShouldEqual, 17) + So(tokens2[0].Text, ShouldEqual, "intersect") + So(tokens2[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens2[1].Text, ShouldEqual, "(") + So(tokens2[1].Type, ShouldEqual, TYPE_LP) + So(tokens2[2].Text, ShouldEqual, "get") + So(tokens2[2].Type, ShouldEqual, TYPE_FUNC) + So(tokens2[3].Text, ShouldEqual, "(") + So(tokens2[3].Type, ShouldEqual, TYPE_LP) + So(tokens2[4].Text, ShouldEqual, "10") + So(tokens2[4].Type, ShouldEqual, TYPE_ID) + So(tokens2[5].Text, ShouldEqual, ")") + So(tokens2[5].Type, ShouldEqual, TYPE_RP) + So(tokens2[6].Text, ShouldEqual, ",") + So(tokens2[6].Type, ShouldEqual, TYPE_COMMA) + So(tokens2[7].Text, ShouldEqual, "get") + So(tokens2[7].Type, ShouldEqual, TYPE_FUNC) + So(tokens2[8].Text, ShouldEqual, "(") + So(tokens2[8].Type, ShouldEqual, TYPE_LP) + So(tokens2[9].Text, ShouldEqual, "11") + So(tokens2[9].Type, ShouldEqual, TYPE_ID) + So(tokens2[10].Text, ShouldEqual, ")") + So(tokens2[10].Type, ShouldEqual, TYPE_RP) + So(tokens2[11].Text, ShouldEqual, ",") + So(tokens2[11].Type, ShouldEqual, TYPE_COMMA) + So(tokens2[12].Text, ShouldEqual, "get") + So(tokens2[12].Type, ShouldEqual, TYPE_FUNC) + So(tokens2[13].Text, ShouldEqual, "(") + So(tokens2[13].Type, ShouldEqual, TYPE_LP) + So(tokens2[14].Text, ShouldEqual, "12") + So(tokens2[14].Type, ShouldEqual, TYPE_ID) + So(tokens2[15].Text, ShouldEqual, ")") + So(tokens2[15].Type, ShouldEqual, TYPE_RP) + So(tokens2[16].Text, ShouldEqual, ")") + So(tokens2[16].Type, ShouldEqual, TYPE_RP) - tokens3 := Lex("intersect(get(10), get(11), concat(12,14))") - So(len(tokens3), ShouldEqual, 19) - So(tokens3[0].Text, ShouldEqual, "intersect") - So(tokens3[0].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[1].Text, ShouldEqual, "(") - So(tokens3[1].Type, ShouldEqual, TYPE_LP) - So(tokens3[2].Text, ShouldEqual, "get") - So(tokens3[2].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[3].Text, ShouldEqual, "(") - So(tokens3[3].Type, ShouldEqual, TYPE_LP) - So(tokens3[4].Text, ShouldEqual, "10") - So(tokens3[4].Type, ShouldEqual, TYPE_ID) - So(tokens3[5].Text, ShouldEqual, ")") - So(tokens3[5].Type, ShouldEqual, TYPE_RP) - So(tokens3[6].Text, ShouldEqual, ",") - So(tokens3[6].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[7].Text, ShouldEqual, "get") - So(tokens3[7].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[8].Text, ShouldEqual, "(") - So(tokens3[8].Type, ShouldEqual, TYPE_LP) - So(tokens3[9].Text, ShouldEqual, "11") - So(tokens3[9].Type, ShouldEqual, TYPE_ID) - So(tokens3[10].Text, ShouldEqual, ")") - So(tokens3[10].Type, ShouldEqual, TYPE_RP) - So(tokens3[11].Text, ShouldEqual, ",") - So(tokens3[11].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[12].Text, ShouldEqual, "concat") - So(tokens3[12].Type, ShouldEqual, TYPE_FUNC) - So(tokens3[13].Text, ShouldEqual, "(") - So(tokens3[13].Type, ShouldEqual, TYPE_LP) - So(tokens3[14].Text, ShouldEqual, "12") - So(tokens3[14].Type, ShouldEqual, TYPE_ID) - So(tokens3[15].Text, ShouldEqual, ",") - So(tokens3[15].Type, ShouldEqual, TYPE_COMMA) - So(tokens3[16].Text, ShouldEqual, "14") - So(tokens3[16].Type, ShouldEqual, TYPE_ID) - So(tokens3[17].Text, ShouldEqual, ")") - So(tokens3[17].Type, ShouldEqual, TYPE_RP) - So(tokens3[18].Text, ShouldEqual, ")") - So(tokens3[18].Type, ShouldEqual, TYPE_RP) + tokens3 := Lex("intersect(get(10), get(11), concat(12,14))") + So(len(tokens3), ShouldEqual, 19) + So(tokens3[0].Text, ShouldEqual, "intersect") + So(tokens3[0].Type, ShouldEqual, TYPE_FUNC) + So(tokens3[1].Text, ShouldEqual, "(") + So(tokens3[1].Type, ShouldEqual, TYPE_LP) + So(tokens3[2].Text, ShouldEqual, "get") + So(tokens3[2].Type, ShouldEqual, TYPE_FUNC) + So(tokens3[3].Text, ShouldEqual, "(") + So(tokens3[3].Type, ShouldEqual, TYPE_LP) + So(tokens3[4].Text, ShouldEqual, "10") + So(tokens3[4].Type, ShouldEqual, TYPE_ID) + So(tokens3[5].Text, ShouldEqual, ")") + So(tokens3[5].Type, ShouldEqual, TYPE_RP) + So(tokens3[6].Text, ShouldEqual, ",") + So(tokens3[6].Type, ShouldEqual, TYPE_COMMA) + So(tokens3[7].Text, ShouldEqual, "get") + So(tokens3[7].Type, ShouldEqual, TYPE_FUNC) + So(tokens3[8].Text, ShouldEqual, "(") + So(tokens3[8].Type, ShouldEqual, TYPE_LP) + So(tokens3[9].Text, ShouldEqual, "11") + So(tokens3[9].Type, ShouldEqual, TYPE_ID) + So(tokens3[10].Text, ShouldEqual, ")") + So(tokens3[10].Type, ShouldEqual, TYPE_RP) + So(tokens3[11].Text, ShouldEqual, ",") + So(tokens3[11].Type, ShouldEqual, TYPE_COMMA) + So(tokens3[12].Text, ShouldEqual, "concat") + So(tokens3[12].Type, ShouldEqual, TYPE_FUNC) + So(tokens3[13].Text, ShouldEqual, "(") + So(tokens3[13].Type, ShouldEqual, TYPE_LP) + So(tokens3[14].Text, ShouldEqual, "12") + So(tokens3[14].Type, ShouldEqual, TYPE_ID) + So(tokens3[15].Text, ShouldEqual, ",") + So(tokens3[15].Type, ShouldEqual, TYPE_COMMA) + So(tokens3[16].Text, ShouldEqual, "14") + So(tokens3[16].Type, ShouldEqual, TYPE_ID) + So(tokens3[17].Text, ShouldEqual, ")") + So(tokens3[17].Type, ShouldEqual, TYPE_RP) + So(tokens3[18].Text, ShouldEqual, ")") + So(tokens3[18].Type, ShouldEqual, TYPE_RP) - tokens4 := Lex("concat(1,2,345,890)") - So(len(tokens4), ShouldEqual, 10) - So(tokens4[6].Text, ShouldEqual, "345") - So(tokens4[6].Type, ShouldEqual, TYPE_ID) - }) + tokens4 := Lex("concat(1,2,345,890)") + So(len(tokens4), ShouldEqual, 10) + So(tokens4[6].Text, ShouldEqual, "345") + So(tokens4[6].Type, ShouldEqual, TYPE_ID) + }) } diff --git a/query/parser.go b/query/parser.go index c94b8cc26..a24b01ba9 100644 --- a/query/parser.go +++ b/query/parser.go @@ -1,13 +1,12 @@ package query import ( - "encoding/json" + "encoding/json" "errors" "pilosa/db" //"github.com/davecgh/go-spew/spew" ) - var InvalidQueryError = errors.New("Invalid query format.") type QueryParser struct { diff --git a/query/planner.go b/query/planner.go index 8551e0438..397864b14 100644 --- a/query/planner.go +++ b/query/planner.go @@ -8,12 +8,12 @@ import ( "pilosa/db" ) -// A single step in the query plan. +// A single step in the query plan. type QueryStep struct { - id uuid.UUID - operation string - inputs []QueryInput - location string + id uuid.UUID + operation string + inputs []QueryInput + location string destination string } @@ -21,19 +21,19 @@ func (q QueryStep) String() string { return fmt.Sprintf("%s %s %s, LOC: %s, DEST: %s", q.operation, q.id.String(), q.inputs, q.location, q.destination) } -type QueryInput interface {} +type QueryInput interface{} // Represents a parsed query. Inputs can be Query or Bitmap objects -type Query struct{ +type Query struct { Operation string - Inputs []QueryInput // Maybe Bitmap and Query objects should have different fields to avoid using interface{} + Inputs []QueryInput // Maybe Bitmap and Query objects should have different fields to avoid using interface{} } // This is the output of the query planner. Contains a list of steps which can be performed in parallel type QueryPlan []QueryStep type QueryPlanner struct { - Cluster *db.Cluster + Cluster *db.Cluster Database *db.Database } @@ -43,9 +43,9 @@ type QueryTree interface { // QueryTree for UNION, INTER, and CAT queries type CompositeQueryTree struct { - operation string + operation string subqueries []QueryTree - location string + location string } // Randomly select location from subqueries (so subqueries roll up into composite queries while minimizing inter-node data traffic) @@ -64,34 +64,34 @@ func (qt *CompositeQueryTree) getLocation(d *db.Database) string { // QueryTree for GET queries type GetQueryTree struct { bitmap db.Bitmap - slice int + slice int } // Uses consistent hashing function to select node containing data for GET operation func (qt *GetQueryTree) getLocation(d *db.Database) string { - /* - frame, err := d.GetFrame(qt.bitmap.FrameType) - if err != nil { - panic(err) - } - slice := frame.Slices[qt.slice] - hashString, err := slice.Hashring.Get(strconv.Itoa(qt.bitmap.Id)) + /* + frame, err := d.GetFrame(qt.bitmap.FrameType) + if err != nil { + panic(err) + } + slice := frame.Slices[qt.slice] + hashString, err := slice.Hashring.Get(strconv.Itoa(qt.bitmap.Id)) - var sliceIndex int - var fragIndex int - fmt.Sscan(hashString, &fragIndex, &sliceIndex) - fragment := slice.Fragments[fragIndex] + var sliceIndex int + var fragIndex int + fmt.Sscan(hashString, &fragIndex, &sliceIndex) + fragment := slice.Fragments[fragIndex] - return fmt.Sprintf(fragment.Node) - */ - return "Nothing yet" + return fmt.Sprintf(fragment.Node) + */ + return "Nothing yet" } // Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { var tree QueryTree if slice == -1 { - tree = &CompositeQueryTree{operation:"cat"} + tree = &CompositeQueryTree{operation: "cat"} numSlices, err := qp.Database.NumSlices() if err != nil { panic(err) @@ -110,7 +110,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { for i, input := range query.Inputs { subqueries[i] = qp.buildTree(input.(*Query), slice) } - tree = &CompositeQueryTree{operation:query.Operation, subqueries:subqueries} + tree = &CompositeQueryTree{operation: query.Operation, subqueries: subqueries} } } return tree diff --git a/query/planner_example.go b/query/planner_example.go index 7e4ceb12b..4e80603de 100644 --- a/query/planner_example.go +++ b/query/planner_example.go @@ -1,4 +1,5 @@ package query + //package main // //import ( diff --git a/query/planner_test.go b/query/planner_test.go index f63aa4160..6cef3873b 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -1,13 +1,13 @@ package query import ( - "testing" + . "github.com/smartystreets/goconvey/convey" "log" - . "github.com/smartystreets/goconvey/convey" + "testing" ) func TestQueryPlanner(t *testing.T) { - Convey("Basic query plan", t, func() { - log.Println("query planner test") - }) + Convey("Basic query plan", t, func() { + log.Println("query planner test") + }) } diff --git a/transport/http.go b/transport/http.go index 913031b86..f9504f2a9 100644 --- a/transport/http.go +++ b/transport/http.go @@ -1,14 +1,14 @@ package transport import ( - "pilosa/db" "log" + "pilosa/db" ) type HttpTransport struct { - port int + port int outbox chan *db.Message - done chan int + done chan int } func (trans *HttpTransport) Init() error { @@ -22,10 +22,10 @@ func (trans *HttpTransport) Loop() { var message *db.Message for { select { - case message = <-trans.outbox: - log.Println(message) - case <-trans.done: - return + case message = <-trans.outbox: + log.Println(message) + case <-trans.done: + return } } } diff --git a/transport/tcp.go b/transport/tcp.go index 1f80fae04..e30cbde09 100644 --- a/transport/tcp.go +++ b/transport/tcp.go @@ -1,13 +1,13 @@ package transport import ( + "log" "pilosa/core" "pilosa/db" - "log" ) type TcpTransport struct { - port int + port int inbox chan *db.Message } diff --git a/transport/transport_test.go b/transport/transport_test.go index d68bf216a..d2378ceb8 100644 --- a/transport/transport_test.go +++ b/transport/transport_test.go @@ -8,13 +8,13 @@ import ( func TestHttpTransport(t *testing.T) { Convey("Test HTTP transport", t, func() { - /* - var com Transporter - com = NewHttpTransport(9009) - com.Init() - com.Send("derp", &db.Message{"derp", 42, db.Location{}}) - com.Close() - So(1, ShouldEqual, 1) - */ + /* + var com Transporter + com = NewHttpTransport(9009) + com.Init() + com.Send("derp", &db.Message{"derp", 42, db.Location{}}) + com.Close() + So(1, ShouldEqual, 1) + */ }) }