diff --git a/brand.go b/brand.go index 7a538949e..33cf34d54 100644 --- a/brand.go +++ b/brand.go @@ -9,7 +9,7 @@ import ( "time" log "github.com/cihub/seelog" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa/statsd" ) var FragmentBase string @@ -159,7 +159,7 @@ func (b *Brand) Rank() { b.rank_count = 0 delta := time.Since(start) - util.SendTimer("brand_Rank", delta.Nanoseconds()) + statsd.SendTimer("brand_Rank", delta.Nanoseconds()) b.rank_time = start } @@ -366,7 +366,7 @@ func (b *Brand) Persist() error { log.Warn("Nothing to save ", b.getFileName()) return nil } - w, err := util.Create(b.getFileName()) + w, err := createFile(b.getFileName()) if err != nil { log.Warn("Error opening outfile ", b.getFileName()) log.Warn(err) @@ -396,7 +396,7 @@ func (b *Brand) Persist() error { func (b *Brand) Load(requestChan chan Command, f *Fragment) { log.Warn("Brand Load") time.Sleep(time.Duration(rand.Intn(32)) * time.Second) //trying to avoid mass cassandra hit - r, err := util.Open(b.getFileName()) + r, err := openFile(b.getFileName()) if err != nil { log.Warn("NO Brand Init File:", b.getFileName()) return diff --git a/brand_test.go b/brand_test.go index 2b3ec908d..6ff6a2895 100644 --- a/brand_test.go +++ b/brand_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/umbel/pilosa" + "github.com/umbel/pilosa/statsd" "github.com/umbel/pilosa/storage/mem" - "github.com/umbel/pilosa/util" ) var ( @@ -18,7 +18,7 @@ var ( func init() { size = 1000 - util.SetupStatsd() + statsd.Setup() // SetupCassandra() membrand = pilosa.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0) diff --git a/cmd/pilosa/config.go b/cmd/pilosa/config.go index 4e5778e28..37bf27534 100644 --- a/cmd/pilosa/config.go +++ b/cmd/pilosa/config.go @@ -4,10 +4,10 @@ import ( "time" "github.com/umbel/pilosa" + "github.com/umbel/pilosa/statsd" _ "github.com/umbel/pilosa/storage" "github.com/umbel/pilosa/storage/cassandra" "github.com/umbel/pilosa/transport" - "github.com/umbel/pilosa/util" ) const ( @@ -31,8 +31,8 @@ var ( // Config represents the configuration for the command. type Config struct { - ID *util.GUID `toml:"id"` - Host string `toml:"host"` + ID *pilosa.GUID `toml:"id"` + Host string `toml:"host"` TCP struct { Port int `toml:"port"` @@ -99,7 +99,7 @@ func NewConfig() Config { c.Storage.SupportedFrames = DefaultSupportedFrames[:] c.Storage.CassandraTimeWindow = Duration(cassandra.DefaultFlushInterval) c.Storage.CassandraMaxSizeBatch = cassandra.DefaultFlushThreshold - c.Statsd.Host = util.DefaultStatsdHost + c.Statsd.Host = statsd.DefaultHost c.ETCD.Hosts = DefaultETCDHosts[:] return c } diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 09db4be03..a575c45d2 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -18,8 +18,8 @@ import ( "github.com/umbel/pilosa/dispatch" "github.com/umbel/pilosa/executor" "github.com/umbel/pilosa/hold" + "github.com/umbel/pilosa/statsd" "github.com/umbel/pilosa/transport" - "github.com/umbel/pilosa/util" ) // Build holds the build information passed in at compile time. @@ -68,7 +68,7 @@ func (m *Main) Run(args ...string) error { // Generate an ID if one is not specified in the config. id := config.ID if id == nil { - *id = util.RandomUUID() + *id = pilosa.RandomUUID() } // Set up profiling. @@ -94,8 +94,8 @@ func (m *Main) Run(args ...string) error { s3util.DefaultConfig.SecretKey = config.AWS.SecretAccessKey // Initialize Statsd. - util.StatsdHost = config.Statsd.Host - util.SetupStatsd() + statsd.Host = config.Statsd.Host + statsd.Setup() // Initialize logging. logger, _ := log.LoggerFromConfigAsBytes([]byte(SeelogProductionConfig(config.Log.Path, *id, config.Log.Level))) diff --git a/cmd/pilosa/seelog.go b/cmd/pilosa/seelog.go index 6b0112390..b87c1c0cf 100644 --- a/cmd/pilosa/seelog.go +++ b/cmd/pilosa/seelog.go @@ -3,10 +3,10 @@ package main import ( "fmt" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa" ) -func SeelogProductionConfig(path string, id util.GUID, level string) string { +func SeelogProductionConfig(path string, id pilosa.GUID, level string) string { if path == "" { path = "/tmp" } diff --git a/core/batch.go b/core/batch.go index 753891d6b..a140607a8 100644 --- a/core/batch.go +++ b/core/batch.go @@ -4,25 +4,25 @@ import ( "encoding/gob" log "github.com/cihub/seelog" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/hold" - "github.com/umbel/pilosa/util" ) type BatchRequest struct { - Id *util.GUID - Source *util.GUID - Fragment_id util.SUUID + Id *pilosa.GUID + Source *pilosa.GUID + Fragment_id pilosa.SUUID Bitmap_id uint64 Compressed_bitmap string Filter uint64 } type BatchResponse struct { - Id *util.GUID + Id *pilosa.GUID } -func (self BatchResponse) ResultId() *util.GUID { +func (self BatchResponse) ResultId() *pilosa.GUID { return self.Id } func (self BatchResponse) ResultData() interface{} { @@ -35,16 +35,16 @@ func init() { } type Batcher struct { - ID util.GUID + ID pilosa.GUID Cluster *db.Cluster Hold *hold.Holder Transport interface { - Send(message *db.Message, host *util.GUID) + Send(message *db.Message, host *pilosa.GUID) } } -func NewBatcher(id util.GUID) *Batcher { +func NewBatcher(id pilosa.GUID) *Batcher { return &Batcher{ID: id} } @@ -58,7 +58,7 @@ func (b *Batcher) Batch(database_name, frame, compressed_bitmap string, bitmap_i fragment, err := database.GetFragmentForBitmap(oslice, &db.Bitmap{Id: bitmap_id, FrameType: frame, Filter: filter}) if err == nil { - id := util.RandomUUID() + id := pilosa.RandomUUID() batch := db.Message{Data: BatchRequest{Id: &id, Source: &b.ID, Fragment_id: fragment.GetId(), Bitmap_id: bitmap_id, Compressed_bitmap: compressed_bitmap}} dest_id := fragment.GetProcess().Id() b.Transport.Send(&batch, &dest_id) diff --git a/core/etcd.go b/core/etcd.go index 6b01b6f7f..436bd95ae 100644 --- a/core/etcd.go +++ b/core/etcd.go @@ -14,8 +14,8 @@ import ( log "github.com/cihub/seelog" "github.com/coreos/go-etcd/etcd" "github.com/davecgh/go-spew/spew" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) const ( @@ -25,7 +25,7 @@ const ( type TopologyMapper struct { namespace string - ID util.GUID + ID pilosa.GUID Cluster *db.Cluster ProcessMap *ProcessMap @@ -41,7 +41,7 @@ type TopologyMapper struct { } Index interface { - AddFragment(db string, frame string, slice int, id util.SUUID) + AddFragment(db string, frame string, slice int, id pilosa.SUUID) } } @@ -193,7 +193,7 @@ func (self *TopologyMapper) AllocateFragment(process_guid, db, frame string, sli //to create the node, just write off the items to etcd and the watch should spawn //be nice if something would notify perhaps queue //so i need db, frame, slice , fragment_id - fuid := util.SUUID_to_Hex(util.Id()) + fuid := pilosa.SUUID_to_Hex(pilosa.Id()) fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid) // need to check value to see how many we have left log.Warn("ALLOC:", process_guid, len(process_guid)) @@ -215,10 +215,10 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { var database *db.Database var frame *db.Frame var fragment *db.Fragment - var fragment_id util.SUUID + var fragment_id pilosa.SUUID var slice *db.Slice var slice_int int - var process_uuid util.GUID + var process_uuid pilosa.GUID var process *db.Process var err error @@ -227,7 +227,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { return errors.New("no process") } - process_uuid, err = util.ParseGUID(node.Value) + process_uuid, err = pilosa.ParseGUID(node.Value) if err != nil { log.Warn("Bad Process Guid", key) return errors.New("No Process Id") @@ -268,7 +268,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { } } if len(bits) > 7 { - fragment_id = util.Hex_to_SUUID(bits[7]) + fragment_id = pilosa.Hex_to_SUUID(bits[7]) fragment = database.GetOrCreateFragment(frame, slice, fragment_id) } @@ -284,7 +284,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { process = db.NewProcess(&process_uuid) fragment.SetProcess(process) - if util.Equal(&self.ID, &process_uuid) { + if pilosa.Equal(&self.ID, &process_uuid) { self.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id) } @@ -299,26 +299,26 @@ func (self *TopologyMapper) remove_fragment(node *etcd.Node) error { func flatten(node *etcd.Node) []*etcd.Node { nodes := []*etcd.Node{node} for i := 0; i < len(node.Nodes); i++ { - nodes = append(nodes, flatten(node.Nodes[i])...) + nodes = append(nodes, flatten(&node.Nodes[i])...) } return nodes } type Node struct { - id *util.GUID + id *pilosa.GUID ip string port_tcp int port_http int } type ProcessMap struct { - nodes map[util.GUID]*db.Process + nodes map[pilosa.GUID]*db.Process mutex sync.Mutex } func NewProcessMap() *ProcessMap { p := ProcessMap{} - p.nodes = make(map[util.GUID]*db.Process) + p.nodes = make(map[pilosa.GUID]*db.Process) return &p } @@ -328,7 +328,7 @@ func (self *ProcessMap) AddProcess(process *db.Process) { self.nodes[process.Id()] = process } -func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) { +func (self *ProcessMap) GetProcess(id *pilosa.GUID) (*db.Process, error) { self.mutex.Lock() defer self.mutex.Unlock() if id == nil { @@ -342,7 +342,7 @@ func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) { return process, nil } -func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process { +func (self *ProcessMap) GetOrAddProcess(id *pilosa.GUID) *db.Process { process, err := self.GetProcess(id) if err != nil { process = db.NewProcess(id) @@ -351,7 +351,7 @@ func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process { return process } -func (self *ProcessMap) GetHost(id *util.GUID) (string, error) { +func (self *ProcessMap) GetHost(id *pilosa.GUID) (string, error) { self.mutex.Lock() defer self.mutex.Unlock() process, ok := self.nodes[*id] @@ -361,7 +361,7 @@ func (self *ProcessMap) GetHost(id *util.GUID) (string, error) { return process.Host(), nil } -func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) { +func (self *ProcessMap) GetPortTcp(id *pilosa.GUID) (int, error) { self.mutex.Lock() defer self.mutex.Unlock() process, ok := self.nodes[*id] @@ -371,7 +371,7 @@ func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) { return process.PortTcp(), nil } -func (self *ProcessMap) GetPortHttp(id *util.GUID) (int, error) { +func (self *ProcessMap) GetPortHttp(id *pilosa.GUID) (int, error) { self.mutex.Lock() defer self.mutex.Unlock() process, ok := self.nodes[*id] @@ -400,7 +400,7 @@ type ProcessMapper struct { commands chan ProcessMapperCommand namespace string - ID util.GUID + ID pilosa.GUID ProcessMap *ProcessMap TCPPort int @@ -431,7 +431,7 @@ func getKey(input string) string { return bits[len(bits)-1] } -func (self *ProcessMapper) getnode(u *util.GUID) *Node { +func (self *ProcessMapper) getnode(u *pilosa.GUID) *Node { return new(Node) } @@ -452,7 +452,7 @@ func (self *ProcessMapper) handlenode(node *etcd.Node) error { } if len(bits) >= 2 { id_string := bits[1] - id, err := util.ParseGUID(id_string) + id, err := pilosa.ParseGUID(id_string) if err != nil { return errors.New("Invalid GUID: " + id_string + " (" + key + ")") } diff --git a/core/http.go b/core/http.go index 95daba26f..e8d7d8f2c 100644 --- a/core/http.go +++ b/core/http.go @@ -7,9 +7,11 @@ import ( "encoding/gob" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/http/httputil" + "os" "reflect" "runtime" "strconv" @@ -20,9 +22,10 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" "github.com/gorilla/websocket" + "github.com/kr/s3/s3util" "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa/statsd" ) const DefaultRequestLogPath = "/tmp/set_bit_log" @@ -32,7 +35,7 @@ var RequestLogPath = DefaultRequestLogPath type WebService struct { end chan bool - ID util.GUID + ID pilosa.GUID Version string Port int DefaultDB string @@ -48,7 +51,7 @@ type WebService struct { } Pinger interface { - Ping(process_id *util.GUID) (*time.Duration, error) + Ping(process_id *pilosa.GUID) (*time.Duration, error) } TopologyMapper interface { @@ -127,7 +130,7 @@ func genFileName(id string) string { func flush(requests []LogRecord, id string, records_to_dump int) { dest := genFileName(id) - w, err := util.Create(dest) + w, err := createFile(dest) if err != nil { log.Warn("Error opening outfile ", dest) log.Warn(err) @@ -367,7 +370,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) { http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed) return } - util.SendInc("webservice_Query") + statsd.SendInc("webservice_Query") err := r.ParseForm() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -671,7 +674,7 @@ func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) { return } process_string := r.Form.Get("process") - process_id, err := util.ParseGUID(process_string) + process_id, err := pilosa.ParseGUID(process_string) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -838,3 +841,14 @@ func (self *WebService) HandleStatus(w http.ResponseWriter, r *http.Request) { `)) } + +func createFile(s string) (io.WriteCloser, error) { + if isURL(s) { + return s3util.Create(s, nil, nil) + } + return os.Create(s) +} + +func isURL(s string) bool { + return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") +} diff --git a/core/ping.go b/core/ping.go index dff2137f9..9c35d015c 100644 --- a/core/ping.go +++ b/core/ping.go @@ -4,20 +4,20 @@ import ( "encoding/gob" "time" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) type PingRequest struct { - Id *util.GUID - Source *util.GUID + Id *pilosa.GUID + Source *pilosa.GUID } type PongRequest struct { - Id *util.GUID + Id *pilosa.GUID } -func (self PongRequest) ResultId() *util.GUID { +func (self PongRequest) ResultId() *pilosa.GUID { return self.Id } func (self PongRequest) ResultData() interface{} { @@ -30,29 +30,29 @@ func init() { } type Pinger struct { - ID util.GUID + ID pilosa.GUID Hold interface { - Get(id *util.GUID, timeout int) (interface{}, error) + Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) } Transport interface { - Send(message *db.Message, host *util.GUID) + Send(message *db.Message, host *pilosa.GUID) } } -func NewPinger(id util.GUID) *Pinger { +func NewPinger(id pilosa.GUID) *Pinger { return &Pinger{ ID: id, } } -func (self *Pinger) Ping(process_id *util.GUID) (*time.Duration, error) { - id := util.RandomUUID() +func (self *Pinger) Ping(process_id *pilosa.GUID) (*time.Duration, error) { + id := pilosa.RandomUUID() ping := db.Message{Data: PingRequest{Id: &id, Source: &self.ID}} start := time.Now() self.Transport.Send(&ping, process_id) - _, err := self.Hold.Get(&id, 60) + _, err := self.Hold.Get(&id, 60*time.Second) if err != nil { return nil, err } diff --git a/core/remotebits.go b/core/remotebits.go index 3b489d171..4283befc7 100644 --- a/core/remotebits.go +++ b/core/remotebits.go @@ -4,23 +4,23 @@ import ( "encoding/gob" log "github.com/cihub/seelog" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) type RemoteSetBit struct { requests []remote_task - cluster map[*util.GUID][]BitmapRequestItem + cluster map[*pilosa.GUID][]BitmapRequestItem - ID util.GUID + ID pilosa.GUID ProcessMap *ProcessMap Hold interface { - Get(id *util.GUID, timeout int) (interface{}, error) + Get(id *pilosa.GUID, timeout int) (interface{}, error) } Transport interface { - Send(message *db.Message, host *util.GUID) + Send(message *db.Message, host *pilosa.GUID) } } @@ -32,12 +32,12 @@ func init() { type BitsRequest struct { Bits []BitmapRequestItem - ReturnProcessId util.GUID - QueryId util.GUID - DestProcessId util.GUID + ReturnProcessId pilosa.GUID + QueryId pilosa.GUID + DestProcessId pilosa.GUID } type BitmapRequestItem struct { - Fragment_id util.SUUID + Fragment_id pilosa.SUUID Bitmap_id uint64 Profile_id uint64 Filter uint64 @@ -47,7 +47,7 @@ type BitmapRequestItem struct { func NewRemoteSetBit() *RemoteSetBit { obj := new(RemoteSetBit) - obj.cluster = make(map[*util.GUID][]BitmapRequestItem) + obj.cluster = make(map[*pilosa.GUID][]BitmapRequestItem) return obj } @@ -55,7 +55,7 @@ func (self *RemoteSetBit) Request() { self.requests = make([]remote_task, 0) source_process, _ := self.ProcessMap.GetProcess(&self.ID) for process, request := range self.cluster { - random_id := util.RandomUUID() + random_id := pilosa.RandomUUID() msg := new(db.Message) msg.Data = BitsRequest{ Bits: request, @@ -74,7 +74,7 @@ func (self *RemoteSetBit) Request() { } type remote_task struct { - id util.GUID + id pilosa.GUID wait_time int } @@ -115,11 +115,11 @@ func (self *RemoteSetBit) Add(frag *db.Fragment, bitmap_id, profile_id, filter u } type BitsResponse struct { - Id *util.GUID + Id *pilosa.GUID Items []SBResult } -func (self *BitsResponse) ResultId() *util.GUID { +func (self *BitsResponse) ResultId() *pilosa.GUID { return self.Id } func (self *BitsResponse) ResultData() interface{} { diff --git a/db/db.go b/db/db.go index 158d8cbfa..cb4ee081e 100644 --- a/db/db.go +++ b/db/db.go @@ -3,7 +3,7 @@ package db import ( "encoding/gob" - . "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa" ) type Message struct { @@ -12,11 +12,11 @@ type Message struct { type Envelope struct { Message *Message - Host *GUID + Host *pilosa.GUID } type HoldResult interface { - ResultId() *GUID + ResultId() *pilosa.GUID ResultData() interface{} } diff --git a/db/topology.go b/db/topology.go index adda65714..a15b2a6eb 100644 --- a/db/topology.go +++ b/db/topology.go @@ -7,7 +7,7 @@ import ( log "github.com/cihub/seelog" "github.com/stathat/consistent" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa" ) // SupportedFrames is a list of frame types that are supported. @@ -20,23 +20,23 @@ var FragmentDoesNotExistError = errors.New("Fragment does not exist.") var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.") type Location struct { - ProcessId *util.GUID - FragmentId util.SUUID + ProcessId *pilosa.GUID + FragmentId pilosa.SUUID } type Process struct { - id *util.GUID + id *pilosa.GUID host string port_tcp int port_http int mutex sync.Mutex } -func NewProcess(id *util.GUID) *Process { +func NewProcess(id *pilosa.GUID) *Process { return &Process{id: id} } -func (self *Process) Id() util.GUID { +func (self *Process) Id() pilosa.GUID { self.mutex.Lock() defer self.mutex.Unlock() return *self.id @@ -302,7 +302,7 @@ func (self *FrameSliceIntersect) GetFragments() []*Fragment { return self.fragments } -func (d *Database) GetFragment(fragment_id util.SUUID) (*Fragment, error) { +func (d *Database) GetFragment(fragment_id pilosa.SUUID) (*Fragment, error) { for _, fsi := range d.frame_slice_intersects { f, err := fsi.GetFragment(fragment_id) if err == nil { @@ -311,7 +311,7 @@ func (d *Database) GetFragment(fragment_id util.SUUID) (*Fragment, error) { } return nil, FragmentDoesNotExistError } -func (self *FrameSliceIntersect) GetFragment(fragment_id util.SUUID) (*Fragment, +func (self *FrameSliceIntersect) GetFragment(fragment_id pilosa.SUUID) (*Fragment, error) { for _, fragment := range self.fragments { if fragment.id == fragment_id { @@ -323,7 +323,7 @@ func (self *FrameSliceIntersect) GetFragment(fragment_id util.SUUID) (*Fragment, func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) { self.fragments = append(self.fragments, fragment) - self.hashring.Add(util.SUUID_to_Hex(fragment.id)) + self.hashring.Add(pilosa.SUUID_to_Hex(fragment.id)) } ///////// FRAGMENTS @@ -333,11 +333,11 @@ func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) { // reference to the responsible node for that fragment. The node is in the form // ip:port type Fragment struct { - id util.SUUID + id pilosa.SUUID process *Process } -func (self *Fragment) GetId() util.SUUID { +func (self *Fragment) GetId() pilosa.SUUID { return self.id } @@ -345,7 +345,7 @@ func (self *Fragment) GetProcess() *Process { return self.process } -func (self *Fragment) GetProcessId() *util.GUID { +func (self *Fragment) GetProcessId() *pilosa.GUID { return self.process.id } @@ -373,7 +373,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment log.Warn(err) return nil, err } - frag_id := util.Hex_to_SUUID(frag_id_s) + frag_id := pilosa.Hex_to_SUUID(frag_id_s) return fsi.GetFragment(frag_id) } @@ -391,11 +391,11 @@ func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragme log.Warn(err) return nil, err } - frag_id := util.Hex_to_SUUID(frag_id_s) + frag_id := pilosa.Hex_to_SUUID(frag_id_s) return fsi.GetFragment(frag_id) } -func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) (*Fragment, error) { +func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) (*Fragment, error) { fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Warn(err) @@ -404,7 +404,7 @@ func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id util.SUUI return fsi.GetFragment(fragment_id) } -func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) *Fragment { +func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment { fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Warn("database.addFragment", err) @@ -415,7 +415,7 @@ func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id util.SUUI return &fragment } -func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) *Fragment { +func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment { d.mutex.Lock() defer d.mutex.Unlock() fragment, err := d.getFragment(frame, slice, fragment_id) diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index a64d2f2f0..db20fa9ce 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -1,6 +1,8 @@ package dispatch import ( + "time" + log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" "github.com/umbel/pilosa" @@ -8,7 +10,6 @@ import ( "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/executor" "github.com/umbel/pilosa/query" - "github.com/umbel/pilosa/util" ) type Dispatch struct { @@ -17,19 +18,19 @@ type Dispatch struct { } Hold interface { - Set(id *util.GUID, value interface{}, timeout int) + Set(id *pilosa.GUID, value interface{}, timeout time.Duration) } Index interface { - ClearBit(fragID util.SUUID, bitmapID uint64, pos uint64) (bool, error) - LoadBitmap(fragID util.SUUID, bitmapID uint64, compressedBitmap string, filter uint64) - SetBit(fragID util.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error) + ClearBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64) (bool, error) + LoadBitmap(fragID pilosa.SUUID, bitmapID uint64, compressedBitmap string, filter uint64) + SetBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error) TopFillBatch(args []pilosa.FillArgs) ([]pilosa.Pair, error) } Transport interface { Receive() *db.Message - Send(message *db.Message, host *util.GUID) + Send(message *db.Message, host *pilosa.GUID) } } @@ -84,7 +85,7 @@ func (d *Dispatch) Run() { case db.HoldResult: log.Trace("Dispatch.Run HoldResult") - d.Hold.Set(data.ResultId(), data.ResultData(), 30) + d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second) case query.PortableQueryStep: log.Trace("Dispatch.Run PortableQueryStep") @@ -95,7 +96,7 @@ func (d *Dispatch) Run() { go d.topFillHandler(message) case core.BitsResponse: - d.Hold.Set(data.ResultId(), data.ResultData(), 30) + d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second) default: spew.Dump(data) diff --git a/executor/executor.go b/executor/executor.go index 30ae1c776..ec7a48f90 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -11,35 +11,36 @@ import ( "github.com/umbel/pilosa/core" "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/query" - "github.com/umbel/pilosa/util" ) +const DefaultTimeout = 30 * time.Second + type Executor struct { inbox chan *db.Message - ID util.GUID + ID pilosa.GUID Cluster *db.Cluster ProcessMap *core.ProcessMap PluginsPath string Hold interface { - Get(id *util.GUID, timeout int) (interface{}, error) - Set(id *util.GUID, value interface{}, timeout int) + Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) + Set(id *pilosa.GUID, value interface{}, timeout time.Duration) } Index interface { - ClearBit(frag_id util.SUUID, bitmap_id uint64, pos uint64) (bool, error) - Count(frag_id util.SUUID, bitmap pilosa.BitmapHandle) (uint64, error) - Difference(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) - FromBytes(frag_id util.SUUID, bytes []byte) (pilosa.BitmapHandle, error) - Get(frag_id util.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error) - GetBytes(frag_id util.SUUID, bh pilosa.BitmapHandle) ([]byte, error) - Intersect(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) - Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error) - SetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) - TopN(frag_id util.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error) - TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]pilosa.Pair, error) - Union(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) + ClearBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64) (bool, error) + Count(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) (uint64, error) + Difference(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) + FromBytes(frag_id pilosa.SUUID, bytes []byte) (pilosa.BitmapHandle, error) + Get(frag_id pilosa.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error) + GetBytes(frag_id pilosa.SUUID, bh pilosa.BitmapHandle) ([]byte, error) + Intersect(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) + Range(frag_id pilosa.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error) + SetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) + TopN(frag_id pilosa.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error) + TopNAll(frag_id pilosa.SUUID, n int, categories []uint64) ([]pilosa.Pair, error) + Union(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) } TopologyMapper interface { @@ -47,11 +48,11 @@ type Executor struct { } Transport interface { - Send(*db.Message, *util.GUID) + Send(*db.Message, *pilosa.GUID) } } -func NewExecutor(id util.GUID) *Executor { +func NewExecutor(id pilosa.GUID) *Executor { log.Trace("NewExector") return &Executor{inbox: make(chan *db.Message)} } @@ -101,7 +102,7 @@ func (self *Executor) CountQueryStepHandler(msg *db.Message) { //spew.Dump("COUNT QUERYSTEP") qs := msg.Data.(query.CountQueryStep) input := qs.Input - value, _ := self.Hold.Get(input, util.TimeOut) + value, _ := self.Hold.Get(input, DefaultTimeout) var bh pilosa.BitmapHandle switch val := value.(type) { case pilosa.BitmapHandle: @@ -168,7 +169,7 @@ func (self *Executor) UnionQueryStepHandler(msg *db.Message) { var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { - value, _ := self.Hold.Get(input, util.TimeOut) + value, _ := self.Hold.Get(input, DefaultTimeout) switch val := value.(type) { case pilosa.BitmapHandle: handles = append(handles, val) @@ -208,7 +209,7 @@ func (self *Executor) IntersectQueryStepHandler(msg *db.Message) { var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { - value, _ := self.Hold.Get(input, util.TimeOut) + value, _ := self.Hold.Get(input, DefaultTimeout) switch val := value.(type) { case pilosa.BitmapHandle: handles = append(handles, val) @@ -248,7 +249,7 @@ func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) { var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { - value, _ := self.Hold.Get(input, util.TimeOut) + value, _ := self.Hold.Get(input, DefaultTimeout) switch val := value.(type) { case pilosa.BitmapHandle: handles = append(handles, val) @@ -288,9 +289,9 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { return_type := "bitmap-handles" var sum uint64 merge_map := make(map[uint64]uint64) - slice_map := make(map[uint64]map[util.SUUID]struct{}) - all_slice := make(map[util.SUUID]struct { - process util.GUID + slice_map := make(map[uint64]map[pilosa.SUUID]struct{}) + all_slice := make(map[pilosa.SUUID]struct { + process pilosa.GUID handle pilosa.BitmapHandle }) @@ -299,8 +300,8 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { num_parts := len(qs.Inputs) for _, input := range qs.Inputs { - go func(id *util.GUID, part chan interface{}) { - value, _ := self.Hold.Get(id, util.TimeOut) + go func(id *pilosa.GUID, part chan interface{}) { + value, _ := self.Hold.Get(id, DefaultTimeout) part <- value }(input, part) } @@ -331,13 +332,13 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { merge_map[pair.Key] += pair.Count mm, ok := slice_map[pair.Key] if !ok { - mm = make(map[util.SUUID]struct{}) + mm = make(map[pilosa.SUUID]struct{}) slice_map[pair.Key] = mm } mm[val.FragmentId] = e } all_slice[val.FragmentId] = struct { - process util.GUID + process pilosa.GUID handle pilosa.BitmapHandle }{val.ProcessId, val.HBitmap} check_pair = true @@ -393,7 +394,7 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { self.Transport.Send(&result_message, qs.Destination.ProcessId) } -func (self *Executor) SendRequest(process_id util.GUID, t *Task) { +func (self *Executor) SendRequest(process_id pilosa.GUID, t *Task) { args := make([]pilosa.FillArgs, len(t.f), len(t.f)) for _, v := range t.f { args = append(args, v) @@ -404,17 +405,17 @@ func (self *Executor) SendRequest(process_id util.GUID, t *Task) { self.Transport.Send(msg, &process_id) } -func (self *Executor) FetchMissing(tasks map[util.GUID]*Task) { +func (self *Executor) FetchMissing(tasks map[pilosa.GUID]*Task) { for k, v := range tasks { go self.SendRequest(k, v) } } -func (self *Executor) GatherResults(tasks map[util.GUID]*Task) map[uint64]uint64 { +func (self *Executor) GatherResults(tasks map[pilosa.GUID]*Task) map[uint64]uint64 { results := make(map[uint64]uint64) answers := make(chan []pilosa.Pair) for _, task := range tasks { - go func(id util.GUID) { + go func(id pilosa.GUID) { value, err := self.Hold.Get(&id, 10) //eiher need to be the frame process or the handler process? if value == nil { log.Warn("Bad TopN Result:", err) @@ -443,7 +444,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) { bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id) if err != nil { spew.Dump(err) - log.Error("GetQueryStepHandler1", util.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) + log.Error("GetQueryStepHandler1", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) log.Error("GetQueryStepHandler2", err) } @@ -454,7 +455,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) { bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh) if err != nil { spew.Dump(err) - log.Error("GetQueryStepHandlerr3", util.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) + log.Error("GetQueryStepHandlerr3", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) log.Error("GetQueryStepHandler4", err) } result = bm @@ -528,8 +529,8 @@ func (self *Executor) StashQueryStepHandler(msg *db.Message) { num_parts := len(qs.Inputs) for _, input := range qs.Inputs { - go func(id *util.GUID, part chan interface{}) { - value, _ := self.Hold.Get(id, util.TimeOut) + go func(id *pilosa.GUID, part chan interface{}) { + value, _ := self.Hold.Get(id, DefaultTimeout) part <- value }(input, part) } @@ -573,7 +574,7 @@ func (self *Executor) runQuery(database *db.Database, qry *query.Query) error { return err } process_id := process.Id() - fragment_id := util.SUUID(0) + fragment_id := pilosa.SUUID(0) destination := db.Location{ProcessId: &process_id, FragmentId: fragment_id} query_plan, err := query.QueryPlanForQuery(database, qry, &destination) @@ -685,34 +686,34 @@ func init() { } type TopNPackage struct { - ProcessId util.GUID - FragmentId util.SUUID + ProcessId pilosa.GUID + FragmentId pilosa.SUUID Pairs []pilosa.Pair HBitmap pilosa.BitmapHandle } type TopFill struct { Args []pilosa.FillArgs - ReturnProcessId util.GUID - QueryId util.GUID - DestProcessId util.GUID + ReturnProcessId pilosa.GUID + QueryId pilosa.GUID + DestProcessId pilosa.GUID } type Task struct { - processid util.GUID - f map[util.SUUID]pilosa.FillArgs - hold_id util.GUID + processid pilosa.GUID + f map[pilosa.SUUID]pilosa.FillArgs + hold_id pilosa.GUID } -func newtask(p util.GUID) *Task { +func newtask(p pilosa.GUID) *Task { result := new(Task) result.processid = p - result.f = make(map[util.SUUID]pilosa.FillArgs) - result.hold_id = util.RandomUUID() + result.f = make(map[pilosa.SUUID]pilosa.FillArgs) + result.hold_id = pilosa.RandomUUID() return result } -func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) { +func (t *Task) Add(frag pilosa.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) { fa, ok := t.f[frag] if !ok { fa = pilosa.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)} @@ -722,13 +723,13 @@ func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle } func BuildTask(merge_map map[uint64]uint64, - slice_map map[uint64]map[util.SUUID]struct{}, - total_fragments map[util.SUUID]struct { - process util.GUID + slice_map map[uint64]map[pilosa.SUUID]struct{}, + total_fragments map[pilosa.SUUID]struct { + process pilosa.GUID handle pilosa.BitmapHandle - }) map[util.GUID]*Task { + }) map[pilosa.GUID]*Task { - tasks := make(map[util.GUID]*Task) + tasks := make(map[pilosa.GUID]*Task) for bitmap_id, _ := range merge_map { //for all brands //for fragment_id, reported_fragments := range slice_map[bitmap_id] { //find missing fragments reporting_fragments := slice_map[bitmap_id] @@ -749,13 +750,13 @@ func BuildTask(merge_map map[uint64]uint64, } type hole struct { - process util.GUID + process pilosa.GUID handle pilosa.BitmapHandle - fragment util.SUUID + fragment pilosa.SUUID } -func missing(fids map[util.SUUID]struct{}, all map[util.SUUID]struct { - process util.GUID +func missing(fids map[pilosa.SUUID]struct{}, all map[pilosa.SUUID]struct { + process pilosa.GUID handle pilosa.BitmapHandle }) []hole { results := make([]hole, 0, 0) @@ -769,7 +770,7 @@ func missing(fids map[util.SUUID]struct{}, all map[util.SUUID]struct { return results } -func (self *TopFill) GetId() *util.GUID { +func (self *TopFill) GetId() *pilosa.GUID { return &self.QueryId } func (self *TopFill) GetLocation() *db.Location { diff --git a/fragment_container.go b/fragment_container.go index df25f34ec..e51310d45 100644 --- a/fragment_container.go +++ b/fragment_container.go @@ -11,7 +11,7 @@ import ( log "github.com/cihub/seelog" "github.com/golang/groupcache/lru" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa/statsd" ) // DefaultBackend is the default data storage layer. @@ -22,20 +22,20 @@ var Backend = DefaultBackend var LevelDBPath string type FragmentContainer struct { - fragments map[util.SUUID]*Fragment + fragments map[SUUID]*Fragment mutex *sync.Mutex } func NewFragmentContainer() *FragmentContainer { f := new(FragmentContainer) - f.fragments = make(map[util.SUUID]*Fragment) + f.fragments = make(map[SUUID]*Fragment) f.mutex = &sync.Mutex{} return f } type BitmapHandle uint64 type FillArgs struct { - Frag_id util.SUUID + Frag_id SUUID Handle BitmapHandle Bitmaps []uint64 } @@ -59,7 +59,7 @@ func (self *FragmentContainer) Shutdown() { log.Warn("Container Shutdown Complete") } -func (self *FragmentContainer) LoadBitmap(frag_id util.SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) { +func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) { log.Trace("LoadBitmap") if fragment, found := self.GetFragment(frag_id); found { request := NewLoader(bitmap_id, compressed_bitmap, filter) @@ -69,7 +69,7 @@ func (self *FragmentContainer) LoadBitmap(frag_id util.SUUID, bitmap_id uint64, } } -func (self *FragmentContainer) GetFragment(frag_id util.SUUID) (*Fragment, bool) { +func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) { log.Trace("index.GetFragment") self.mutex.Lock() c, v := self.fragments[frag_id] @@ -77,7 +77,7 @@ func (self *FragmentContainer) GetFragment(frag_id util.SUUID) (*Fragment, bool) return c, v } -func (self *FragmentContainer) Stats(frag_id util.SUUID) interface{} { +func (self *FragmentContainer) Stats(frag_id SUUID) interface{} { if fragment, found := self.GetFragment(frag_id); found { request := NewStats() fragment.requestChan <- request @@ -85,7 +85,7 @@ func (self *FragmentContainer) Stats(frag_id util.SUUID) interface{} { } return nil } -func (self *FragmentContainer) Empty(frag_id util.SUUID) (BitmapHandle, error) { +func (self *FragmentContainer) Empty(frag_id SUUID) (BitmapHandle, error) { log.Trace("index.Empty") if fragment, found := self.GetFragment(frag_id); found { request := NewEmpty() @@ -95,97 +95,97 @@ func (self *FragmentContainer) Empty(frag_id util.SUUID) (BitmapHandle, error) { return 0, errors.New("Invalid Bitmap Handle Empty") } -func (self *FragmentContainer) Intersect(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) { +func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { log.Trace("index.Intersect") if fragment, found := self.GetFragment(frag_id); found { request := NewIntersect(bh) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Intersect", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Intersect", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle Intersect") } -func (self *FragmentContainer) Union(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) { +func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { log.Trace("index.Union") if fragment, found := self.GetFragment(frag_id); found { request := NewUnion(bh) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Union", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Union", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle Union") } -func (self *FragmentContainer) Difference(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) { +func (self *FragmentContainer) Difference(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { log.Trace("index.Difference") if fragment, found := self.GetFragment(frag_id); found { request := NewDifference(bh) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Difference", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Difference", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle Diff") } -func (self *FragmentContainer) Get(frag_id util.SUUID, bitmap_id uint64) (BitmapHandle, error) { +func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) { log.Trace("index.Get") if fragment, found := self.GetFragment(frag_id); found { request := NewGet(bitmap_id) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Get", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Get", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle Get") } -func (self *FragmentContainer) Mask(frag_id util.SUUID, start, end uint64) (BitmapHandle, error) { +func (self *FragmentContainer) Mask(frag_id SUUID, start, end uint64) (BitmapHandle, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewMask(start, end) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Mask", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Mask", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (BitmapHandle, error) { +func (self *FragmentContainer) Range(frag_id SUUID, bitmap_id uint64, start, end time.Time) (BitmapHandle, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewRange(bitmap_id, start, end) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Range", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Range", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) TopN(frag_id util.SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) { +func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) { log.Trace("index.TopN") if fragment, found := self.GetFragment(frag_id); found { request := NewTopN(bh, n, categories) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_TopN", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_TopN", result.exec_time.Nanoseconds()) return result.answer.([]Pair), nil } - return nil, errors.New(fmt.Sprintf("Fragment not found:%s", util.SUUID_to_Hex(frag_id))) + return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id))) } -func (self *FragmentContainer) TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]Pair, error) { +func (self *FragmentContainer) TopNAll(frag_id SUUID, n int, categories []uint64) ([]Pair, error) { log.Trace("index.TopNAll") if fragment, found := self.GetFragment(frag_id); found { request := NewTopNAll(n, categories) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_TopNAll", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_TopNAll", result.exec_time.Nanoseconds()) return result.answer.([]Pair), nil } - return nil, errors.New(fmt.Sprintf("Fragment not found:%s", util.SUUID_to_Hex(frag_id))) + return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id))) } func (self *FragmentContainer) TopFillBatch(args []FillArgs) ([]Pair, error) { @@ -216,87 +216,87 @@ func (self *FragmentContainer) TopFillFragment(arg FillArgs) ([]Pair, error) { request := NewTopFill(arg) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_TopFillFragment", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_TopFillFragment", result.exec_time.Nanoseconds()) return result.answer.([]Pair), nil } return nil, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) GetList(frag_id util.SUUID, bitmap_id []uint64) ([]BitmapHandle, error) { +func (self *FragmentContainer) GetList(frag_id SUUID, bitmap_id []uint64) ([]BitmapHandle, error) { log.Trace("index.GetList") if fragment, found := self.GetFragment(frag_id); found { request := NewGetList(bitmap_id) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_GetList", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_GetList", result.exec_time.Nanoseconds()) return result.answer.([]BitmapHandle), nil } return nil, errors.New("Invalid Bitmap Handle GetList") } -func (self *FragmentContainer) Count(frag_id util.SUUID, bitmap BitmapHandle) (uint64, error) { +func (self *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) { log.Trace("index.Count") if fragment, found := self.GetFragment(frag_id); found { request := NewCount(bitmap) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_Count", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_Count", result.exec_time.Nanoseconds()) return result.answer.(uint64), nil } return 0, errors.New("Invalid Bitmap Handle Count") } -func (self *FragmentContainer) GetBytes(frag_id util.SUUID, bh BitmapHandle) ([]byte, error) { +func (self *FragmentContainer) GetBytes(frag_id SUUID, bh BitmapHandle) ([]byte, error) { log.Trace("index.GetBytes") if fragment, found := self.GetFragment(frag_id); found { request := NewGetBytes(bh) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_GetBytes", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_GetBytes", result.exec_time.Nanoseconds()) return result.answer.([]byte), nil } return nil, errors.New("Invalid Bitmap Handle GetBytes") } -func (self *FragmentContainer) FromBytes(frag_id util.SUUID, bytes []byte) (BitmapHandle, error) { +func (self *FragmentContainer) FromBytes(frag_id SUUID, bytes []byte) (BitmapHandle, error) { log.Trace("index.FromBytes") if fragment, found := self.GetFragment(frag_id); found { request := NewFromBytes(bytes) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_FromBytes", result.exec_time.Nanoseconds()) + statsd.SendTimer("fragmant_container_FromBytes", result.exec_time.Nanoseconds()) return result.answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle FromBytes") } -func (self *FragmentContainer) SetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) { +func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) { log.Trace("SetBit", frag_id, bitmap_id, pos, category) if fragment, found := self.GetFragment(frag_id); found { request := NewSetBit(bitmap_id, pos, category) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_SetBit", result.exec_time.Nanoseconds()) - util.SendInc("fragmant_container_SetBit") + statsd.SendTimer("fragmant_container_SetBit", result.exec_time.Nanoseconds()) + statsd.SendInc("fragmant_container_SetBit") return result.answer.(bool), nil } return false, errors.New("Invalid Bitmap Handle SetBit") } -func (self *FragmentContainer) ClearBit(frag_id util.SUUID, bitmap_id uint64, pos uint64) (bool, error) { +func (self *FragmentContainer) ClearBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) { log.Trace("ClearBit", frag_id, bitmap_id, pos) if fragment, found := self.GetFragment(frag_id); found { request := NewClearBit(bitmap_id, pos) fragment.requestChan <- request result := request.Response() - util.SendTimer("fragmant_container_ClearBit", result.exec_time.Nanoseconds()) - util.SendInc("fragmant_container_ClearBit") + statsd.SendTimer("fragmant_container_ClearBit", result.exec_time.Nanoseconds()) + statsd.SendInc("fragmant_container_ClearBit") return result.answer.(bool), nil } return false, errors.New("Invalid Bitmap Handle ClearBit") } -func (self *FragmentContainer) Clear(frag_id util.SUUID) (bool, error) { +func (self *FragmentContainer) Clear(frag_id SUUID) (bool, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewClear() fragment.requestChan <- request @@ -313,13 +313,13 @@ func dumpHandlesToLog() { log.Warn(mesg) } -func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id util.SUUID) { +func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) { self.mutex.Lock() defer self.mutex.Unlock() _, ok := self.fragments[id] if !ok { // dumpHandlesToLog() - log.Warn("ADD FRAGMENT", frame, db, slice, util.SUUID_to_Hex(id)) + log.Warn("ADD FRAGMENT", frame, db, slice, SUUID_to_Hex(id)) f := NewFragment(id, db, slice, frame) loader := make(chan Command) self.fragments[id] = f @@ -345,7 +345,7 @@ type Pilosa interface { type Fragment struct { requestChan chan Command - fragment_id util.SUUID + fragment_id SUUID impl Pilosa counter uint64 slice int @@ -356,7 +356,7 @@ type Fragment struct { queue_size int } -func NewFragment(frag_id util.SUUID, db string, slice int, frame string) *Fragment { +func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment { var impl Pilosa log.Warn(fmt.Sprintf("XXXXXXXXXXXXXXXXXXXXXXXXXXX(%s)", frame)) diff --git a/fragment_container_test.go b/fragment_container_test.go index 1bda3490a..20d0331d3 100644 --- a/fragment_container_test.go +++ b/fragment_container_test.go @@ -5,7 +5,6 @@ import ( "github.com/umbel/pilosa" _ "github.com/umbel/pilosa/storage" - "github.com/umbel/pilosa/util" ) func init() { @@ -18,7 +17,7 @@ func TestFragmentContainer_Get(t *testing.T) { fc.AddFragment("25", "general", 0, 1) fc.MustClear(1) - if bh, err := fc.Get(util.SUUID(1), 1234); err != nil { + if bh, err := fc.Get(pilosa.SUUID(1), 1234); err != nil { t.Fatal(err) } else if bh == 0 { t.Fatal("expected non-zero bitmap handle") @@ -49,20 +48,20 @@ func TestFragmentContainer_SetBit(t *testing.T) { // Ensure the number of bits on a bitmap can be counted. func TestFragmentContainer_Count(t *testing.T) { fc := NewFragmentContainer() - fc.AddFragment("25", "general", 0, util.SUUID(1)) + fc.AddFragment("25", "general", 0, pilosa.SUUID(1)) // Set a bit on the bitmap. bi1 := uint64(1234) - if changed, err := fc.SetBit(util.SUUID(1), bi1, 1, 0); err != nil { + if changed, err := fc.SetBit(pilosa.SUUID(1), bi1, 1, 0); err != nil { t.Fatal(err) } else if changed == false { t.Fatal("expected change") } // Verify that one bit is set. - if bh, err := fc.Get(util.SUUID(1), bi1); err != nil { + if bh, err := fc.Get(pilosa.SUUID(1), bi1); err != nil { t.Fatal(err) - } else if n, err := fc.Count(util.SUUID(1), bh); err != nil { + } else if n, err := fc.Count(pilosa.SUUID(1), bh); err != nil { t.Fatal(err) } else if n != 1 { t.Fatalf("unexpected count: %d", n) @@ -273,7 +272,7 @@ func NewFragmentContainer() *FragmentContainer { } // MustGet retrieves a bitmap by id. Panic on error. -func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) pilosa.BitmapHandle { +func (fc *FragmentContainer) MustGet(frag_id pilosa.SUUID, bitmap_id uint64) pilosa.BitmapHandle { bh, err := fc.Get(frag_id, bitmap_id) if err != nil { panic(err) @@ -282,7 +281,7 @@ func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) pilos } // MustSetBit sets a bit in a bitmap. Panic on error. -func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) bool { +func (fc *FragmentContainer) MustSetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) bool { changed, err := fc.SetBit(frag_id, bitmap_id, pos, category) if err != nil { panic(err) @@ -291,7 +290,7 @@ func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, po } // MustClear clears a fragment. Panic on error. -func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool { +func (fc *FragmentContainer) MustClear(fragmentID pilosa.SUUID) bool { v, err := fc.Clear(fragmentID) if err != nil { panic(err) @@ -300,7 +299,7 @@ func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool { } // MustCount returns the number of set bits in a bitmap. Panic on error. -func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap pilosa.BitmapHandle) uint64 { +func (fc *FragmentContainer) MustCount(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) uint64 { v, err := fc.Count(frag_id, bitmap) if err != nil { panic(err) diff --git a/general.go b/general.go index e1e4cd1ad..e7f2f3eef 100644 --- a/general.go +++ b/general.go @@ -6,7 +6,6 @@ import ( log "github.com/cihub/seelog" "github.com/golang/groupcache/lru" - "github.com/umbel/pilosa/util" ) type General struct { @@ -90,7 +89,7 @@ func (self *General) getFileName() string { func (self *General) Persist() error { log.Warn("General Persist") - w, err := util.Create(self.getFileName()) + w, err := createFile(self.getFileName()) if err != nil { log.Warn("Error saving:", err) return err @@ -111,7 +110,7 @@ func (self *General) Persist() error { func (self *General) Load(requestChan chan Command, f *Fragment) { log.Warn("General Load") - r, err := util.Open(self.getFileName()) + r, err := openFile(self.getFileName()) if err != nil { log.Warn("NO General Init File:", self.getFileName()) return diff --git a/hold/hold.go b/hold/hold.go index 33fbe810a..d05d81ac9 100644 --- a/hold/hold.go +++ b/hold/hold.go @@ -5,39 +5,39 @@ import ( "time" log "github.com/cihub/seelog" - . "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa" ) type holdchan chan interface{} type gethold struct { - id *GUID + id *pilosa.GUID reply chan holdchan } type delhold struct { - id *GUID + id *pilosa.GUID } type Holder struct { - data map[GUID]holdchan + data map[pilosa.GUID]holdchan getchan chan gethold delchan chan delhold } func NewHolder() *Holder { return &Holder{ - data: make(map[GUID]holdchan), + data: make(map[pilosa.GUID]holdchan), getchan: make(chan gethold), delchan: make(chan delhold), } } -func (self *Holder) DelChan(id *GUID) { +func (self *Holder) DelChan(id *pilosa.GUID) { log.Trace("Holder.DelChan", id) req := delhold{id} self.delchan <- req } -func (self *Holder) GetChan(id *GUID) holdchan { +func (self *Holder) GetChan(id *pilosa.GUID) holdchan { log.Trace("Holder.GetChan", id) reply := make(chan holdchan) req := gethold{id, reply} @@ -45,25 +45,25 @@ func (self *Holder) GetChan(id *GUID) holdchan { return <-reply } -func (self *Holder) Get(id *GUID, timeout int) (interface{}, error) { - log.Trace("Holder.Get", id, timeout) +func (self *Holder) Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) { + log.Trace("Holder.Get", id, timeout.String()) ch := self.GetChan(id) select { case val := <-ch: return val, nil - case <-time.After(time.Duration(timeout) * time.Second): + case <-time.After(timeout): self.DelChan(id) return nil, errors.New("Timeout getting from holder") } } -func (self *Holder) Set(id *GUID, value interface{}, timeout int) { - log.Trace("Holder.Set", id, value, timeout) +func (self *Holder) Set(id *pilosa.GUID, value interface{}, timeout time.Duration) { + log.Trace("Holder.Set", id, value, timeout.String()) ch := self.GetChan(id) go func() { select { case ch <- value: - case <-time.After(time.Duration(timeout) * time.Second): + case <-time.After(timeout): } self.DelChan(id) }() diff --git a/util/id.go b/id.go similarity index 99% rename from util/id.go rename to id.go index f297154a0..e747d2186 100644 --- a/util/id.go +++ b/id.go @@ -1,4 +1,4 @@ -package util +package pilosa import ( "bytes" @@ -44,6 +44,7 @@ func SUUID_to_Hex(a SUUID) string { binary.Write(buf, binary.BigEndian, a) return hex.EncodeToString(buf.Bytes()) } + func Hex_to_SUUID(str string) SUUID { l := len(str) var m string diff --git a/util/id_test.go b/id_test.go similarity index 71% rename from util/id_test.go rename to id_test.go index 8bd1aba2c..f4c047ca3 100644 --- a/util/id_test.go +++ b/id_test.go @@ -1,20 +1,22 @@ -package util +package pilosa_test import ( "fmt" "testing" + + "github.com/umbel/pilosa" ) // Ensure id can be parsed from string. func TestId_Small(t *testing.T) { - if v := Hex_to_SUUID("1"); v != 1 { + if v := pilosa.Hex_to_SUUID("1"); v != 1 { t.Fatalf("unexpected SUUID: %v", v) } } // Ensure generated IDs are unique. func TestId_Unique(t *testing.T) { - a, b := Id(), Id() + a, b := pilosa.Id(), pilosa.Id() if a == b { t.Fatalf("ids should be unique: %v != %v", a, b) } @@ -22,8 +24,8 @@ func TestId_Unique(t *testing.T) { // Ensure ids can be converted to and from hex. func TestId_Hex(t *testing.T) { - a := Id() - b := Hex_to_SUUID(SUUID_to_Hex(a)) + a := pilosa.Id() + b := pilosa.Hex_to_SUUID(pilosa.SUUID_to_Hex(a)) if a != b { t.Fatalf("ids not equal: %v != %v", a, b) } @@ -32,18 +34,18 @@ func TestId_Hex(t *testing.T) { // Ensure ids can be generated in sequence. func TestId_Multiple(t *testing.T) { for i := 0; i < 10; i++ { - println(SUUID_to_Hex(Id())) + println(pilosa.SUUID_to_Hex(pilosa.Id())) } } // Ensure a random UUID can be converted to a string. func TestRandomUUID_String(t *testing.T) { - fmt.Println(RandomUUID().String()) + fmt.Println(pilosa.RandomUUID().String()) } func BenchmarkId(b *testing.B) { // run the Fib function b.N times for n := 0; n < b.N; n++ { - Id() + pilosa.Id() } } diff --git a/query/parser.go b/query/parser.go index 2cc0e6107..63a68487f 100644 --- a/query/parser.go +++ b/query/parser.go @@ -9,7 +9,6 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" "github.com/umbel/pilosa" - "github.com/umbel/pilosa/util" ) var InvalidQueryError = errors.New("Invalid query format.") @@ -53,7 +52,7 @@ func (self *QueryParser) Parse() (query *Query, err error) { }() var token *Token - id := util.RandomUUID() + id := pilosa.RandomUUID() query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})} token = self.next() @@ -215,7 +214,7 @@ ArgLoop: if err != nil { return nil, fmt.Errorf("Expecting fragment id! (%v)", err) } - recallArgs.Add(util.SUUID(i)) //constructs a new arg + recallArgs.Add(pilosa.SUUID(i)) //constructs a new arg } else { i, err := strconv.ParseUint(token.Text, 10, 64) if err != nil { diff --git a/query/planner.go b/query/planner.go index d4132c958..0f7d962de 100644 --- a/query/planner.go +++ b/query/planner.go @@ -11,11 +11,10 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) type PortableQueryStep interface { - GetId() *util.GUID + GetId() *pilosa.GUID GetLocation() *db.Location } @@ -59,13 +58,13 @@ func (self *FragmentNotFound) Error() string { /////////////////////////////////////////////////////////////////////////////////////////////////// type BaseQueryStep struct { - Id *util.GUID + Id *pilosa.GUID Operation string Location *db.Location Destination *db.Location } -func (self *BaseQueryStep) GetId() *util.GUID { +func (self *BaseQueryStep) GetId() *pilosa.GUID { log.Trace("BaseQueryStep", *self.Id) return self.Id } @@ -76,7 +75,7 @@ func (self *BaseQueryStep) GetLocation() *db.Location { func (self *BaseQueryStep) LocIsDest() bool { log.Trace("BaseQueryStep.LocIsDest") - if util.Equal(self.Location.ProcessId, self.Destination.ProcessId) && + if pilosa.Equal(self.Location.ProcessId, self.Destination.ProcessId) && self.Location.FragmentId == self.Destination.FragmentId { log.Trace("BaseQueryStep.LocIsDest Return true") return true @@ -86,11 +85,11 @@ func (self *BaseQueryStep) LocIsDest() bool { } type BaseQueryResult struct { - Id *util.GUID + Id *pilosa.GUID Data interface{} } -func (self *BaseQueryResult) ResultId() *util.GUID { +func (self *BaseQueryResult) ResultId() *pilosa.GUID { log.Trace("BaseQueryStep.ResultId", self) return self.Id @@ -106,7 +105,7 @@ func (self *BaseQueryResult) ResultData() interface{} { /////////////////////////////////////////////////////////////////////////////////////////////////// type CountQueryStep struct { *BaseQueryStep - Input *util.GUID + Input *pilosa.GUID } type CountQueryResult struct { @@ -130,7 +129,7 @@ func (qt *CountQueryTree) getLocation(d *db.Database) (*db.Location, error) { /////////////////////////////////////////////////////////////////////////////////////////////////// type TopNQueryStep struct { *BaseQueryStep - Input *util.GUID + Input *pilosa.GUID Filters []uint64 N int Frame string @@ -173,7 +172,7 @@ func (qt *TopNQueryTree) getLocation(d *db.Database) (*db.Location, error) { /////////////////////////////////////////////////////////////////////////////////////////////////// type UnionQueryStep struct { *BaseQueryStep - Inputs []*util.GUID + Inputs []*pilosa.GUID } type UnionQueryResult struct { @@ -206,7 +205,7 @@ func (qt *UnionQueryTree) getLocation(d *db.Database) (*db.Location, error) { /////////////////////////////////////////////////////////////////////////////////////////////////// type IntersectQueryStep struct { *BaseQueryStep - Inputs []*util.GUID + Inputs []*pilosa.GUID } type IntersectQueryResult struct { @@ -239,7 +238,7 @@ func (qt *IntersectQueryTree) getLocation(d *db.Database) (*db.Location, error) /////////////////////////////////////////////////////////////////////////////////////////////////// type DifferenceQueryStep struct { *BaseQueryStep - Inputs []*util.GUID + Inputs []*pilosa.GUID } type DifferenceQueryResult struct { @@ -272,7 +271,7 @@ func (qt *DifferenceQueryTree) getLocation(d *db.Database) (*db.Location, error) /////////////////////////////////////////////////////////////////////////////////////////////////// type CatQueryStep struct { *BaseQueryStep - Inputs []*util.GUID + Inputs []*pilosa.GUID N int } type Appendable interface { @@ -606,18 +605,18 @@ func (self *QueryPlanner) buildTree(query *Query, slice int) (QueryTree, error) } // Produces flattened QueryPlan from QueryTree input -func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Location) (*QueryPlan, error) { +func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Location) (*QueryPlan, error) { log.Trace("QueryPlanner.flatten", self, qt, id, location) plan := QueryPlan{} if cat, ok := qt.(*CatQueryTree); ok { - inputs := make([]*util.GUID, len(cat.subqueries)) + inputs := make([]*pilosa.GUID, len(cat.subqueries)) loc, err := cat.getLocation(self.Database) if err != nil { return nil, err } step := CatQueryStep{&BaseQueryStep{id, "cat", loc, location}, inputs, cat.N} for index, subq := range cat.subqueries { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -628,14 +627,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca plan = append(plan, step) } else if stash, ok := qt.(*StashQueryTree); ok { - inputs := make([]*util.GUID, len(stash.subqueries)) + inputs := make([]*pilosa.GUID, len(stash.subqueries)) loc, err := stash.getLocation(self.Database) if err != nil { return nil, err } step := StashQueryStep{&BaseQueryStep{id, "stash", loc, location}, inputs, stash.N} for index, subq := range stash.subqueries { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -645,14 +644,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca } plan = append(plan, step) } else if union, ok := qt.(*UnionQueryTree); ok { - inputs := make([]*util.GUID, len(union.subqueries)) + inputs := make([]*pilosa.GUID, len(union.subqueries)) loc, err := union.getLocation(self.Database) if err != nil { return nil, err } step := UnionQueryStep{&BaseQueryStep{id, "union", loc, location}, inputs} for index, subq := range union.subqueries { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -662,14 +661,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca } plan = append(plan, step) } else if intersect, ok := qt.(*IntersectQueryTree); ok { - inputs := make([]*util.GUID, len(intersect.subqueries)) + inputs := make([]*pilosa.GUID, len(intersect.subqueries)) loc, err := intersect.getLocation(self.Database) if err != nil { return nil, err } step := IntersectQueryStep{&BaseQueryStep{id, "intersect", loc, location}, inputs} for index, subq := range intersect.subqueries { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -679,14 +678,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca } plan = append(plan, step) } else if difference, ok := qt.(*DifferenceQueryTree); ok { - inputs := make([]*util.GUID, len(difference.subqueries)) + inputs := make([]*pilosa.GUID, len(difference.subqueries)) loc, err := difference.getLocation(self.Database) if err != nil { return nil, err } step := DifferenceQueryStep{&BaseQueryStep{id, "difference", loc, location}, inputs} for index, subq := range difference.subqueries { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -728,7 +727,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca plan := QueryPlan{step} return &plan, nil } else if cnt, ok := qt.(*CountQueryTree); ok { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() loc, err := cnt.getLocation(self.Database) if err != nil { return nil, err @@ -741,7 +740,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca plan = append(plan, *subq_steps...) plan = append(plan, step) } else if topn, ok := qt.(*TopNQueryTree); ok { - sub_id := util.RandomUUID() + sub_id := pilosa.RandomUUID() loc, err := topn.getLocation(self.Database) if err != nil { return nil, err @@ -764,7 +763,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca } // Transforms Query into QueryTree and flattens to QueryPlan object -func (self *QueryPlanner) Plan(query *Query, id *util.GUID, destination *db.Location) (*QueryPlan, error) { +func (self *QueryPlanner) Plan(query *Query, id *pilosa.GUID, destination *db.Location) (*QueryPlan, error) { log.Trace("QueryPlanner.Plan", self, query, id, destination) queryTree, err := self.buildTree(query, -1) if err != nil { @@ -844,7 +843,7 @@ type FillResult struct { /////////////////////////////////////////////////////////////////////////////////////////////////// type CacheItem struct { - FragmentId util.SUUID + FragmentId pilosa.SUUID Handle pilosa.BitmapHandle } @@ -857,7 +856,7 @@ func NewStash() Stash { return Stash{make([]CacheItem, 0), false} } -func (st *Stash) Add(i util.SUUID) { +func (st *Stash) Add(i pilosa.SUUID) { item := CacheItem{i, 0} st.Stash = append(st.Stash, item) st.incomplete = true @@ -870,7 +869,7 @@ func (st *Stash) Assign(i pilosa.BitmapHandle) { type StashQueryStep struct { *BaseQueryStep - Inputs []*util.GUID + Inputs []*pilosa.GUID N int } diff --git a/query/planner_test.go b/query/planner_test.go index d5f268468..bc71fc4eb 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -48,7 +48,7 @@ func TestQueryPlanner_Plan_Get(t *testing.T) { if step := (*plan)[2].(query.CatQueryStep); step.Operation != "cat" { t.Fatalf("unexpected step(2) operation: %s", step.Operation) - } else if !reflect.DeepEqual(step.Inputs, []*util.GUID{ + } else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{ (*plan)[0].(query.GetQueryStep).Id, (*plan)[1].(query.GetQueryStep).Id, }) { @@ -212,7 +212,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) { // Third step should union the first two steps. if step := (*plan)[2].(query.UnionQueryStep); step.Operation != "union" { t.Fatalf("unexpected step(2) operation: %s", step.Operation) - } else if !reflect.DeepEqual(step.Inputs, []*util.GUID{ + } else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{ (*plan)[0].(query.GetQueryStep).Id, (*plan)[1].(query.GetQueryStep).Id, }) { @@ -240,7 +240,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) { // Sixth step should union the previous two steps. if step := (*plan)[5].(query.UnionQueryStep); step.Operation != "union" { t.Fatalf("unexpected step(5) operation: %s", step.Operation) - } else if !reflect.DeepEqual(step.Inputs, []*util.GUID{ + } else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{ (*plan)[3].(query.GetQueryStep).Id, (*plan)[4].(query.GetQueryStep).Id, }) { @@ -250,7 +250,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) { // Final step should concatenate the two union steps. if step := (*plan)[6].(query.CatQueryStep); step.Operation != "cat" { t.Fatalf("unexpected step(6) operation: %s", step.Operation) - } else if !reflect.DeepEqual(step.Inputs, []*util.GUID{ + } else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{ (*plan)[2].(query.UnionQueryStep).Id, (*plan)[5].(query.UnionQueryStep).Id, }) { diff --git a/query/query.go b/query/query.go index dbc39e7ce..f012d4d27 100644 --- a/query/query.go +++ b/query/query.go @@ -4,8 +4,8 @@ import ( "strings" log "github.com/cihub/seelog" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) type QueryInput interface{} @@ -17,13 +17,13 @@ type QueryResults struct { type PqlList []PqlListItem type PqlListItem struct { - Id *util.GUID + Id *pilosa.GUID Label string PQL string } type Query struct { - Id *util.GUID + Id *pilosa.GUID Operation string Args map[string]interface{} Subqueries []Query @@ -68,7 +68,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) (*QueryPlan, error) { log.Trace("QueryPlanForQuery", database, query, destination) query_planner := QueryPlanner{Database: database, Query: query} - id := util.RandomUUID() + id := pilosa.RandomUUID() query_plan, err := query_planner.Plan(query, &id, destination) if err != nil { return nil, err diff --git a/util/statd.go b/statsd/statd.go similarity index 69% rename from util/statd.go rename to statsd/statd.go index 24951495f..4ca32bed6 100644 --- a/util/statd.go +++ b/statsd/statd.go @@ -1,4 +1,4 @@ -package util +package statsd import ( "time" @@ -7,11 +7,11 @@ import ( log "github.com/cihub/seelog" ) -// DefaultStatsdHost is the default host to send statsd data to. -const DefaultStatsdHost = "127.0.0.1:8125" +// DefaultHost is the default host to send statsd data to. +const DefaultHost = "127.0.0.1:8125" -// StatsdHost is the host to send statsd data to. -var StatsdHost = DefaultStatsdHost +// Host is the host to send statsd data to. +var Host = DefaultHost type args struct { stat string @@ -19,19 +19,17 @@ type args struct { rate float32 } -var ( - timer chan args - count chan string - end chan bool -) +var timer chan args +var count chan string +var end chan bool -func SetupStatsd() { +func Setup() { timer = make(chan args, 32768) count = make(chan string, 32768) end = make(chan bool) - log.Warn("New Stats", StatsdHost) - stats, _ := statsd.New(StatsdHost, "") + log.Warn("New Stats", Host) + stats, _ := statsd.New(Host, "") go func() { for { diff --git a/storage.go b/storage.go index 64ac66ff3..679daab67 100644 --- a/storage.go +++ b/storage.go @@ -1,9 +1,5 @@ package pilosa -import ( - "github.com/umbel/pilosa/util" -) - // Storage represents type Storage interface { Open() error @@ -47,7 +43,7 @@ type StorageOptions struct { DB string Slice int Frame string - FragmentID util.SUUID + FragmentID SUUID LevelDBPath string } diff --git a/storage/cassandra/cassandra.go b/storage/cassandra/cassandra.go index 4186a6f2a..9ec0adcba 100644 --- a/storage/cassandra/cassandra.go +++ b/storage/cassandra/cassandra.go @@ -6,7 +6,7 @@ import ( log "github.com/cihub/seelog" "github.com/gocql/gocql" "github.com/umbel/pilosa" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa/statsd" ) func init() { @@ -124,8 +124,8 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*p lastKey = chunkKey } - util.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds()) - util.SendInc("cassandra_storage_Read") + statsd.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds()) + statsd.SendInc("cassandra_storage_Read") // Set total bits set. bm.SetCount(uint64(count)) @@ -148,8 +148,8 @@ func (s *Storage) endBatch() { start := time.Now() s.Flush() - util.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds()) - util.SendInc("cassandra_storage_Write") + statsd.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds()) + statsd.SendInc("cassandra_storage_Write") } // Flush flushes the current batch to storage. @@ -168,7 +168,7 @@ func (s *Storage) Flush() { s.batchTime = time.Now() s.batchN = 0 - util.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds()) + statsd.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds()) } // Store saves a bitmap to storage. @@ -202,7 +202,7 @@ func (s *Storage) StoreBlock(bid uint64, db string, frame string, slice int, fil start := time.Now() s.batch.Query(`INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?) USING timestamp ?;`, id, db, frame, slice, int(filter), chunk, blockIndex, block, start.UnixNano()) - util.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds()) + statsd.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds()) return nil } @@ -223,7 +223,7 @@ func (s *Storage) RemoveBlock(bid uint64, db string, frame string, slice int, bc s.batch.Query(`DELETE FROM bitmap USING TIMESTAMP ? WHERE bitmap_id=? AND db=? AND frame=? AND slice=? AND chunkkey=? AND blockindex=?`, startTime.UnixNano(), id, db, frame, slice, chunk, blockIndex) - util.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds()) + statsd.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds()) } func (s *Storage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, val, count uint64) { diff --git a/storage/leveldb/leveldb.go b/storage/leveldb/leveldb.go index bcdf8cc58..4af2920cd 100644 --- a/storage/leveldb/leveldb.go +++ b/storage/leveldb/leveldb.go @@ -10,7 +10,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" . "github.com/syndtr/goleveldb/leveldb/util" "github.com/umbel/pilosa" - "github.com/umbel/pilosa/util" + "github.com/umbel/pilosa/statsd" ) func init() { @@ -40,7 +40,7 @@ func NewStorage(opt pilosa.StorageOptions) *Storage { opt.DB, strconv.Itoa(opt.Slice), opt.Frame, - util.SUUID_to_Hex(opt.FragmentID), + pilosa.SUUID_to_Hex(opt.FragmentID), ) return &Storage{path: path} @@ -101,7 +101,7 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*p lastKey = key } - util.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds()) + statsd.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds()) // Set bit count on bitmap. bm.SetCount(uint64(count)) @@ -126,7 +126,7 @@ func (s *Storage) endBatch() { if time.Since(s.batchTime) > 15*time.Second || s.batchN > 300 { s.Flush() } - util.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds()) + statsd.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds()) } func (s *Storage) Flush() { @@ -142,7 +142,7 @@ func (s *Storage) Flush() { s.batchTime = time.Now() s.batchN = 0 - util.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds()) + statsd.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds()) } // Store saves a bitmap to storage. @@ -167,7 +167,7 @@ func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, fil func (s *Storage) StoreBlock(bitmapID uint64, db string, frame string, slice int, filter uint64, chunk uint64, index int32, block uint64) error { start := time.Now() s.batch.Put(marshalKey(bitmapID, chunk, uint8(index)), marshalValue(block, filter)) - util.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds()) + statsd.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds()) return nil } diff --git a/transport/tcp.go b/transport/tcp.go index 5421ae771..1afe6dcb5 100644 --- a/transport/tcp.go +++ b/transport/tcp.go @@ -9,9 +9,9 @@ import ( notify "github.com/bitly/go-notify" log "github.com/cihub/seelog" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/core" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/util" ) const DefaultTCPPort = 12001 @@ -21,16 +21,16 @@ type connection struct { inbox chan *db.Message outbox chan *db.Message conn *net.Conn - process *util.GUID + process *pilosa.GUID } type newconnection struct { - id *util.GUID + id *pilosa.GUID connection *connection } func init() { - gob.Register(util.GUID{}) + gob.Register(pilosa.GUID{}) } func (self *connection) manage() { @@ -79,7 +79,7 @@ BeginManageConnection: return } case message := <-self.inbox: - identifier, ok := message.Data.(util.GUID) + identifier, ok := message.Data.(pilosa.GUID) if ok { // message is connection registration; bypass inbox and register self.process = &identifier @@ -102,19 +102,19 @@ BeginManageConnection: type TcpTransport struct { inbox chan *db.Message outbox chan db.Envelope - connections map[util.GUID]*connection + connections map[pilosa.GUID]*connection reg chan *newconnection - ID util.GUID + ID pilosa.GUID Port int ProcessMap *core.ProcessMap } -func NewTcpTransport(id util.GUID) *TcpTransport { +func NewTcpTransport(id pilosa.GUID) *TcpTransport { return &TcpTransport{ inbox: make(chan *db.Message, 100), outbox: make(chan db.Envelope, 100), - connections: make(map[util.GUID]*connection), + connections: make(map[pilosa.GUID]*connection), reg: make(chan *newconnection), ID: id, @@ -168,7 +168,7 @@ func (self *TcpTransport) Close() { log.Warn("Shutting down TCP transport") } -func (self *TcpTransport) Send(message *db.Message, host *util.GUID) { +func (self *TcpTransport) Send(message *db.Message, host *pilosa.GUID) { log.Trace("TcpTransport.Send", message, host) envelope := db.Envelope{Message: message, Host: host} notify.Post("outbox", &envelope) diff --git a/util/storage.go b/util.go similarity index 73% rename from util/storage.go rename to util.go index eace2a47f..8becf9a5f 100644 --- a/util/storage.go +++ b/util.go @@ -1,4 +1,4 @@ -package util +package pilosa import ( "io" @@ -8,14 +8,14 @@ import ( "github.com/kr/s3/s3util" ) -func Open(s string) (io.ReadCloser, error) { +func openFile(s string) (io.ReadCloser, error) { if isURL(s) { return s3util.Open(s, nil) } return os.Open(s) } -func Create(s string) (io.WriteCloser, error) { +func createFile(s string) (io.WriteCloser, error) { if isURL(s) { return s3util.Create(s, nil, nil) } diff --git a/util/constants.go b/util/constants.go deleted file mode 100644 index d0a83db06..000000000 --- a/util/constants.go +++ /dev/null @@ -1,3 +0,0 @@ -package util - -const TimeOut = 30