scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

flatten util into pilosa

Browse source 
This commit moves ID-related types and functions to the top
level package and moves remaining statsd wrapper into a `statsd`
package.
This commit is contained in:
Ben Johnson 2015-09-03 14:43:34 -06:00
parent f822215be9
commit 503e5c95be
31 changed files with 343 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
brand.go
View file

@ -9,7 +9,7 @@ import (
"time"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa/statsd"
)
var FragmentBase string
@ -159,7 +159,7 @@ func (b *Brand) Rank() {
b.rank_count = 0
delta := time.Since(start)
util.SendTimer("brand_Rank", delta.Nanoseconds())
statsd.SendTimer("brand_Rank", delta.Nanoseconds())
b.rank_time = start
}
@ -366,7 +366,7 @@ func (b *Brand) Persist() error {
log.Warn("Nothing to save ", b.getFileName())
return nil
}
w, err := util.Create(b.getFileName())
w, err := createFile(b.getFileName())
if err != nil {
log.Warn("Error opening outfile ", b.getFileName())
log.Warn(err)
@ -396,7 +396,7 @@ func (b *Brand) Persist() error {
func (b *Brand) Load(requestChan chan Command, f *Fragment) {
log.Warn("Brand Load")
time.Sleep(time.Duration(rand.Intn(32)) * time.Second) //trying to avoid mass cassandra hit
r, err := util.Open(b.getFileName())
r, err := openFile(b.getFileName())
if err != nil {
log.Warn("NO Brand Init File:", b.getFileName())
return

4
brand_test.go
View file

@ -5,8 +5,8 @@ import (
"testing"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/storage/mem"
"github.com/umbel/pilosa/util"
)
var (
@ -18,7 +18,7 @@ var (
func init() {
size = 1000
util.SetupStatsd()
statsd.Setup()
// SetupCassandra()
membrand = pilosa.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0)

8
cmd/pilosa/config.go
View file

@ -4,10 +4,10 @@ import (
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
_ "github.com/umbel/pilosa/storage"
"github.com/umbel/pilosa/storage/cassandra"
"github.com/umbel/pilosa/transport"
"github.com/umbel/pilosa/util"
)
const (
@ -31,8 +31,8 @@ var (
// Config represents the configuration for the command.
type Config struct {
ID *util.GUID `toml:"id"`
Host string `toml:"host"`
ID *pilosa.GUID `toml:"id"`
Host string `toml:"host"`
TCP struct {
Port int `toml:"port"`
@ -99,7 +99,7 @@ func NewConfig() Config {
c.Storage.SupportedFrames = DefaultSupportedFrames[:]
c.Storage.CassandraTimeWindow = Duration(cassandra.DefaultFlushInterval)
c.Storage.CassandraMaxSizeBatch = cassandra.DefaultFlushThreshold
c.Statsd.Host = util.DefaultStatsdHost
c.Statsd.Host = statsd.DefaultHost
c.ETCD.Hosts = DefaultETCDHosts[:]
return c
}

8
cmd/pilosa/main.go
View file

@ -18,8 +18,8 @@ import (
"github.com/umbel/pilosa/dispatch"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/transport"
"github.com/umbel/pilosa/util"
)
// Build holds the build information passed in at compile time.
@ -68,7 +68,7 @@ func (m *Main) Run(args ...string) error {
// Generate an ID if one is not specified in the config.
id := config.ID
if id == nil {
*id = util.RandomUUID()
*id = pilosa.RandomUUID()
}
// Set up profiling.
@ -94,8 +94,8 @@ func (m *Main) Run(args ...string) error {
s3util.DefaultConfig.SecretKey = config.AWS.SecretAccessKey
// Initialize Statsd.
util.StatsdHost = config.Statsd.Host
util.SetupStatsd()
statsd.Host = config.Statsd.Host
statsd.Setup()
// Initialize logging.
logger, _ := log.LoggerFromConfigAsBytes([]byte(SeelogProductionConfig(config.Log.Path, *id, config.Log.Level)))

4
cmd/pilosa/seelog.go
View file

@ -3,10 +3,10 @@ package main
import (
"fmt"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa"
)
func SeelogProductionConfig(path string, id util.GUID, level string) string {
func SeelogProductionConfig(path string, id pilosa.GUID, level string) string {
if path == "" {
path = "/tmp"
}

20
core/batch.go
View file

@ -4,25 +4,25 @@ import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/util"
)
type BatchRequest struct {
Id *util.GUID
Source *util.GUID
Fragment_id util.SUUID
Id *pilosa.GUID
Source *pilosa.GUID
Fragment_id pilosa.SUUID
Bitmap_id uint64
Compressed_bitmap string
Filter uint64
}
type BatchResponse struct {
Id *util.GUID
Id *pilosa.GUID
}
func (self BatchResponse) ResultId() *util.GUID {
func (self BatchResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self BatchResponse) ResultData() interface{} {
@ -35,16 +35,16 @@ func init() {
}
type Batcher struct {
ID util.GUID
ID pilosa.GUID
Cluster *db.Cluster
Hold *hold.Holder
Transport interface {
Send(message *db.Message, host *util.GUID)
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewBatcher(id util.GUID) *Batcher {
func NewBatcher(id pilosa.GUID) *Batcher {
return &Batcher{ID: id}
}
@ -58,7 +58,7 @@ func (b *Batcher) Batch(database_name, frame, compressed_bitmap string, bitmap_i
fragment, err := database.GetFragmentForBitmap(oslice, &db.Bitmap{Id: bitmap_id, FrameType: frame, Filter: filter})
if err == nil {
id := util.RandomUUID()
id := pilosa.RandomUUID()
batch := db.Message{Data: BatchRequest{Id: &id, Source: &b.ID, Fragment_id: fragment.GetId(), Bitmap_id: bitmap_id, Compressed_bitmap: compressed_bitmap}}
dest_id := fragment.GetProcess().Id()
b.Transport.Send(&batch, &dest_id)

42
core/etcd.go
View file

@ -14,8 +14,8 @@ import (
log "github.com/cihub/seelog"
"github.com/coreos/go-etcd/etcd"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
const (
@ -25,7 +25,7 @@ const (
type TopologyMapper struct {
namespace string
ID util.GUID
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *ProcessMap
@ -41,7 +41,7 @@ type TopologyMapper struct {
}
Index interface {
AddFragment(db string, frame string, slice int, id util.SUUID)
AddFragment(db string, frame string, slice int, id pilosa.SUUID)
}
}
@ -193,7 +193,7 @@ func (self *TopologyMapper) AllocateFragment(process_guid, db, frame string, sli
//to create the node, just write off the items to etcd and the watch should spawn
//be nice if something would notify perhaps queue
//so i need db, frame, slice , fragment_id
fuid := util.SUUID_to_Hex(util.Id())
fuid := pilosa.SUUID_to_Hex(pilosa.Id())
fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid)
// need to check value to see how many we have left
log.Warn("ALLOC:", process_guid, len(process_guid))
@ -215,10 +215,10 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error {
var database *db.Database
var frame *db.Frame
var fragment *db.Fragment
var fragment_id util.SUUID
var fragment_id pilosa.SUUID
var slice *db.Slice
var slice_int int
var process_uuid util.GUID
var process_uuid pilosa.GUID
var process *db.Process
var err error
@ -227,7 +227,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error {
return errors.New("no process")
}
process_uuid, err = util.ParseGUID(node.Value)
process_uuid, err = pilosa.ParseGUID(node.Value)
if err != nil {
log.Warn("Bad Process Guid", key)
return errors.New("No Process Id")
@ -268,7 +268,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error {
}
}
if len(bits) > 7 {
fragment_id = util.Hex_to_SUUID(bits[7])
fragment_id = pilosa.Hex_to_SUUID(bits[7])
fragment = database.GetOrCreateFragment(frame, slice, fragment_id)
}
@ -284,7 +284,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error {
process = db.NewProcess(&process_uuid)
fragment.SetProcess(process)
if util.Equal(&self.ID, &process_uuid) {
if pilosa.Equal(&self.ID, &process_uuid) {
self.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id)
}
@ -299,26 +299,26 @@ func (self *TopologyMapper) remove_fragment(node *etcd.Node) error {
func flatten(node *etcd.Node) []*etcd.Node {
nodes := []*etcd.Node{node}
for i := 0; i < len(node.Nodes); i++ {
nodes = append(nodes, flatten(node.Nodes[i])...)
nodes = append(nodes, flatten(&node.Nodes[i])...)
}
return nodes
}
type Node struct {
id *util.GUID
id *pilosa.GUID
ip string
port_tcp int
port_http int
}
type ProcessMap struct {
nodes map[util.GUID]*db.Process
nodes map[pilosa.GUID]*db.Process
mutex sync.Mutex
}
func NewProcessMap() *ProcessMap {
p := ProcessMap{}
p.nodes = make(map[util.GUID]*db.Process)
p.nodes = make(map[pilosa.GUID]*db.Process)
return &p
}
@ -328,7 +328,7 @@ func (self *ProcessMap) AddProcess(process *db.Process) {
self.nodes[process.Id()] = process
}
func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) {
func (self *ProcessMap) GetProcess(id *pilosa.GUID) (*db.Process, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
if id == nil {
@ -342,7 +342,7 @@ func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) {
return process, nil
}
func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process {
func (self *ProcessMap) GetOrAddProcess(id *pilosa.GUID) *db.Process {
process, err := self.GetProcess(id)
if err != nil {
process = db.NewProcess(id)
@ -351,7 +351,7 @@ func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process {
return process
}
func (self *ProcessMap) GetHost(id *util.GUID) (string, error) {
func (self *ProcessMap) GetHost(id *pilosa.GUID) (string, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
@ -361,7 +361,7 @@ func (self *ProcessMap) GetHost(id *util.GUID) (string, error) {
return process.Host(), nil
}
func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) {
func (self *ProcessMap) GetPortTcp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
@ -371,7 +371,7 @@ func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) {
return process.PortTcp(), nil
}
func (self *ProcessMap) GetPortHttp(id *util.GUID) (int, error) {
func (self *ProcessMap) GetPortHttp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
@ -400,7 +400,7 @@ type ProcessMapper struct {
commands chan ProcessMapperCommand
namespace string
ID util.GUID
ID pilosa.GUID
ProcessMap *ProcessMap
TCPPort int
@ -431,7 +431,7 @@ func getKey(input string) string {
return bits[len(bits)-1]
}
func (self *ProcessMapper) getnode(u *util.GUID) *Node {
func (self *ProcessMapper) getnode(u *pilosa.GUID) *Node {
return new(Node)
}
@ -452,7 +452,7 @@ func (self *ProcessMapper) handlenode(node *etcd.Node) error {
}
if len(bits) >= 2 {
id_string := bits[1]
id, err := util.ParseGUID(id_string)
id, err := pilosa.ParseGUID(id_string)
if err != nil {
return errors.New("Invalid GUID: " + id_string + " (" + key + ")")
}

26
core/http.go
View file

@ -7,9 +7,11 @@ import (
"encoding/gob"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"reflect"
"runtime"
"strconv"
@ -20,9 +22,10 @@ import (
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/gorilla/websocket"
"github.com/kr/s3/s3util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa/statsd"
)
const DefaultRequestLogPath = "/tmp/set_bit_log"
@ -32,7 +35,7 @@ var RequestLogPath = DefaultRequestLogPath
type WebService struct {
end chan bool
ID util.GUID
ID pilosa.GUID
Version string
Port int
DefaultDB string
@ -48,7 +51,7 @@ type WebService struct {
}
Pinger interface {
Ping(process_id *util.GUID) (*time.Duration, error)
Ping(process_id *pilosa.GUID) (*time.Duration, error)
}
TopologyMapper interface {
@ -127,7 +130,7 @@ func genFileName(id string) string {
func flush(requests []LogRecord, id string, records_to_dump int) {
dest := genFileName(id)
w, err := util.Create(dest)
w, err := createFile(dest)
if err != nil {
log.Warn("Error opening outfile ", dest)
log.Warn(err)
@ -367,7 +370,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
util.SendInc("webservice_Query")
statsd.SendInc("webservice_Query")
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
@ -671,7 +674,7 @@ func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
return
}
process_string := r.Form.Get("process")
process_id, err := util.ParseGUID(process_string)
process_id, err := pilosa.ParseGUID(process_string)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -838,3 +841,14 @@ func (self *WebService) HandleStatus(w http.ResponseWriter, r *http.Request) {
</body></html>
`))
}
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}
return os.Create(s)
}
func isURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}

24
core/ping.go
View file

@ -4,20 +4,20 @@ import (
"encoding/gob"
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
type PingRequest struct {
Id *util.GUID
Source *util.GUID
Id *pilosa.GUID
Source *pilosa.GUID
}
type PongRequest struct {
Id *util.GUID
Id *pilosa.GUID
}
func (self PongRequest) ResultId() *util.GUID {
func (self PongRequest) ResultId() *pilosa.GUID {
return self.Id
}
func (self PongRequest) ResultData() interface{} {
@ -30,29 +30,29 @@ func init() {
}
type Pinger struct {
ID util.GUID
ID pilosa.GUID
Hold interface {
Get(id *util.GUID, timeout int) (interface{}, error)
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *util.GUID)
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewPinger(id util.GUID) *Pinger {
func NewPinger(id pilosa.GUID) *Pinger {
return &Pinger{
ID: id,
}
}
func (self *Pinger) Ping(process_id *util.GUID) (*time.Duration, error) {
id := util.RandomUUID()
func (self *Pinger) Ping(process_id *pilosa.GUID) (*time.Duration, error) {
id := pilosa.RandomUUID()
ping := db.Message{Data: PingRequest{Id: &id, Source: &self.ID}}
start := time.Now()
self.Transport.Send(&ping, process_id)
_, err := self.Hold.Get(&id, 60)
_, err := self.Hold.Get(&id, 60*time.Second)
if err != nil {
return nil, err
}

28
core/remotebits.go
View file

@ -4,23 +4,23 @@ import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
type RemoteSetBit struct {
requests []remote_task
cluster map[*util.GUID][]BitmapRequestItem
cluster map[*pilosa.GUID][]BitmapRequestItem
ID util.GUID
ID pilosa.GUID
ProcessMap *ProcessMap
Hold interface {
Get(id *util.GUID, timeout int) (interface{}, error)
Get(id *pilosa.GUID, timeout int) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *util.GUID)
Send(message *db.Message, host *pilosa.GUID)
}
}
@ -32,12 +32,12 @@ func init() {
type BitsRequest struct {
Bits []BitmapRequestItem
ReturnProcessId util.GUID
QueryId util.GUID
DestProcessId util.GUID
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type BitmapRequestItem struct {
Fragment_id util.SUUID
Fragment_id pilosa.SUUID
Bitmap_id uint64
Profile_id uint64
Filter uint64
@ -47,7 +47,7 @@ type BitmapRequestItem struct {
func NewRemoteSetBit() *RemoteSetBit {
obj := new(RemoteSetBit)
obj.cluster = make(map[*util.GUID][]BitmapRequestItem)
obj.cluster = make(map[*pilosa.GUID][]BitmapRequestItem)
return obj
}
@ -55,7 +55,7 @@ func (self *RemoteSetBit) Request() {
self.requests = make([]remote_task, 0)
source_process, _ := self.ProcessMap.GetProcess(&self.ID)
for process, request := range self.cluster {
random_id := util.RandomUUID()
random_id := pilosa.RandomUUID()
msg := new(db.Message)
msg.Data = BitsRequest{
Bits: request,
@ -74,7 +74,7 @@ func (self *RemoteSetBit) Request() {
}
type remote_task struct {
id util.GUID
id pilosa.GUID
wait_time int
}
@ -115,11 +115,11 @@ func (self *RemoteSetBit) Add(frag *db.Fragment, bitmap_id, profile_id, filter u
}
type BitsResponse struct {
Id *util.GUID
Id *pilosa.GUID
Items []SBResult
}
func (self *BitsResponse) ResultId() *util.GUID {
func (self *BitsResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self *BitsResponse) ResultData() interface{} {

6
db/db.go
View file

@ -3,7 +3,7 @@ package db
import (
"encoding/gob"
. "github.com/umbel/pilosa/util"
"github.com/umbel/pilosa"
)
type Message struct {
@ -12,11 +12,11 @@ type Message struct {
type Envelope struct {
Message *Message
Host *GUID
Host *pilosa.GUID
}
type HoldResult interface {
ResultId() *GUID
ResultId() *pilosa.GUID
ResultData() interface{}
}

34
db/topology.go
View file

@ -7,7 +7,7 @@ import (
log "github.com/cihub/seelog"
"github.com/stathat/consistent"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa"
)
// SupportedFrames is a list of frame types that are supported.
@ -20,23 +20,23 @@ var FragmentDoesNotExistError = errors.New("Fragment does not exist.")
var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.")
type Location struct {
ProcessId *util.GUID
FragmentId util.SUUID
ProcessId *pilosa.GUID
FragmentId pilosa.SUUID
}
type Process struct {
id *util.GUID
id *pilosa.GUID
host string
port_tcp int
port_http int
mutex sync.Mutex
}
func NewProcess(id *util.GUID) *Process {
func NewProcess(id *pilosa.GUID) *Process {
return &Process{id: id}
}
func (self *Process) Id() util.GUID {
func (self *Process) Id() pilosa.GUID {
self.mutex.Lock()
defer self.mutex.Unlock()
return *self.id
@ -302,7 +302,7 @@ func (self *FrameSliceIntersect) GetFragments() []*Fragment {
return self.fragments
}
func (d *Database) GetFragment(fragment_id util.SUUID) (*Fragment, error) {
func (d *Database) GetFragment(fragment_id pilosa.SUUID) (*Fragment, error) {
for _, fsi := range d.frame_slice_intersects {
f, err := fsi.GetFragment(fragment_id)
if err == nil {
@ -311,7 +311,7 @@ func (d *Database) GetFragment(fragment_id util.SUUID) (*Fragment, error) {
}
return nil, FragmentDoesNotExistError
}
func (self *FrameSliceIntersect) GetFragment(fragment_id util.SUUID) (*Fragment,
func (self *FrameSliceIntersect) GetFragment(fragment_id pilosa.SUUID) (*Fragment,
error) {
for _, fragment := range self.fragments {
if fragment.id == fragment_id {
@ -323,7 +323,7 @@ func (self *FrameSliceIntersect) GetFragment(fragment_id util.SUUID) (*Fragment,
func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) {
self.fragments = append(self.fragments, fragment)
self.hashring.Add(util.SUUID_to_Hex(fragment.id))
self.hashring.Add(pilosa.SUUID_to_Hex(fragment.id))
}
///////// FRAGMENTS
@ -333,11 +333,11 @@ func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) {
// reference to the responsible node for that fragment. The node is in the form
// ip:port
type Fragment struct {
id util.SUUID
id pilosa.SUUID
process *Process
}
func (self *Fragment) GetId() util.SUUID {
func (self *Fragment) GetId() pilosa.SUUID {
return self.id
}
@ -345,7 +345,7 @@ func (self *Fragment) GetProcess() *Process {
return self.process
}
func (self *Fragment) GetProcessId() *util.GUID {
func (self *Fragment) GetProcessId() *pilosa.GUID {
return self.process.id
}
@ -373,7 +373,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment
log.Warn(err)
return nil, err
}
frag_id := util.Hex_to_SUUID(frag_id_s)
frag_id := pilosa.Hex_to_SUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
@ -391,11 +391,11 @@ func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragme
log.Warn(err)
return nil, err
}
frag_id := util.Hex_to_SUUID(frag_id_s)
frag_id := pilosa.Hex_to_SUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) (*Fragment, error) {
func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn(err)
@ -404,7 +404,7 @@ func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id util.SUUI
return fsi.GetFragment(fragment_id)
}
func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) *Fragment {
func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("database.addFragment", err)
@ -415,7 +415,7 @@ func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id util.SUUI
return &fragment
}
func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) *Fragment {
func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
d.mutex.Lock()
defer d.mutex.Unlock()
fragment, err := d.getFragment(frame, slice, fragment_id)

17
dispatch/dispatch.go
View file

@ -1,6 +1,8 @@
package dispatch
import (
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
@ -8,7 +10,6 @@ import (
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/query"
"github.com/umbel/pilosa/util"
)
type Dispatch struct {
@ -17,19 +18,19 @@ type Dispatch struct {
}
Hold interface {
Set(id *util.GUID, value interface{}, timeout int)
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(fragID util.SUUID, bitmapID uint64, pos uint64) (bool, error)
LoadBitmap(fragID util.SUUID, bitmapID uint64, compressedBitmap string, filter uint64)
SetBit(fragID util.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error)
ClearBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64) (bool, error)
LoadBitmap(fragID pilosa.SUUID, bitmapID uint64, compressedBitmap string, filter uint64)
SetBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error)
TopFillBatch(args []pilosa.FillArgs) ([]pilosa.Pair, error)
}
Transport interface {
Receive() *db.Message
Send(message *db.Message, host *util.GUID)
Send(message *db.Message, host *pilosa.GUID)
}
}
@ -84,7 +85,7 @@ func (d *Dispatch) Run() {
case db.HoldResult:
log.Trace("Dispatch.Run HoldResult")
d.Hold.Set(data.ResultId(), data.ResultData(), 30)
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
case query.PortableQueryStep:
log.Trace("Dispatch.Run PortableQueryStep")
@ -95,7 +96,7 @@ func (d *Dispatch) Run() {
go d.topFillHandler(message)
case core.BitsResponse:
d.Hold.Set(data.ResultId(), data.ResultData(), 30)
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
default:
spew.Dump(data)

121
executor/executor.go
View file

@ -11,35 +11,36 @@ import (
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/query"
"github.com/umbel/pilosa/util"
)
const DefaultTimeout = 30 * time.Second
type Executor struct {
inbox chan *db.Message
ID util.GUID
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *core.ProcessMap
PluginsPath string
Hold interface {
Get(id *util.GUID, timeout int) (interface{}, error)
Set(id *util.GUID, value interface{}, timeout int)
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(frag_id util.SUUID, bitmap_id uint64, pos uint64) (bool, error)
Count(frag_id util.SUUID, bitmap pilosa.BitmapHandle) (uint64, error)
Difference(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
FromBytes(frag_id util.SUUID, bytes []byte) (pilosa.BitmapHandle, error)
Get(frag_id util.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error)
GetBytes(frag_id util.SUUID, bh pilosa.BitmapHandle) ([]byte, error)
Intersect(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error)
SetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error)
TopN(frag_id util.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error)
TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]pilosa.Pair, error)
Union(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
ClearBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64) (bool, error)
Count(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) (uint64, error)
Difference(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
FromBytes(frag_id pilosa.SUUID, bytes []byte) (pilosa.BitmapHandle, error)
Get(frag_id pilosa.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error)
GetBytes(frag_id pilosa.SUUID, bh pilosa.BitmapHandle) ([]byte, error)
Intersect(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
Range(frag_id pilosa.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error)
SetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error)
TopN(frag_id pilosa.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error)
TopNAll(frag_id pilosa.SUUID, n int, categories []uint64) ([]pilosa.Pair, error)
Union(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
}
TopologyMapper interface {
@ -47,11 +48,11 @@ type Executor struct {
}
Transport interface {
Send(*db.Message, *util.GUID)
Send(*db.Message, *pilosa.GUID)
}
}
func NewExecutor(id util.GUID) *Executor {
func NewExecutor(id pilosa.GUID) *Executor {
log.Trace("NewExector")
return &Executor{inbox: make(chan *db.Message)}
}
@ -101,7 +102,7 @@ func (self *Executor) CountQueryStepHandler(msg *db.Message) {
//spew.Dump("COUNT QUERYSTEP")
qs := msg.Data.(query.CountQueryStep)
input := qs.Input
value, _ := self.Hold.Get(input, util.TimeOut)
value, _ := self.Hold.Get(input, DefaultTimeout)
var bh pilosa.BitmapHandle
switch val := value.(type) {
case pilosa.BitmapHandle:
@ -168,7 +169,7 @@ func (self *Executor) UnionQueryStepHandler(msg *db.Message) {
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, util.TimeOut)
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
@ -208,7 +209,7 @@ func (self *Executor) IntersectQueryStepHandler(msg *db.Message) {
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, util.TimeOut)
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
@ -248,7 +249,7 @@ func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) {
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, util.TimeOut)
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
@ -288,9 +289,9 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) {
return_type := "bitmap-handles"
var sum uint64
merge_map := make(map[uint64]uint64)
slice_map := make(map[uint64]map[util.SUUID]struct{})
all_slice := make(map[util.SUUID]struct {
process util.GUID
slice_map := make(map[uint64]map[pilosa.SUUID]struct{})
all_slice := make(map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
})
@ -299,8 +300,8 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) {
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *util.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, util.TimeOut)
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
@ -331,13 +332,13 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) {
merge_map[pair.Key] += pair.Count
mm, ok := slice_map[pair.Key]
if !ok {
mm = make(map[util.SUUID]struct{})
mm = make(map[pilosa.SUUID]struct{})
slice_map[pair.Key] = mm
}
mm[val.FragmentId] = e
}
all_slice[val.FragmentId] = struct {
process util.GUID
process pilosa.GUID
handle pilosa.BitmapHandle
}{val.ProcessId, val.HBitmap}
check_pair = true
@ -393,7 +394,7 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) {
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) SendRequest(process_id util.GUID, t *Task) {
func (self *Executor) SendRequest(process_id pilosa.GUID, t *Task) {
args := make([]pilosa.FillArgs, len(t.f), len(t.f))
for _, v := range t.f {
args = append(args, v)
@ -404,17 +405,17 @@ func (self *Executor) SendRequest(process_id util.GUID, t *Task) {
self.Transport.Send(msg, &process_id)
}
func (self *Executor) FetchMissing(tasks map[util.GUID]*Task) {
func (self *Executor) FetchMissing(tasks map[pilosa.GUID]*Task) {
for k, v := range tasks {
go self.SendRequest(k, v)
}
}
func (self *Executor) GatherResults(tasks map[util.GUID]*Task) map[uint64]uint64 {
func (self *Executor) GatherResults(tasks map[pilosa.GUID]*Task) map[uint64]uint64 {
results := make(map[uint64]uint64)
answers := make(chan []pilosa.Pair)
for _, task := range tasks {
go func(id util.GUID) {
go func(id pilosa.GUID) {
value, err := self.Hold.Get(&id, 10) //eiher need to be the frame process or the handler process?
if value == nil {
log.Warn("Bad TopN Result:", err)
@ -443,7 +444,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) {
bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandler1", util.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id)
log.Error("GetQueryStepHandler1", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id)
log.Error("GetQueryStepHandler2", err)
}
@ -454,7 +455,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandlerr3", util.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id)
log.Error("GetQueryStepHandlerr3", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id)
log.Error("GetQueryStepHandler4", err)
}
result = bm
@ -528,8 +529,8 @@ func (self *Executor) StashQueryStepHandler(msg *db.Message) {
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *util.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, util.TimeOut)
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
@ -573,7 +574,7 @@ func (self *Executor) runQuery(database *db.Database, qry *query.Query) error {
return err
}
process_id := process.Id()
fragment_id := util.SUUID(0)
fragment_id := pilosa.SUUID(0)
destination := db.Location{ProcessId: &process_id, FragmentId: fragment_id}
query_plan, err := query.QueryPlanForQuery(database, qry, &destination)
@ -685,34 +686,34 @@ func init() {
}
type TopNPackage struct {
ProcessId util.GUID
FragmentId util.SUUID
ProcessId pilosa.GUID
FragmentId pilosa.SUUID
Pairs []pilosa.Pair
HBitmap pilosa.BitmapHandle
}
type TopFill struct {
Args []pilosa.FillArgs
ReturnProcessId util.GUID
QueryId util.GUID
DestProcessId util.GUID
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type Task struct {
processid util.GUID
f map[util.SUUID]pilosa.FillArgs
hold_id util.GUID
processid pilosa.GUID
f map[pilosa.SUUID]pilosa.FillArgs
hold_id pilosa.GUID
}
func newtask(p util.GUID) *Task {
func newtask(p pilosa.GUID) *Task {
result := new(Task)
result.processid = p
result.f = make(map[util.SUUID]pilosa.FillArgs)
result.hold_id = util.RandomUUID()
result.f = make(map[pilosa.SUUID]pilosa.FillArgs)
result.hold_id = pilosa.RandomUUID()
return result
}
func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) {
func (t *Task) Add(frag pilosa.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) {
fa, ok := t.f[frag]
if !ok {
fa = pilosa.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)}
@ -722,13 +723,13 @@ func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle
}
func BuildTask(merge_map map[uint64]uint64,
slice_map map[uint64]map[util.SUUID]struct{},
total_fragments map[util.SUUID]struct {
process util.GUID
slice_map map[uint64]map[pilosa.SUUID]struct{},
total_fragments map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) map[util.GUID]*Task {
}) map[pilosa.GUID]*Task {
tasks := make(map[util.GUID]*Task)
tasks := make(map[pilosa.GUID]*Task)
for bitmap_id, _ := range merge_map { //for all brands
//for fragment_id, reported_fragments := range slice_map[bitmap_id] { //find missing fragments
reporting_fragments := slice_map[bitmap_id]
@ -749,13 +750,13 @@ func BuildTask(merge_map map[uint64]uint64,
}
type hole struct {
process util.GUID
process pilosa.GUID
handle pilosa.BitmapHandle
fragment util.SUUID
fragment pilosa.SUUID
}
func missing(fids map[util.SUUID]struct{}, all map[util.SUUID]struct {
process util.GUID
func missing(fids map[pilosa.SUUID]struct{}, all map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) []hole {
results := make([]hole, 0, 0)
@ -769,7 +770,7 @@ func missing(fids map[util.SUUID]struct{}, all map[util.SUUID]struct {
return results
}
func (self *TopFill) GetId() *util.GUID {
func (self *TopFill) GetId() *pilosa.GUID {
return &self.QueryId
}
func (self *TopFill) GetLocation() *db.Location {

92
fragment_container.go
View file

@ -11,7 +11,7 @@ import (
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa/statsd"
)
// DefaultBackend is the default data storage layer.
@ -22,20 +22,20 @@ var Backend = DefaultBackend
var LevelDBPath string
type FragmentContainer struct {
fragments map[util.SUUID]*Fragment
fragments map[SUUID]*Fragment
mutex *sync.Mutex
}
func NewFragmentContainer() *FragmentContainer {
f := new(FragmentContainer)
f.fragments = make(map[util.SUUID]*Fragment)
f.fragments = make(map[SUUID]*Fragment)
f.mutex = &sync.Mutex{}
return f
}
type BitmapHandle uint64
type FillArgs struct {
Frag_id util.SUUID
Frag_id SUUID
Handle BitmapHandle
Bitmaps []uint64
}
@ -59,7 +59,7 @@ func (self *FragmentContainer) Shutdown() {
log.Warn("Container Shutdown Complete")
}
func (self *FragmentContainer) LoadBitmap(frag_id util.SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) {
func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) {
log.Trace("LoadBitmap")
if fragment, found := self.GetFragment(frag_id); found {
request := NewLoader(bitmap_id, compressed_bitmap, filter)
@ -69,7 +69,7 @@ func (self *FragmentContainer) LoadBitmap(frag_id util.SUUID, bitmap_id uint64,
}
}
func (self *FragmentContainer) GetFragment(frag_id util.SUUID) (*Fragment, bool) {
func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) {
log.Trace("index.GetFragment")
self.mutex.Lock()
c, v := self.fragments[frag_id]
@ -77,7 +77,7 @@ func (self *FragmentContainer) GetFragment(frag_id util.SUUID) (*Fragment, bool)
return c, v
}
func (self *FragmentContainer) Stats(frag_id util.SUUID) interface{} {
func (self *FragmentContainer) Stats(frag_id SUUID) interface{} {
if fragment, found := self.GetFragment(frag_id); found {
request := NewStats()
fragment.requestChan <- request
@ -85,7 +85,7 @@ func (self *FragmentContainer) Stats(frag_id util.SUUID) interface{} {
}
return nil
}
func (self *FragmentContainer) Empty(frag_id util.SUUID) (BitmapHandle, error) {
func (self *FragmentContainer) Empty(frag_id SUUID) (BitmapHandle, error) {
log.Trace("index.Empty")
if fragment, found := self.GetFragment(frag_id); found {
request := NewEmpty()
@ -95,97 +95,97 @@ func (self *FragmentContainer) Empty(frag_id util.SUUID) (BitmapHandle, error) {
return 0, errors.New("Invalid Bitmap Handle Empty")
}
func (self *FragmentContainer) Intersect(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) {
func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
log.Trace("index.Intersect")
if fragment, found := self.GetFragment(frag_id); found {
request := NewIntersect(bh)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Intersect", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Intersect", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle Intersect")
}
func (self *FragmentContainer) Union(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) {
func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
log.Trace("index.Union")
if fragment, found := self.GetFragment(frag_id); found {
request := NewUnion(bh)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Union", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Union", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle Union")
}
func (self *FragmentContainer) Difference(frag_id util.SUUID, bh []BitmapHandle) (BitmapHandle, error) {
func (self *FragmentContainer) Difference(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
log.Trace("index.Difference")
if fragment, found := self.GetFragment(frag_id); found {
request := NewDifference(bh)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Difference", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Difference", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle Diff")
}
func (self *FragmentContainer) Get(frag_id util.SUUID, bitmap_id uint64) (BitmapHandle, error) {
func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) {
log.Trace("index.Get")
if fragment, found := self.GetFragment(frag_id); found {
request := NewGet(bitmap_id)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Get", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Get", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle Get")
}
func (self *FragmentContainer) Mask(frag_id util.SUUID, start, end uint64) (BitmapHandle, error) {
func (self *FragmentContainer) Mask(frag_id SUUID, start, end uint64) (BitmapHandle, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewMask(start, end)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Mask", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Mask", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (BitmapHandle, error) {
func (self *FragmentContainer) Range(frag_id SUUID, bitmap_id uint64, start, end time.Time) (BitmapHandle, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewRange(bitmap_id, start, end)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Range", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Range", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) TopN(frag_id util.SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) {
func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) {
log.Trace("index.TopN")
if fragment, found := self.GetFragment(frag_id); found {
request := NewTopN(bh, n, categories)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_TopN", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_TopN", result.exec_time.Nanoseconds())
return result.answer.([]Pair), nil
}
return nil, errors.New(fmt.Sprintf("Fragment not found:%s", util.SUUID_to_Hex(frag_id)))
return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id)))
}
func (self *FragmentContainer) TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]Pair, error) {
func (self *FragmentContainer) TopNAll(frag_id SUUID, n int, categories []uint64) ([]Pair, error) {
log.Trace("index.TopNAll")
if fragment, found := self.GetFragment(frag_id); found {
request := NewTopNAll(n, categories)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_TopNAll", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_TopNAll", result.exec_time.Nanoseconds())
return result.answer.([]Pair), nil
}
return nil, errors.New(fmt.Sprintf("Fragment not found:%s", util.SUUID_to_Hex(frag_id)))
return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id)))
}
func (self *FragmentContainer) TopFillBatch(args []FillArgs) ([]Pair, error) {
@ -216,87 +216,87 @@ func (self *FragmentContainer) TopFillFragment(arg FillArgs) ([]Pair, error) {
request := NewTopFill(arg)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_TopFillFragment", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_TopFillFragment", result.exec_time.Nanoseconds())
return result.answer.([]Pair), nil
}
return nil, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) GetList(frag_id util.SUUID, bitmap_id []uint64) ([]BitmapHandle, error) {
func (self *FragmentContainer) GetList(frag_id SUUID, bitmap_id []uint64) ([]BitmapHandle, error) {
log.Trace("index.GetList")
if fragment, found := self.GetFragment(frag_id); found {
request := NewGetList(bitmap_id)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_GetList", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_GetList", result.exec_time.Nanoseconds())
return result.answer.([]BitmapHandle), nil
}
return nil, errors.New("Invalid Bitmap Handle GetList")
}
func (self *FragmentContainer) Count(frag_id util.SUUID, bitmap BitmapHandle) (uint64, error) {
func (self *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) {
log.Trace("index.Count")
if fragment, found := self.GetFragment(frag_id); found {
request := NewCount(bitmap)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_Count", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_Count", result.exec_time.Nanoseconds())
return result.answer.(uint64), nil
}
return 0, errors.New("Invalid Bitmap Handle Count")
}
func (self *FragmentContainer) GetBytes(frag_id util.SUUID, bh BitmapHandle) ([]byte, error) {
func (self *FragmentContainer) GetBytes(frag_id SUUID, bh BitmapHandle) ([]byte, error) {
log.Trace("index.GetBytes")
if fragment, found := self.GetFragment(frag_id); found {
request := NewGetBytes(bh)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_GetBytes", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_GetBytes", result.exec_time.Nanoseconds())
return result.answer.([]byte), nil
}
return nil, errors.New("Invalid Bitmap Handle GetBytes")
}
func (self *FragmentContainer) FromBytes(frag_id util.SUUID, bytes []byte) (BitmapHandle, error) {
func (self *FragmentContainer) FromBytes(frag_id SUUID, bytes []byte) (BitmapHandle, error) {
log.Trace("index.FromBytes")
if fragment, found := self.GetFragment(frag_id); found {
request := NewFromBytes(bytes)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_FromBytes", result.exec_time.Nanoseconds())
statsd.SendTimer("fragmant_container_FromBytes", result.exec_time.Nanoseconds())
return result.answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle FromBytes")
}
func (self *FragmentContainer) SetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) {
func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) {
log.Trace("SetBit", frag_id, bitmap_id, pos, category)
if fragment, found := self.GetFragment(frag_id); found {
request := NewSetBit(bitmap_id, pos, category)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_SetBit", result.exec_time.Nanoseconds())
util.SendInc("fragmant_container_SetBit")
statsd.SendTimer("fragmant_container_SetBit", result.exec_time.Nanoseconds())
statsd.SendInc("fragmant_container_SetBit")
return result.answer.(bool), nil
}
return false, errors.New("Invalid Bitmap Handle SetBit")
}
func (self *FragmentContainer) ClearBit(frag_id util.SUUID, bitmap_id uint64, pos uint64) (bool, error) {
func (self *FragmentContainer) ClearBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) {
log.Trace("ClearBit", frag_id, bitmap_id, pos)
if fragment, found := self.GetFragment(frag_id); found {
request := NewClearBit(bitmap_id, pos)
fragment.requestChan <- request
result := request.Response()
util.SendTimer("fragmant_container_ClearBit", result.exec_time.Nanoseconds())
util.SendInc("fragmant_container_ClearBit")
statsd.SendTimer("fragmant_container_ClearBit", result.exec_time.Nanoseconds())
statsd.SendInc("fragmant_container_ClearBit")
return result.answer.(bool), nil
}
return false, errors.New("Invalid Bitmap Handle ClearBit")
}
func (self *FragmentContainer) Clear(frag_id util.SUUID) (bool, error) {
func (self *FragmentContainer) Clear(frag_id SUUID) (bool, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewClear()
fragment.requestChan <- request
@ -313,13 +313,13 @@ func dumpHandlesToLog() {
log.Warn(mesg)
}
func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id util.SUUID) {
func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
self.mutex.Lock()
defer self.mutex.Unlock()
_, ok := self.fragments[id]
if !ok {
// dumpHandlesToLog()
log.Warn("ADD FRAGMENT", frame, db, slice, util.SUUID_to_Hex(id))
log.Warn("ADD FRAGMENT", frame, db, slice, SUUID_to_Hex(id))
f := NewFragment(id, db, slice, frame)
loader := make(chan Command)
self.fragments[id] = f
@ -345,7 +345,7 @@ type Pilosa interface {
type Fragment struct {
requestChan chan Command
fragment_id util.SUUID
fragment_id SUUID
impl Pilosa
counter uint64
slice int
@ -356,7 +356,7 @@ type Fragment struct {
queue_size int
}
func NewFragment(frag_id util.SUUID, db string, slice int, frame string) *Fragment {
func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
var impl Pilosa
log.Warn(fmt.Sprintf("XXXXXXXXXXXXXXXXXXXXXXXXXXX(%s)", frame))

19
fragment_container_test.go
View file

@ -5,7 +5,6 @@ import (
"github.com/umbel/pilosa"
_ "github.com/umbel/pilosa/storage"
"github.com/umbel/pilosa/util"
)
func init() {
@ -18,7 +17,7 @@ func TestFragmentContainer_Get(t *testing.T) {
fc.AddFragment("25", "general", 0, 1)
fc.MustClear(1)
if bh, err := fc.Get(util.SUUID(1), 1234); err != nil {
if bh, err := fc.Get(pilosa.SUUID(1), 1234); err != nil {
t.Fatal(err)
} else if bh == 0 {
t.Fatal("expected non-zero bitmap handle")
@ -49,20 +48,20 @@ func TestFragmentContainer_SetBit(t *testing.T) {
// Ensure the number of bits on a bitmap can be counted.
func TestFragmentContainer_Count(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, util.SUUID(1))
fc.AddFragment("25", "general", 0, pilosa.SUUID(1))
// Set a bit on the bitmap.
bi1 := uint64(1234)
if changed, err := fc.SetBit(util.SUUID(1), bi1, 1, 0); err != nil {
if changed, err := fc.SetBit(pilosa.SUUID(1), bi1, 1, 0); err != nil {
t.Fatal(err)
} else if changed == false {
t.Fatal("expected change")
}
// Verify that one bit is set.
if bh, err := fc.Get(util.SUUID(1), bi1); err != nil {
if bh, err := fc.Get(pilosa.SUUID(1), bi1); err != nil {
t.Fatal(err)
} else if n, err := fc.Count(util.SUUID(1), bh); err != nil {
} else if n, err := fc.Count(pilosa.SUUID(1), bh); err != nil {
t.Fatal(err)
} else if n != 1 {
t.Fatalf("unexpected count: %d", n)
@ -273,7 +272,7 @@ func NewFragmentContainer() *FragmentContainer {
}
// MustGet retrieves a bitmap by id. Panic on error.
func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) pilosa.BitmapHandle {
func (fc *FragmentContainer) MustGet(frag_id pilosa.SUUID, bitmap_id uint64) pilosa.BitmapHandle {
bh, err := fc.Get(frag_id, bitmap_id)
if err != nil {
panic(err)
@ -282,7 +281,7 @@ func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) pilos
}
// MustSetBit sets a bit in a bitmap. Panic on error.
func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) bool {
func (fc *FragmentContainer) MustSetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) bool {
changed, err := fc.SetBit(frag_id, bitmap_id, pos, category)
if err != nil {
panic(err)
@ -291,7 +290,7 @@ func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, po
}
// MustClear clears a fragment. Panic on error.
func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool {
func (fc *FragmentContainer) MustClear(fragmentID pilosa.SUUID) bool {
v, err := fc.Clear(fragmentID)
if err != nil {
panic(err)
@ -300,7 +299,7 @@ func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool {
}
// MustCount returns the number of set bits in a bitmap. Panic on error.
func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap pilosa.BitmapHandle) uint64 {
func (fc *FragmentContainer) MustCount(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) uint64 {
v, err := fc.Count(frag_id, bitmap)
if err != nil {
panic(err)

5
general.go
View file

@ -6,7 +6,6 @@ import (
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
"github.com/umbel/pilosa/util"
)
type General struct {
@ -90,7 +89,7 @@ func (self *General) getFileName() string {
func (self *General) Persist() error {
log.Warn("General Persist")
w, err := util.Create(self.getFileName())
w, err := createFile(self.getFileName())
if err != nil {
log.Warn("Error saving:", err)
return err
@ -111,7 +110,7 @@ func (self *General) Persist() error {
func (self *General) Load(requestChan chan Command, f *Fragment) {
log.Warn("General Load")
r, err := util.Open(self.getFileName())
r, err := openFile(self.getFileName())
if err != nil {
log.Warn("NO General Init File:", self.getFileName())
return

26
hold/hold.go
View file

@ -5,39 +5,39 @@ import (
"time"
log "github.com/cihub/seelog"
. "github.com/umbel/pilosa/util"
"github.com/umbel/pilosa"
)
type holdchan chan interface{}
type gethold struct {
id *GUID
id *pilosa.GUID
reply chan holdchan
}
type delhold struct {
id *GUID
id *pilosa.GUID
}
type Holder struct {
data map[GUID]holdchan
data map[pilosa.GUID]holdchan
getchan chan gethold
delchan chan delhold
}
func NewHolder() *Holder {
return &Holder{
data: make(map[GUID]holdchan),
data: make(map[pilosa.GUID]holdchan),
getchan: make(chan gethold),
delchan: make(chan delhold),
}
}
func (self *Holder) DelChan(id *GUID) {
func (self *Holder) DelChan(id *pilosa.GUID) {
log.Trace("Holder.DelChan", id)
req := delhold{id}
self.delchan <- req
}
func (self *Holder) GetChan(id *GUID) holdchan {
func (self *Holder) GetChan(id *pilosa.GUID) holdchan {
log.Trace("Holder.GetChan", id)
reply := make(chan holdchan)
req := gethold{id, reply}
@ -45,25 +45,25 @@ func (self *Holder) GetChan(id *GUID) holdchan {
return <-reply
}
func (self *Holder) Get(id *GUID, timeout int) (interface{}, error) {
log.Trace("Holder.Get", id, timeout)
func (self *Holder) Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) {
log.Trace("Holder.Get", id, timeout.String())
ch := self.GetChan(id)
select {
case val := <-ch:
return val, nil
case <-time.After(time.Duration(timeout) * time.Second):
case <-time.After(timeout):
self.DelChan(id)
return nil, errors.New("Timeout getting from holder")
}
}
func (self *Holder) Set(id *GUID, value interface{}, timeout int) {
log.Trace("Holder.Set", id, value, timeout)
func (self *Holder) Set(id *pilosa.GUID, value interface{}, timeout time.Duration) {
log.Trace("Holder.Set", id, value, timeout.String())
ch := self.GetChan(id)
go func() {
select {
case ch <- value:
case <-time.After(time.Duration(timeout) * time.Second):
case <-time.After(timeout):
}
self.DelChan(id)
}()

3
util/id.go → id.go
View file

@ -1,4 +1,4 @@
package util
package pilosa
import (
"bytes"
@ -44,6 +44,7 @@ func SUUID_to_Hex(a SUUID) string {
binary.Write(buf, binary.BigEndian, a)
return hex.EncodeToString(buf.Bytes())
}
func Hex_to_SUUID(str string) SUUID {
l := len(str)
var m string

18
util/id_test.go → id_test.go
View file

@ -1,20 +1,22 @@
package util
package pilosa_test
import (
"fmt"
"testing"
"github.com/umbel/pilosa"
)
// Ensure id can be parsed from string.
func TestId_Small(t *testing.T) {
if v := Hex_to_SUUID("1"); v != 1 {
if v := pilosa.Hex_to_SUUID("1"); v != 1 {
t.Fatalf("unexpected SUUID: %v", v)
}
}
// Ensure generated IDs are unique.
func TestId_Unique(t *testing.T) {
a, b := Id(), Id()
a, b := pilosa.Id(), pilosa.Id()
if a == b {
t.Fatalf("ids should be unique: %v != %v", a, b)
}
@ -22,8 +24,8 @@ func TestId_Unique(t *testing.T) {
// Ensure ids can be converted to and from hex.
func TestId_Hex(t *testing.T) {
a := Id()
b := Hex_to_SUUID(SUUID_to_Hex(a))
a := pilosa.Id()
b := pilosa.Hex_to_SUUID(pilosa.SUUID_to_Hex(a))
if a != b {
t.Fatalf("ids not equal: %v != %v", a, b)
}
@ -32,18 +34,18 @@ func TestId_Hex(t *testing.T) {
// Ensure ids can be generated in sequence.
func TestId_Multiple(t *testing.T) {
for i := 0; i < 10; i++ {
println(SUUID_to_Hex(Id()))
println(pilosa.SUUID_to_Hex(pilosa.Id()))
}
}
// Ensure a random UUID can be converted to a string.
func TestRandomUUID_String(t *testing.T) {
fmt.Println(RandomUUID().String())
fmt.Println(pilosa.RandomUUID().String())
}
func BenchmarkId(b *testing.B) {
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
Id()
pilosa.Id()
}
}

5
query/parser.go
View file

@ -9,7 +9,6 @@ import (
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/util"
)
var InvalidQueryError = errors.New("Invalid query format.")
@ -53,7 +52,7 @@ func (self *QueryParser) Parse() (query *Query, err error) {
}()
var token *Token
id := util.RandomUUID()
id := pilosa.RandomUUID()
query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})}
token = self.next()
@ -215,7 +214,7 @@ ArgLoop:
if err != nil {
return nil, fmt.Errorf("Expecting fragment id! (%v)", err)
}
recallArgs.Add(util.SUUID(i)) //constructs a new arg
recallArgs.Add(pilosa.SUUID(i)) //constructs a new arg
} else {
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {

59
query/planner.go
View file

@ -11,11 +11,10 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
type PortableQueryStep interface {
GetId() *util.GUID
GetId() *pilosa.GUID
GetLocation() *db.Location
}
@ -59,13 +58,13 @@ func (self *FragmentNotFound) Error() string {
///////////////////////////////////////////////////////////////////////////////////////////////////
type BaseQueryStep struct {
Id *util.GUID
Id *pilosa.GUID
Operation string
Location *db.Location
Destination *db.Location
}
func (self *BaseQueryStep) GetId() *util.GUID {
func (self *BaseQueryStep) GetId() *pilosa.GUID {
log.Trace("BaseQueryStep", *self.Id)
return self.Id
}
@ -76,7 +75,7 @@ func (self *BaseQueryStep) GetLocation() *db.Location {
func (self *BaseQueryStep) LocIsDest() bool {
log.Trace("BaseQueryStep.LocIsDest")
if util.Equal(self.Location.ProcessId, self.Destination.ProcessId) &&
if pilosa.Equal(self.Location.ProcessId, self.Destination.ProcessId) &&
self.Location.FragmentId == self.Destination.FragmentId {
log.Trace("BaseQueryStep.LocIsDest Return true")
return true
@ -86,11 +85,11 @@ func (self *BaseQueryStep) LocIsDest() bool {
}
type BaseQueryResult struct {
Id *util.GUID
Id *pilosa.GUID
Data interface{}
}
func (self *BaseQueryResult) ResultId() *util.GUID {
func (self *BaseQueryResult) ResultId() *pilosa.GUID {
log.Trace("BaseQueryStep.ResultId", self)
return self.Id
@ -106,7 +105,7 @@ func (self *BaseQueryResult) ResultData() interface{} {
///////////////////////////////////////////////////////////////////////////////////////////////////
type CountQueryStep struct {
*BaseQueryStep
Input *util.GUID
Input *pilosa.GUID
}
type CountQueryResult struct {
@ -130,7 +129,7 @@ func (qt *CountQueryTree) getLocation(d *db.Database) (*db.Location, error) {
///////////////////////////////////////////////////////////////////////////////////////////////////
type TopNQueryStep struct {
*BaseQueryStep
Input *util.GUID
Input *pilosa.GUID
Filters []uint64
N int
Frame string
@ -173,7 +172,7 @@ func (qt *TopNQueryTree) getLocation(d *db.Database) (*db.Location, error) {
///////////////////////////////////////////////////////////////////////////////////////////////////
type UnionQueryStep struct {
*BaseQueryStep
Inputs []*util.GUID
Inputs []*pilosa.GUID
}
type UnionQueryResult struct {
@ -206,7 +205,7 @@ func (qt *UnionQueryTree) getLocation(d *db.Database) (*db.Location, error) {
///////////////////////////////////////////////////////////////////////////////////////////////////
type IntersectQueryStep struct {
*BaseQueryStep
Inputs []*util.GUID
Inputs []*pilosa.GUID
}
type IntersectQueryResult struct {
@ -239,7 +238,7 @@ func (qt *IntersectQueryTree) getLocation(d *db.Database) (*db.Location, error)
///////////////////////////////////////////////////////////////////////////////////////////////////
type DifferenceQueryStep struct {
*BaseQueryStep
Inputs []*util.GUID
Inputs []*pilosa.GUID
}
type DifferenceQueryResult struct {
@ -272,7 +271,7 @@ func (qt *DifferenceQueryTree) getLocation(d *db.Database) (*db.Location, error)
///////////////////////////////////////////////////////////////////////////////////////////////////
type CatQueryStep struct {
*BaseQueryStep
Inputs []*util.GUID
Inputs []*pilosa.GUID
N int
}
type Appendable interface {
@ -606,18 +605,18 @@ func (self *QueryPlanner) buildTree(query *Query, slice int) (QueryTree, error)
}
// Produces flattened QueryPlan from QueryTree input
func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Location) (*QueryPlan, error) {
func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.flatten", self, qt, id, location)
plan := QueryPlan{}
if cat, ok := qt.(*CatQueryTree); ok {
inputs := make([]*util.GUID, len(cat.subqueries))
inputs := make([]*pilosa.GUID, len(cat.subqueries))
loc, err := cat.getLocation(self.Database)
if err != nil {
return nil, err
}
step := CatQueryStep{&BaseQueryStep{id, "cat", loc, location}, inputs, cat.N}
for index, subq := range cat.subqueries {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
@ -628,14 +627,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
plan = append(plan, step)
} else if stash, ok := qt.(*StashQueryTree); ok {
inputs := make([]*util.GUID, len(stash.subqueries))
inputs := make([]*pilosa.GUID, len(stash.subqueries))
loc, err := stash.getLocation(self.Database)
if err != nil {
return nil, err
}
step := StashQueryStep{&BaseQueryStep{id, "stash", loc, location}, inputs, stash.N}
for index, subq := range stash.subqueries {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
@ -645,14 +644,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
}
plan = append(plan, step)
} else if union, ok := qt.(*UnionQueryTree); ok {
inputs := make([]*util.GUID, len(union.subqueries))
inputs := make([]*pilosa.GUID, len(union.subqueries))
loc, err := union.getLocation(self.Database)
if err != nil {
return nil, err
}
step := UnionQueryStep{&BaseQueryStep{id, "union", loc, location}, inputs}
for index, subq := range union.subqueries {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
@ -662,14 +661,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
}
plan = append(plan, step)
} else if intersect, ok := qt.(*IntersectQueryTree); ok {
inputs := make([]*util.GUID, len(intersect.subqueries))
inputs := make([]*pilosa.GUID, len(intersect.subqueries))
loc, err := intersect.getLocation(self.Database)
if err != nil {
return nil, err
}
step := IntersectQueryStep{&BaseQueryStep{id, "intersect", loc, location}, inputs}
for index, subq := range intersect.subqueries {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
@ -679,14 +678,14 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
}
plan = append(plan, step)
} else if difference, ok := qt.(*DifferenceQueryTree); ok {
inputs := make([]*util.GUID, len(difference.subqueries))
inputs := make([]*pilosa.GUID, len(difference.subqueries))
loc, err := difference.getLocation(self.Database)
if err != nil {
return nil, err
}
step := DifferenceQueryStep{&BaseQueryStep{id, "difference", loc, location}, inputs}
for index, subq := range difference.subqueries {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
@ -728,7 +727,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
plan := QueryPlan{step}
return &plan, nil
} else if cnt, ok := qt.(*CountQueryTree); ok {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
loc, err := cnt.getLocation(self.Database)
if err != nil {
return nil, err
@ -741,7 +740,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
plan = append(plan, *subq_steps...)
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := util.RandomUUID()
sub_id := pilosa.RandomUUID()
loc, err := topn.getLocation(self.Database)
if err != nil {
return nil, err
@ -764,7 +763,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca
}
// Transforms Query into QueryTree and flattens to QueryPlan object
func (self *QueryPlanner) Plan(query *Query, id *util.GUID, destination *db.Location) (*QueryPlan, error) {
func (self *QueryPlanner) Plan(query *Query, id *pilosa.GUID, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.Plan", self, query, id, destination)
queryTree, err := self.buildTree(query, -1)
if err != nil {
@ -844,7 +843,7 @@ type FillResult struct {
///////////////////////////////////////////////////////////////////////////////////////////////////
type CacheItem struct {
FragmentId util.SUUID
FragmentId pilosa.SUUID
Handle pilosa.BitmapHandle
}
@ -857,7 +856,7 @@ func NewStash() Stash {
return Stash{make([]CacheItem, 0), false}
}
func (st *Stash) Add(i util.SUUID) {
func (st *Stash) Add(i pilosa.SUUID) {
item := CacheItem{i, 0}
st.Stash = append(st.Stash, item)
st.incomplete = true
@ -870,7 +869,7 @@ func (st *Stash) Assign(i pilosa.BitmapHandle) {
type StashQueryStep struct {
*BaseQueryStep
Inputs []*util.GUID
Inputs []*pilosa.GUID
N int
}

8
query/planner_test.go
View file

@ -48,7 +48,7 @@ func TestQueryPlanner_Plan_Get(t *testing.T) {
if step := (*plan)[2].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*util.GUID{
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
@ -212,7 +212,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) {
// Third step should union the first two steps.
if step := (*plan)[2].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*util.GUID{
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
@ -240,7 +240,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) {
// Sixth step should union the previous two steps.
if step := (*plan)[5].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(5) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*util.GUID{
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[3].(query.GetQueryStep).Id,
(*plan)[4].(query.GetQueryStep).Id,
}) {
@ -250,7 +250,7 @@ func TestQueryPlanner_Plan_Union(t *testing.T) {
// Final step should concatenate the two union steps.
if step := (*plan)[6].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(6) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*util.GUID{
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[2].(query.UnionQueryStep).Id,
(*plan)[5].(query.UnionQueryStep).Id,
}) {

8
query/query.go
View file

@ -4,8 +4,8 @@ import (
"strings"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
type QueryInput interface{}
@ -17,13 +17,13 @@ type QueryResults struct {
type PqlList []PqlListItem
type PqlListItem struct {
Id *util.GUID
Id *pilosa.GUID
Label string
PQL string
}
type Query struct {
Id *util.GUID
Id *pilosa.GUID
Operation string
Args map[string]interface{}
Subqueries []Query
@ -68,7 +68,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L
func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanForQuery", database, query, destination)
query_planner := QueryPlanner{Database: database, Query: query}
id := util.RandomUUID()
id := pilosa.RandomUUID()
query_plan, err := query_planner.Plan(query, &id, destination)
if err != nil {
return nil, err

24
util/statd.go → statsd/statd.go
View file

@ -1,4 +1,4 @@
package util
package statsd
import (
"time"
@ -7,11 +7,11 @@ import (
log "github.com/cihub/seelog"
)
// DefaultStatsdHost is the default host to send statsd data to.
const DefaultStatsdHost = "127.0.0.1:8125"
// DefaultHost is the default host to send statsd data to.
const DefaultHost = "127.0.0.1:8125"
// StatsdHost is the host to send statsd data to.
var StatsdHost = DefaultStatsdHost
// Host is the host to send statsd data to.
var Host = DefaultHost
type args struct {
stat string
@ -19,19 +19,17 @@ type args struct {
rate float32
}
var (
timer chan args
count chan string
end chan bool
)
var timer chan args
var count chan string
var end chan bool
func SetupStatsd() {
func Setup() {
timer = make(chan args, 32768)
count = make(chan string, 32768)
end = make(chan bool)
log.Warn("New Stats", StatsdHost)
stats, _ := statsd.New(StatsdHost, "")
log.Warn("New Stats", Host)
stats, _ := statsd.New(Host, "")
go func() {
for {

6
storage.go
View file

@ -1,9 +1,5 @@
package pilosa
import (
"github.com/umbel/pilosa/util"
)
// Storage represents
type Storage interface {
Open() error
@ -47,7 +43,7 @@ type StorageOptions struct {
DB string
Slice int
Frame string
FragmentID util.SUUID
FragmentID SUUID
LevelDBPath string
}

16
storage/cassandra/cassandra.go
View file

@ -6,7 +6,7 @@ import (
log "github.com/cihub/seelog"
"github.com/gocql/gocql"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa/statsd"
)
func init() {
@ -124,8 +124,8 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*p
lastKey = chunkKey
}
util.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds())
util.SendInc("cassandra_storage_Read")
statsd.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Read")
// Set total bits set.
bm.SetCount(uint64(count))
@ -148,8 +148,8 @@ func (s *Storage) endBatch() {
start := time.Now()
s.Flush()
util.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds())
util.SendInc("cassandra_storage_Write")
statsd.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Write")
}
// Flush flushes the current batch to storage.
@ -168,7 +168,7 @@ func (s *Storage) Flush() {
s.batchTime = time.Now()
s.batchN = 0
util.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds())
statsd.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
@ -202,7 +202,7 @@ func (s *Storage) StoreBlock(bid uint64, db string, frame string, slice int, fil
start := time.Now()
s.batch.Query(`INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?) USING timestamp ?;`,
id, db, frame, slice, int(filter), chunk, blockIndex, block, start.UnixNano())
util.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds())
statsd.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}
@ -223,7 +223,7 @@ func (s *Storage) RemoveBlock(bid uint64, db string, frame string, slice int, bc
s.batch.Query(`DELETE FROM bitmap USING TIMESTAMP ? WHERE bitmap_id=? AND db=? AND frame=? AND slice=? AND chunkkey=? AND blockindex=?`,
startTime.UnixNano(), id, db, frame, slice, chunk, blockIndex)
util.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds())
statsd.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds())
}
func (s *Storage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, val, count uint64) {

12
storage/leveldb/leveldb.go
View file

@ -10,7 +10,7 @@ import (
"github.com/syndtr/goleveldb/leveldb"
. "github.com/syndtr/goleveldb/leveldb/util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/util"
"github.com/umbel/pilosa/statsd"
)
func init() {
@ -40,7 +40,7 @@ func NewStorage(opt pilosa.StorageOptions) *Storage {
opt.DB,
strconv.Itoa(opt.Slice),
opt.Frame,
util.SUUID_to_Hex(opt.FragmentID),
pilosa.SUUID_to_Hex(opt.FragmentID),
)
return &Storage{path: path}
@ -101,7 +101,7 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*p
lastKey = key
}
util.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds())
statsd.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds())
// Set bit count on bitmap.
bm.SetCount(uint64(count))
@ -126,7 +126,7 @@ func (s *Storage) endBatch() {
if time.Since(s.batchTime) > 15*time.Second || s.batchN > 300 {
s.Flush()
}
util.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds())
statsd.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds())
}
func (s *Storage) Flush() {
@ -142,7 +142,7 @@ func (s *Storage) Flush() {
s.batchTime = time.Now()
s.batchN = 0
util.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds())
statsd.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
@ -167,7 +167,7 @@ func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, fil
func (s *Storage) StoreBlock(bitmapID uint64, db string, frame string, slice int, filter uint64, chunk uint64, index int32, block uint64) error {
start := time.Now()
s.batch.Put(marshalKey(bitmapID, chunk, uint8(index)), marshalValue(block, filter))
util.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds())
statsd.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}

20
transport/tcp.go
View file

@ -9,9 +9,9 @@ import (
notify "github.com/bitly/go-notify"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
const DefaultTCPPort = 12001
@ -21,16 +21,16 @@ type connection struct {
inbox chan *db.Message
outbox chan *db.Message
conn *net.Conn
process *util.GUID
process *pilosa.GUID
}
type newconnection struct {
id *util.GUID
id *pilosa.GUID
connection *connection
}
func init() {
gob.Register(util.GUID{})
gob.Register(pilosa.GUID{})
}
func (self *connection) manage() {
@ -79,7 +79,7 @@ BeginManageConnection:
return
}
case message := <-self.inbox:
identifier, ok := message.Data.(util.GUID)
identifier, ok := message.Data.(pilosa.GUID)
if ok {
// message is connection registration; bypass inbox and register
self.process = &identifier
@ -102,19 +102,19 @@ BeginManageConnection:
type TcpTransport struct {
inbox chan *db.Message
outbox chan db.Envelope
connections map[util.GUID]*connection
connections map[pilosa.GUID]*connection
reg chan *newconnection
ID util.GUID
ID pilosa.GUID
Port int
ProcessMap *core.ProcessMap
}
func NewTcpTransport(id util.GUID) *TcpTransport {
func NewTcpTransport(id pilosa.GUID) *TcpTransport {
return &TcpTransport{
inbox: make(chan *db.Message, 100),
outbox: make(chan db.Envelope, 100),
connections: make(map[util.GUID]*connection),
connections: make(map[pilosa.GUID]*connection),
reg: make(chan *newconnection),
ID: id,
@ -168,7 +168,7 @@ func (self *TcpTransport) Close() {
log.Warn("Shutting down TCP transport")
}
func (self *TcpTransport) Send(message *db.Message, host *util.GUID) {
func (self *TcpTransport) Send(message *db.Message, host *pilosa.GUID) {
log.Trace("TcpTransport.Send", message, host)
envelope := db.Envelope{Message: message, Host: host}
notify.Post("outbox", &envelope)

6
util/storage.go → util.go
View file

@ -1,4 +1,4 @@
package util
package pilosa
import (
"io"
@ -8,14 +8,14 @@ import (
"github.com/kr/s3/s3util"
)
func Open(s string) (io.ReadCloser, error) {
func openFile(s string) (io.ReadCloser, error) {
if isURL(s) {
return s3util.Open(s, nil)
}
return os.Open(s)
}
func Create(s string) (io.WriteCloser, error) {
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}

3
util/constants.go
View file

@ -1,3 +0,0 @@
package util
const TimeOut = 30