From 3c832591dfd02ed38d611574b0eb820d65dd0e01 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 31 Aug 2015 16:54:53 -0600 Subject: [PATCH] move index package to root package This commit moves the index package to the root. Because pilosa is an index at its core, it's redundant to have an index subpackage. It also provides better naming such as `pilosa.Bitmap` instead of `index.Bitmap`. --- ...togen_test.go => benchmark_autogen_test.go | 2 +- index/bitmap.go => bitmap.go | 2 +- index/blocks.go => blocks.go | 2 +- index/brand.go => brand.go | 2 +- index/brand_test.go => brand_test.go | 14 +-- cmd/pilosa/config.go | 8 +- cmd/pilosa/main.go | 10 +-- index/commands.go => commands.go | 2 +- core/http.go | 16 ++-- core/load.go | 16 ++-- dispatch/dispatch.go | 4 +- default.yaml => etc/default.yaml | 0 executor/executor.go | 90 +++++++++---------- ...ment_container.go => fragment_container.go | 15 +--- ...iner_test.go => fragment_container_test.go | 26 +++--- index/general.go => general.go | 2 +- index/storage/storage.go | 7 -- index/storage_test.go | 68 -------------- index/popcnt.go => popcnt.go | 2 +- index/popcnt_amd64.s => popcnt_amd64.s | 0 index/popcnt_asm.go => popcnt_asm.go | 2 +- index/popcnt_generic.go => popcnt_generic.go | 2 +- query/parser.go | 4 +- query/planner.go | 8 +- index/storage.go => storage.go | 2 +- .../cassandra/cassandra.go | 24 ++--- .../cassandra/cassandra_test.go | 0 {index/storage => storage}/leveldb/leveldb.go | 26 +++--- .../leveldb/leveldb_test.go | 6 +- {index/storage => storage}/mem/mem.go | 16 ++-- {index/storage => storage}/mem/mem_test.go | 2 +- storage/storage.go | 7 ++ index/timeframe.go => timeframe.go | 2 +- index/timeframe_test.go => timeframe_test.go | 28 +++--- util/constants.go | 28 ------ util/id.go | 25 +----- 36 files changed, 169 insertions(+), 301 deletions(-) rename index/benchmark_autogen_test.go => benchmark_autogen_test.go (99%) rename index/bitmap.go => bitmap.go (99%) rename index/blocks.go => blocks.go (98%) rename index/brand.go => brand.go (99%) rename index/brand_test.go => brand_test.go (71%) rename index/commands.go => commands.go (99%) rename default.yaml => etc/default.yaml (100%) rename index/fragment_container.go => fragment_container.go (98%) rename index/fragment_container_test.go => fragment_container_test.go (91%) rename index/general.go => general.go (99%) delete mode 100644 index/storage/storage.go delete mode 100644 index/storage_test.go rename index/popcnt.go => popcnt.go (98%) rename index/popcnt_amd64.s => popcnt_amd64.s (100%) rename index/popcnt_asm.go => popcnt_asm.go (98%) rename index/popcnt_generic.go => popcnt_generic.go (96%) rename index/storage.go => storage.go (99%) rename {index/storage => storage}/cassandra/cassandra.go (91%) rename {index/storage => storage}/cassandra/cassandra_test.go (100%) rename {index/storage => storage}/leveldb/leveldb.go (89%) rename {index/storage => storage}/leveldb/leveldb_test.go (86%) rename {index/storage => storage}/mem/mem.go (88%) rename {index/storage => storage}/mem/mem_test.go (92%) create mode 100644 storage/storage.go rename index/timeframe.go => timeframe.go (99%) rename index/timeframe_test.go => timeframe_test.go (85%) diff --git a/index/benchmark_autogen_test.go b/benchmark_autogen_test.go similarity index 99% rename from index/benchmark_autogen_test.go rename to benchmark_autogen_test.go index 24a5bb542..b9185ba44 100644 --- a/index/benchmark_autogen_test.go +++ b/benchmark_autogen_test.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "math/rand" diff --git a/index/bitmap.go b/bitmap.go similarity index 99% rename from index/bitmap.go rename to bitmap.go index 8cf7c4fbf..c457e8e62 100644 --- a/index/bitmap.go +++ b/bitmap.go @@ -1,4 +1,4 @@ -package index +package pilosa // #cgo CFLAGS:-mpopcnt diff --git a/index/blocks.go b/blocks.go similarity index 98% rename from index/blocks.go rename to blocks.go index 8e389cbc5..6803b84aa 100644 --- a/index/blocks.go +++ b/blocks.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "fmt" diff --git a/index/brand.go b/brand.go similarity index 99% rename from index/brand.go rename to brand.go index 18498f355..f3faf4e2f 100644 --- a/index/brand.go +++ b/brand.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "encoding/json" diff --git a/index/brand_test.go b/brand_test.go similarity index 71% rename from index/brand_test.go rename to brand_test.go index fdabd05b2..2b3ec908d 100644 --- a/index/brand_test.go +++ b/brand_test.go @@ -1,18 +1,18 @@ -package index_test +package pilosa_test import ( "math/rand" "testing" - "github.com/umbel/pilosa/index" - "github.com/umbel/pilosa/index/storage/mem" + "github.com/umbel/pilosa" + "github.com/umbel/pilosa/storage/mem" "github.com/umbel/pilosa/util" ) var ( size int - membrand *index.Brand - cassbrand *index.Brand + membrand *pilosa.Brand + cassbrand *pilosa.Brand ) func init() { @@ -21,7 +21,7 @@ func init() { util.SetupStatsd() // SetupCassandra() - membrand = index.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0) + membrand = pilosa.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0) for i := uint64(0); i < uint64(size); i++ { membrand.SetBit(i, 0, 1) } @@ -31,7 +31,7 @@ func init() { // } } -func benchmarkBrand(b *testing.B, size int, fill int, brand *index.Brand) { +func benchmarkBrand(b *testing.B, size int, fill int, brand *pilosa.Brand) { println(b.N) for i := 0; i < b.N; i++ { bid := rand.Int() % size diff --git a/cmd/pilosa/config.go b/cmd/pilosa/config.go index 81c4a8d70..4e5778e28 100644 --- a/cmd/pilosa/config.go +++ b/cmd/pilosa/config.go @@ -3,9 +3,9 @@ package main import ( "time" - "github.com/umbel/pilosa/index" - _ "github.com/umbel/pilosa/index/storage" - "github.com/umbel/pilosa/index/storage/cassandra" + "github.com/umbel/pilosa" + _ "github.com/umbel/pilosa/storage" + "github.com/umbel/pilosa/storage/cassandra" "github.com/umbel/pilosa/transport" "github.com/umbel/pilosa/util" ) @@ -92,7 +92,7 @@ func NewConfig() Config { c.TCP.Port = transport.DefaultTCPPort c.HTTP.Port = transport.DefaultHTTPPort c.Log.Path = DefaultLogPath - c.Storage.Backend = index.DefaultBackend + c.Storage.Backend = pilosa.DefaultBackend c.Storage.Hosts = cassandra.DefaultHosts c.Storage.Keyspace = cassandra.DefaultKeyspace c.Storage.FragmentBase = DefaultFragmentBase diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 1b4f80544..088a6ef1b 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -13,12 +13,12 @@ import ( "github.com/coreos/go-etcd/etcd" "github.com/kr/s3/s3util" "github.com/mitchellh/panicwrap" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/core" "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/dispatch" "github.com/umbel/pilosa/executor" "github.com/umbel/pilosa/hold" - "github.com/umbel/pilosa/index" "github.com/umbel/pilosa/transport" "github.com/umbel/pilosa/util" ) @@ -96,9 +96,9 @@ func (m *Main) Run(args ...string) error { // Pass configuration to packages. // NOTE: This is temporary. These config options should be encapsulated in the types. db.SupportedFrames = config.Storage.SupportedFrames - index.FragmentBase = config.Storage.FragmentBase - index.Backend = config.Storage.Backend - index.LevelDBPath = config.LevelDB.Path + pilosa.FragmentBase = config.Storage.FragmentBase + pilosa.Backend = config.Storage.Backend + pilosa.LevelDBPath = config.LevelDB.Path // Initialize AWS storage. s3util.DefaultConfig.AccessKey = config.AWS.AccessKeyID @@ -119,7 +119,7 @@ func (m *Main) Run(args ...string) error { cluster := db.NewCluster() // Create index. - idx := index.NewFragmentContainer() + idx := pilosa.NewFragmentContainer() // Initialize the holder. hold := hold.NewHolder() diff --git a/index/commands.go b/commands.go similarity index 99% rename from index/commands.go rename to commands.go index d1ad4302f..73c723d3e 100644 --- a/index/commands.go +++ b/commands.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "bytes" diff --git a/core/http.go b/core/http.go index c63058221..95daba26f 100644 --- a/core/http.go +++ b/core/http.go @@ -20,8 +20,8 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" "github.com/gorilla/websocket" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/index" "github.com/umbel/pilosa/util" ) @@ -401,7 +401,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) { return } switch r := results.(type) { //a hack to handle empty sets - case []index.Pair: + case []pilosa.Pair: if len(r) == 0 { results = []int{} @@ -410,7 +410,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) { if bits { reader, _ := gzip.NewReader(bytes.NewReader(r)) b, _ := ioutil.ReadAll(reader) - result := index.NewBitmap() + result := pilosa.NewBitmap() result.FromBytes(b) results = result.Bits() } @@ -460,15 +460,15 @@ func bitmaps(frame string, obj JsonObject) chan uint64 { if !present || timestamp == "2014-01-01 00:00:00" { //skip the default timestamp c <- base_id } else { - quantum := index.YMDH + quantum := pilosa.YMDH if val, ok := obj["time_granularity"]; ok { switch val { case "Y": - quantum = index.Y + quantum = pilosa.Y case "M": - quantum = index.YM + quantum = pilosa.YM case "D": - quantum = index.YMD + quantum = pilosa.YMD } } shortForm := shortFormS @@ -477,7 +477,7 @@ func bitmaps(frame string, obj JsonObject) chan uint64 { } atime, _ := time.Parse(shortForm, timestamp) - for i, id := range index.GetTimeIds(base_id, atime, quantum) { + for i, id := range pilosa.GetTimeIds(base_id, atime, quantum) { c <- id if i > 10 { log.Warn("TO MANY TIMEIDS", base_id, atime, quantum) diff --git a/core/load.go b/core/load.go index afee1b615..a98f871ef 100644 --- a/core/load.go +++ b/core/load.go @@ -10,17 +10,17 @@ import ( "errors" log "github.com/cihub/seelog" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" ) -func copy_raw(src [32]uint64) index.Blocks { - o := make(index.Blocks, 32) +func copy_raw(src [32]uint64) pilosa.Blocks { + o := make(pilosa.Blocks, 32) for k, v := range src { o[k] = v } return o } -func sendBitmap(batcher *Batcher, bitmap *index.Bitmap, db string, frame string, bitmap_id, filter uint64, slice int, finish chan error) { +func sendBitmap(batcher *Batcher, bitmap *pilosa.Bitmap, db string, frame string, bitmap_id, filter uint64, slice int, finish chan error) { if slice < 0 { log.Warn("Bad split", db, frame, slice, bitmap_id) finish <- errors.New("BadSplit") @@ -57,8 +57,8 @@ func FromApiString(batcher *Batcher, db string, frame string, api_string string, return "Bad" } first := true - bitmap := index.NewBitmap() - last_slice := index.CounterMask + bitmap := pilosa.NewBitmap() + last_slice := pilosa.CounterMask sent_count := 0 finish := make(chan error) @@ -76,12 +76,12 @@ func FromApiString(batcher *Batcher, db string, frame string, api_string string, //make async later sent_count += 1 go sendBitmap(batcher, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish) - bitmap = index.NewBitmap() + bitmap = pilosa.NewBitmap() } last_slice = slice } o := copy_raw(raw.Block) - chunk := &index.Chunk{Key: raw.Key, Value: o} + chunk := &pilosa.Chunk{Key: raw.Key, Value: o} bitmap.AddChunk(chunk) } diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index af2bff717..a64d2f2f0 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -3,10 +3,10 @@ package dispatch import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/core" "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/executor" - "github.com/umbel/pilosa/index" "github.com/umbel/pilosa/query" "github.com/umbel/pilosa/util" ) @@ -24,7 +24,7 @@ type Dispatch struct { ClearBit(fragID util.SUUID, bitmapID uint64, pos uint64) (bool, error) LoadBitmap(fragID util.SUUID, bitmapID uint64, compressedBitmap string, filter uint64) SetBit(fragID util.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error) - TopFillBatch(args []index.FillArgs) ([]index.Pair, error) + TopFillBatch(args []pilosa.FillArgs) ([]pilosa.Pair, error) } Transport interface { diff --git a/default.yaml b/etc/default.yaml similarity index 100% rename from default.yaml rename to etc/default.yaml diff --git a/executor/executor.go b/executor/executor.go index a10470ea7..30ae1c776 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -7,9 +7,9 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/core" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/index" "github.com/umbel/pilosa/query" "github.com/umbel/pilosa/util" ) @@ -29,17 +29,17 @@ type Executor struct { Index interface { ClearBit(frag_id util.SUUID, bitmap_id uint64, pos uint64) (bool, error) - Count(frag_id util.SUUID, bitmap index.BitmapHandle) (uint64, error) - Difference(frag_id util.SUUID, bh []index.BitmapHandle) (index.BitmapHandle, error) - FromBytes(frag_id util.SUUID, bytes []byte) (index.BitmapHandle, error) - Get(frag_id util.SUUID, bitmap_id uint64) (index.BitmapHandle, error) - GetBytes(frag_id util.SUUID, bh index.BitmapHandle) ([]byte, error) - Intersect(frag_id util.SUUID, bh []index.BitmapHandle) (index.BitmapHandle, error) - Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (index.BitmapHandle, error) + Count(frag_id util.SUUID, bitmap pilosa.BitmapHandle) (uint64, error) + Difference(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) + FromBytes(frag_id util.SUUID, bytes []byte) (pilosa.BitmapHandle, error) + Get(frag_id util.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error) + GetBytes(frag_id util.SUUID, bh pilosa.BitmapHandle) ([]byte, error) + Intersect(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) + Range(frag_id util.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error) SetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) - TopN(frag_id util.SUUID, bh index.BitmapHandle, n int, categories []uint64) ([]index.Pair, error) - TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]index.Pair, error) - Union(frag_id util.SUUID, bh []index.BitmapHandle) (index.BitmapHandle, error) + TopN(frag_id util.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error) + TopNAll(frag_id util.SUUID, n int, categories []uint64) ([]pilosa.Pair, error) + Union(frag_id util.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error) } TopologyMapper interface { @@ -102,9 +102,9 @@ func (self *Executor) CountQueryStepHandler(msg *db.Message) { qs := msg.Data.(query.CountQueryStep) input := qs.Input value, _ := self.Hold.Get(input, util.TimeOut) - var bh index.BitmapHandle + var bh pilosa.BitmapHandle switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: bh = val case []byte: bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val) @@ -124,7 +124,7 @@ func (self *Executor) CountQueryStepHandler(msg *db.Message) { func (self *Executor) TopNQueryStepHandler(msg *db.Message) { qs := msg.Data.(query.TopNQueryStep) - var bh index.BitmapHandle + var bh pilosa.BitmapHandle var topnPackage TopNPackage // if we have an input, hold for it. if we don't, we assume an all() query @@ -137,9 +137,9 @@ func (self *Executor) TopNQueryStepHandler(msg *db.Message) { } else { input := qs.Input value, _ := self.Hold.Get(input, 10) - //var bh index.BitmapHandle + //var bh pilosa.BitmapHandle switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: bh = val case []byte: bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val) @@ -165,12 +165,12 @@ func (self *Executor) UnionQueryStepHandler(msg *db.Message) { //spew.Dump("UNION QUERYSTEP") qs := msg.Data.(query.UnionQueryStep) - var handles []index.BitmapHandle + var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { value, _ := self.Hold.Get(input, util.TimeOut) switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: handles = append(handles, val) case []byte: bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val) @@ -205,12 +205,12 @@ func (self *Executor) IntersectQueryStepHandler(msg *db.Message) { log.Trace("IntersectQueryStepHandler") //spew.Dump("INTERSECT QUERYSTEP") qs := msg.Data.(query.IntersectQueryStep) - var handles []index.BitmapHandle + var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { value, _ := self.Hold.Get(input, util.TimeOut) switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: handles = append(handles, val) case []byte: bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val) @@ -245,12 +245,12 @@ func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) { log.Trace("DifferenceQueryStepHandler") //spew.Dump("DIFFERENCE QUERYSTEP") qs := msg.Data.(query.DifferenceQueryStep) - var handles []index.BitmapHandle + var handles []pilosa.BitmapHandle // create a list of bitmap handles for _, input := range qs.Inputs { value, _ := self.Hold.Get(input, util.TimeOut) switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: handles = append(handles, val) case []byte: bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val) @@ -284,14 +284,14 @@ func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) { func (self *Executor) CatQueryStepHandler(msg *db.Message) { log.Trace("CatQueryStepHandler") qs := msg.Data.(query.CatQueryStep) - var handles []index.BitmapHandle + var handles []pilosa.BitmapHandle return_type := "bitmap-handles" var sum uint64 merge_map := make(map[uint64]uint64) slice_map := make(map[uint64]map[util.SUUID]struct{}) all_slice := make(map[util.SUUID]struct { process util.GUID - handle index.BitmapHandle + handle pilosa.BitmapHandle }) // either create a list of bitmap handles to cat (i.e. union), or sum the integer values @@ -311,7 +311,7 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { value := <-part switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: handles = append(handles, val) case []byte: bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val) @@ -338,7 +338,7 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { } all_slice[val.FragmentId] = struct { process util.GUID - handle index.BitmapHandle + handle pilosa.BitmapHandle }{val.ProcessId, val.HBitmap} check_pair = true } @@ -362,18 +362,18 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { spew.Dump(err) } } else if return_type == "pair-list" { - rank_list := make(index.RankList, 0, len(merge_map)) + rank_list := make(pilosa.RankList, 0, len(merge_map)) for k, v := range merge_map { if k == 0 || v == 0 { continue //shouldn't be getting 0 keys or values anyway } - rank := new(index.Rank) - rank.Pair = &index.Pair{Key: k, Count: v} + rank := new(pilosa.Rank) + rank.Pair = &pilosa.Pair{Key: k, Count: v} rank_list = append(rank_list, rank) } sort.Sort(rank_list) // kinda seems like this copy is wasteful..i'll ponder items_size := min(len(merge_map), qs.N) - pair_list := make([]index.Pair, 0, items_size+1) + pair_list := make([]pilosa.Pair, 0, items_size+1) for i, r := range rank_list { if i < items_size { pair_list = append(pair_list, *r.Pair) @@ -394,7 +394,7 @@ func (self *Executor) CatQueryStepHandler(msg *db.Message) { } func (self *Executor) SendRequest(process_id util.GUID, t *Task) { - args := make([]index.FillArgs, len(t.f), len(t.f)) + args := make([]pilosa.FillArgs, len(t.f), len(t.f)) for _, v := range t.f { args = append(args, v) } @@ -412,17 +412,17 @@ func (self *Executor) FetchMissing(tasks map[util.GUID]*Task) { func (self *Executor) GatherResults(tasks map[util.GUID]*Task) map[uint64]uint64 { results := make(map[uint64]uint64) - answers := make(chan []index.Pair) + answers := make(chan []pilosa.Pair) for _, task := range tasks { go func(id util.GUID) { value, err := self.Hold.Get(&id, 10) //eiher need to be the frame process or the handler process? if value == nil { log.Warn("Bad TopN Result:", err) - empty := make([]index.Pair, 0, 0) + empty := make([]pilosa.Pair, 0, 0) answers <- empty } else { - answers <- value.([]index.Pair) + answers <- value.([]pilosa.Pair) } }(task.hold_id) } @@ -539,7 +539,7 @@ func (self *Executor) StashQueryStepHandler(msg *db.Message) { value := <-part switch val := value.(type) { - case index.BitmapHandle: + case pilosa.BitmapHandle: log.Info("STASH ADDING HANDLE", val) //not sure what to do here.... //result.Handles = append(result.Handles, val) @@ -687,12 +687,12 @@ func init() { type TopNPackage struct { ProcessId util.GUID FragmentId util.SUUID - Pairs []index.Pair - HBitmap index.BitmapHandle + Pairs []pilosa.Pair + HBitmap pilosa.BitmapHandle } type TopFill struct { - Args []index.FillArgs + Args []pilosa.FillArgs ReturnProcessId util.GUID QueryId util.GUID DestProcessId util.GUID @@ -700,22 +700,22 @@ type TopFill struct { type Task struct { processid util.GUID - f map[util.SUUID]index.FillArgs + f map[util.SUUID]pilosa.FillArgs hold_id util.GUID } func newtask(p util.GUID) *Task { result := new(Task) result.processid = p - result.f = make(map[util.SUUID]index.FillArgs) + result.f = make(map[util.SUUID]pilosa.FillArgs) result.hold_id = util.RandomUUID() return result } -func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle index.BitmapHandle) { +func (t *Task) Add(frag util.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) { fa, ok := t.f[frag] if !ok { - fa = index.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)} + fa = pilosa.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)} } fa.Bitmaps = append(fa.Bitmaps, bitmap_id) t.f[frag] = fa @@ -725,7 +725,7 @@ func BuildTask(merge_map map[uint64]uint64, slice_map map[uint64]map[util.SUUID]struct{}, total_fragments map[util.SUUID]struct { process util.GUID - handle index.BitmapHandle + handle pilosa.BitmapHandle }) map[util.GUID]*Task { tasks := make(map[util.GUID]*Task) @@ -750,13 +750,13 @@ func BuildTask(merge_map map[uint64]uint64, type hole struct { process util.GUID - handle index.BitmapHandle + handle pilosa.BitmapHandle fragment util.SUUID } func missing(fids map[util.SUUID]struct{}, all map[util.SUUID]struct { process util.GUID - handle index.BitmapHandle + handle pilosa.BitmapHandle }) []hole { results := make([]hole, 0, 0) diff --git a/index/fragment_container.go b/fragment_container.go similarity index 98% rename from index/fragment_container.go rename to fragment_container.go index a1591de0c..df25f34ec 100644 --- a/index/fragment_container.go +++ b/fragment_container.go @@ -1,7 +1,6 @@ -package index +package pilosa import ( - "database/sql" "encoding/gob" "errors" "fmt" @@ -11,7 +10,6 @@ import ( "time" log "github.com/cihub/seelog" - _ "github.com/go-sql-driver/mysql" "github.com/golang/groupcache/lru" "github.com/umbel/pilosa/util" ) @@ -28,17 +26,6 @@ type FragmentContainer struct { mutex *sync.Mutex } -func lookup(stmt *sql.Stmt, tile_id uint64) int { - var category int - err := stmt.QueryRow(tile_id).Scan(&category) // WHERE number = 13 - if err != nil { - log.Warn(err.Error()) - return 0 - } - return category - -} - func NewFragmentContainer() *FragmentContainer { f := new(FragmentContainer) f.fragments = make(map[util.SUUID]*Fragment) diff --git a/index/fragment_container_test.go b/fragment_container_test.go similarity index 91% rename from index/fragment_container_test.go rename to fragment_container_test.go index c92daccce..1bda3490a 100644 --- a/index/fragment_container_test.go +++ b/fragment_container_test.go @@ -1,15 +1,15 @@ -package index_test +package pilosa_test import ( "testing" - "github.com/umbel/pilosa/index" - _ "github.com/umbel/pilosa/index/storage" + "github.com/umbel/pilosa" + _ "github.com/umbel/pilosa/storage" "github.com/umbel/pilosa/util" ) func init() { - index.Backend = "memory" + pilosa.Backend = "memory" } // Ensure a fragment can be retrieved from the container. @@ -77,7 +77,7 @@ func TestFragmentContainer_Union(t *testing.T) { fc.MustSetBit(1, 4321, 65537, 0) // Union the handles together. - if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { t.Fatal(err) } else if n := fc.MustCount(1, result); n != 2 { t.Fatalf("unexpected union bit count: %d", n) @@ -91,7 +91,7 @@ func TestFragmentContainer_Union_Empty(t *testing.T) { fc.MustSetBit(1, 1234, 1, 0) // Union the handles together. - if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { t.Fatal(err) } else if n := fc.MustCount(1, result); n != 1 { t.Fatalf("unexpected empty union bit count: %d", n) @@ -106,7 +106,7 @@ func TestFragmentContainer_Intersect(t *testing.T) { fc.MustSetBit(1, 4321, 65537, 0) // Intersect the handles together. - if result, err := fc.Intersect(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + if result, err := fc.Intersect(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { t.Fatal(err) } else if n := fc.MustCount(1, result); n != 0 { t.Fatalf("unexpected intersect bit count: %d", n) @@ -121,7 +121,7 @@ func TestFragmentContainer_Difference(t *testing.T) { fc.MustSetBit(1, 4321, 65537, 0) // Compute the difference between the handles. - if result, err := fc.Difference(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + if result, err := fc.Difference(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { t.Fatal(err) } else if n := fc.MustCount(1, result); n != 1 { t.Fatalf("unexpected difference bit count: %s", err) @@ -262,18 +262,18 @@ func TestFragmentContainer_FromBytes(t *testing.T) { } } -// FragementContainer is a test wrapper for index.FragmentContainer. +// FragementContainer is a test wrapper for pilosa.FragmentContainer. type FragmentContainer struct { - *index.FragmentContainer + *pilosa.FragmentContainer } // NewFragmentContainer returns a new instance of FragmentContainer. func NewFragmentContainer() *FragmentContainer { - return &FragmentContainer{index.NewFragmentContainer()} + return &FragmentContainer{pilosa.NewFragmentContainer()} } // MustGet retrieves a bitmap by id. Panic on error. -func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) index.BitmapHandle { +func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) pilosa.BitmapHandle { bh, err := fc.Get(frag_id, bitmap_id) if err != nil { panic(err) @@ -300,7 +300,7 @@ func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool { } // MustCount returns the number of set bits in a bitmap. Panic on error. -func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap index.BitmapHandle) uint64 { +func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap pilosa.BitmapHandle) uint64 { v, err := fc.Count(frag_id, bitmap) if err != nil { panic(err) diff --git a/index/general.go b/general.go similarity index 99% rename from index/general.go rename to general.go index dd1f0cc05..e1e4cd1ad 100644 --- a/index/general.go +++ b/general.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "encoding/json" diff --git a/index/storage/storage.go b/index/storage/storage.go deleted file mode 100644 index 40ae92486..000000000 --- a/index/storage/storage.go +++ /dev/null @@ -1,7 +0,0 @@ -package storage - -import ( - _ "github.com/umbel/pilosa/index/storage/cassandra" - _ "github.com/umbel/pilosa/index/storage/leveldb" - _ "github.com/umbel/pilosa/index/storage/mem" -) diff --git a/index/storage_test.go b/index/storage_test.go deleted file mode 100644 index 236d9812e..000000000 --- a/index/storage_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package index - -/* -import ( - "fmt" - "net" - "testing" - "time" - -) - -func TestStorage(t *testing.T) { - if true { - return - } - db := "db" - frame := "main" - slice := 0 - filter := 10 - bitmap_id := uint64(999999) - - c, err := net.DialTimeout("tcp", "127.0.0.1:9042", 100*time.Millisecond) - if err != nil { - fmt.Println("NO cassandra. Skipping test.") - } else { - c.Close() - Convey("cassandra", t, func() { - fmt.Println("GO") - //storage := NewCassStorage("127.0.0.1", "pilosa") - storage := NewCassStorage() - - fmt.Println("FETCH") - bm, _ := storage.Fetch(bitmap_id, db, frame, slice) - for i := uint64(0); i < 256; i++ { - SetBit(bm, i) - } - fmt.Println("STORE") - storage.Store(uint64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap)) - fmt.Println("FETCH") - bm2, _ := storage.Fetch(bitmap_id, db, frame, slice) - So(BitCount(bm), ShouldEqual, BitCount(bm2)) - So(BitCount(bm), ShouldEqual, bm.Count()) - So(BitCount(bm), ShouldEqual, 256) - - }) - } - - // Convey("leveldb", t, func() { - // storage := NewLevelDBStorage("./basic/one") - - // fmt.Println("FETCH") - // bm, _ := storage.Fetch(bitmap_id, db, frame, slice) - // //spew.Dump(bm) - // SetBit(bm, 0) - // SetBit(bm, 1) - // SetBit(bm, 2) - // fmt.Println("STORE") - // storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap)) - // //storage.Flush() - // fmt.Println("FETCH") - // bm2, _ := storage.Fetch(bitmap_id, db, frame, slice) - // So(BitCount(bm), ShouldEqual, BitCount(bm2)) - // So(BitCount(bm), ShouldEqual, bm.Count()) - // So(BitCount(bm), ShouldEqual, 3) - // storage.Close() - // }) -} -*/ diff --git a/index/popcnt.go b/popcnt.go similarity index 98% rename from index/popcnt.go rename to popcnt.go index a1f25c8a8..ff9085fc7 100644 --- a/index/popcnt.go +++ b/popcnt.go @@ -1,4 +1,4 @@ -package index +package pilosa // bit population count, take from // https://code.google.com/p/go/issues/detail?id=4988#c11 diff --git a/index/popcnt_amd64.s b/popcnt_amd64.s similarity index 100% rename from index/popcnt_amd64.s rename to popcnt_amd64.s diff --git a/index/popcnt_asm.go b/popcnt_asm.go similarity index 98% rename from index/popcnt_asm.go rename to popcnt_asm.go index 6af8ad422..c63dca950 100644 --- a/index/popcnt_asm.go +++ b/popcnt_asm.go @@ -1,6 +1,6 @@ // +build amd64 -package index +package pilosa //go:noescape diff --git a/index/popcnt_generic.go b/popcnt_generic.go similarity index 96% rename from index/popcnt_generic.go rename to popcnt_generic.go index d79741573..72cdbb46a 100644 --- a/index/popcnt_generic.go +++ b/popcnt_generic.go @@ -1,6 +1,6 @@ // +build !amd64 -package index +package pilosa func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) diff --git a/query/parser.go b/query/parser.go index c6db4f359..2cc0e6107 100644 --- a/query/parser.go +++ b/query/parser.go @@ -8,7 +8,7 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/util" ) @@ -221,7 +221,7 @@ ArgLoop: if err != nil { return nil, fmt.Errorf("Expecting handle! (%v)", err) } - recallArgs.Assign(index.BitmapHandle(i)) //sets the value of the last created arg + recallArgs.Assign(pilosa.BitmapHandle(i)) //sets the value of the last created arg } default: spew.Dump("UNPROCESSED VALUE", token) diff --git a/query/planner.go b/query/planner.go index 4a0326616..d4132c958 100644 --- a/query/planner.go +++ b/query/planner.go @@ -9,8 +9,8 @@ import ( log "github.com/cihub/seelog" "github.com/davecgh/go-spew/spew" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/db" - "github.com/umbel/pilosa/index" "github.com/umbel/pilosa/util" ) @@ -845,11 +845,11 @@ type FillResult struct { type CacheItem struct { FragmentId util.SUUID - Handle index.BitmapHandle + Handle pilosa.BitmapHandle } type Stash struct { - Stash []CacheItem //index.BitmapHandle //probably need to make the a struct with fragment_id and handle + Stash []CacheItem //pilosa.BitmapHandle //probably need to make the a struct with fragment_id and handle incomplete bool } @@ -863,7 +863,7 @@ func (st *Stash) Add(i util.SUUID) { st.incomplete = true } -func (st *Stash) Assign(i index.BitmapHandle) { +func (st *Stash) Assign(i pilosa.BitmapHandle) { st.Stash[len(st.Stash)-1].Handle = i //big assumption that item already exists st.incomplete = false } diff --git a/index/storage.go b/storage.go similarity index 99% rename from index/storage.go rename to storage.go index 7ee94f481..64ac66ff3 100644 --- a/index/storage.go +++ b/storage.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "github.com/umbel/pilosa/util" diff --git a/index/storage/cassandra/cassandra.go b/storage/cassandra/cassandra.go similarity index 91% rename from index/storage/cassandra/cassandra.go rename to storage/cassandra/cassandra.go index 28a44ed4f..4186a6f2a 100644 --- a/index/storage/cassandra/cassandra.go +++ b/storage/cassandra/cassandra.go @@ -5,13 +5,13 @@ import ( log "github.com/cihub/seelog" "github.com/gocql/gocql" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/util" ) func init() { - index.RegisterStorage("cassandra", - func(opt index.StorageOptions) index.Storage { + pilosa.RegisterStorage("cassandra", + func(opt pilosa.StorageOptions) pilosa.Storage { return NewStorage(opt) }, ) @@ -52,7 +52,7 @@ type Storage struct { } // NewStorage returns a new, uninitialized instance of Storage. -func NewStorage(opt index.StorageOptions) *Storage { +func NewStorage(opt pilosa.StorageOptions) *Storage { return &Storage{ batchTime: time.Now(), @@ -94,8 +94,8 @@ func (s *Storage) Close() error { } // Fetch returns a bitmap by ID. -func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*index.Bitmap, uint64) { - bm := index.NewBitmap() +func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) { + bm := pilosa.NewBitmap() // Start benchmark. start := time.Now() @@ -105,7 +105,7 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*i u64toi64(bitmapID), db, frame, slice).Iter() // Iterate over chunks and materialize bitmap object. - var chunk *index.Chunk + var chunk *pilosa.Chunk var chunkKey, block, count int64 var blockIndex uint32 var filter int @@ -114,7 +114,7 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*i for itr.Scan(&filter, &chunkKey, &blockIndex, &block) { if chunkKey != int64(-1) { if chunkKey != lastKey { - chunk = &index.Chunk{uint64(chunkKey), index.NewBlocks()} + chunk = &pilosa.Chunk{uint64(chunkKey), pilosa.NewBlocks()} bm.AddChunk(chunk) } chunk.Value[uint8(blockIndex)] = uint64(block) @@ -172,7 +172,7 @@ func (s *Storage) Flush() { } // Store saves a bitmap to storage. -func (s *Storage) Store(id uint64, db string, frame string, slice int, filter uint64, bm *index.Bitmap) error { +func (s *Storage) Store(id uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error { s.beginBatch() for i := bm.ChunkIterator(); !i.Limit(); i = i.Next() { @@ -184,7 +184,7 @@ func (s *Storage) Store(id uint64, db string, frame string, slice int, filter ui } } - s.StoreBlock(id, db, frame, slice, filter, index.CounterMask, 0, bm.BitCount()) + s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount()) s.endBatch() return nil } @@ -229,7 +229,7 @@ func (s *Storage) RemoveBlock(bid uint64, db string, frame string, slice int, bc func (s *Storage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, val, count uint64) { s.beginBatch() s.StoreBlock(bid, db, frame, slice, filter, chunk, blockIndex, val) - s.StoreBlock(bid, db, frame, slice, filter, index.CounterMask, 0, count) + s.StoreBlock(bid, db, frame, slice, filter, pilosa.CounterMask, 0, count) s.endBatch() } @@ -237,7 +237,7 @@ func (s *Storage) RemoveBit(id uint64, db string, frame string, slice int, filte log.Trace("RemoveBit", id, db, frame, slice, chunk, blockIndex) s.beginBatch() s.RemoveBlock(id, db, frame, slice, chunk, blockIndex) - s.StoreBlock(id, db, frame, slice, filter, index.CounterMask, 0, count) + s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, count) s.endBatch() } diff --git a/index/storage/cassandra/cassandra_test.go b/storage/cassandra/cassandra_test.go similarity index 100% rename from index/storage/cassandra/cassandra_test.go rename to storage/cassandra/cassandra_test.go diff --git a/index/storage/leveldb/leveldb.go b/storage/leveldb/leveldb.go similarity index 89% rename from index/storage/leveldb/leveldb.go rename to storage/leveldb/leveldb.go index 7320d6da0..bcdf8cc58 100644 --- a/index/storage/leveldb/leveldb.go +++ b/storage/leveldb/leveldb.go @@ -9,13 +9,13 @@ import ( "github.com/syndtr/goleveldb/leveldb" . "github.com/syndtr/goleveldb/leveldb/util" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" "github.com/umbel/pilosa/util" ) func init() { - index.RegisterStorage("leveldb", - func(opt index.StorageOptions) index.Storage { + pilosa.RegisterStorage("leveldb", + func(opt pilosa.StorageOptions) pilosa.Storage { return NewStorage(opt) }, ) @@ -34,7 +34,7 @@ type Storage struct { } // NewStorage returns a new instance of Storage. -func NewStorage(opt index.StorageOptions) *Storage { +func NewStorage(opt pilosa.StorageOptions) *Storage { path := filepath.Join( opt.LevelDBPath, opt.DB, @@ -67,8 +67,8 @@ func (s *Storage) Close() error { return s.db.Close() } -func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*index.Bitmap, uint64) { - bm := index.NewBitmap() +func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) { + bm := pilosa.NewBitmap() // Begin benchmark. start := time.Now() @@ -81,16 +81,16 @@ func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*i defer iter.Release() // Iterate over blocks in database and create bitmap. - var chunk *index.Chunk + var chunk *pilosa.Chunk var filter, block, count uint64 - lastKey := uint64(index.CounterMask) + lastKey := uint64(pilosa.CounterMask) for iter.Next() { _, key, idx := unmarshalKey(iter.Key()) block, filter = unmarshalValue(iter.Value()) - if key != index.CounterMask { + if key != pilosa.CounterMask { if key != lastKey { - chunk = &index.Chunk{key, index.NewBlocks()} + chunk = &pilosa.Chunk{key, pilosa.NewBlocks()} bm.AddChunk(chunk) } chunk.Value[idx] = block @@ -146,7 +146,7 @@ func (s *Storage) Flush() { } // Store saves a bitmap to storage. -func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, filter uint64, bm *index.Bitmap) error { +func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error { s.beginBatch() for itr := bm.ChunkIterator(); !itr.Limit(); itr = itr.Next() { @@ -157,7 +157,7 @@ func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, fil } } - s.StoreBlock(bitmapID, db, frame, slice, filter, index.CounterMask, 0, bm.BitCount()) + s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount()) s.endBatch() return nil @@ -179,7 +179,7 @@ func (s *Storage) RemoveBlock(bitmapID uint64, db string, frame string, slice in func (s *Storage) StoreBit(bitmapID uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, bblock, count uint64) { s.beginBatch() s.StoreBlock(bitmapID, db, frame, slice, filter, bchunk, blockIndex, bblock) - s.StoreBlock(bitmapID, db, frame, slice, filter, index.CounterMask, 0, count) + s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, count) s.endBatch() } diff --git a/index/storage/leveldb/leveldb_test.go b/storage/leveldb/leveldb_test.go similarity index 86% rename from index/storage/leveldb/leveldb_test.go rename to storage/leveldb/leveldb_test.go index c55cc53e5..e956078c8 100644 --- a/index/storage/leveldb/leveldb_test.go +++ b/storage/leveldb/leveldb_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/umbel/pilosa/index" - "github.com/umbel/pilosa/index/storage/leveldb" + "github.com/umbel/pilosa" + "github.com/umbel/pilosa/storage/leveldb" "github.com/umbel/pilosa/util" ) @@ -34,7 +34,7 @@ func NewStorage() *Storage { f.Close() os.Remove(f.Name()) - return &Storage{leveldb.NewStorage(index.StorageOptions{ + return &Storage{leveldb.NewStorage(pilosa.StorageOptions{ LevelDBPath: f.Name(), })} } diff --git a/index/storage/mem/mem.go b/storage/mem/mem.go similarity index 88% rename from index/storage/mem/mem.go rename to storage/mem/mem.go index 5924cf408..bf5bdbc82 100644 --- a/index/storage/mem/mem.go +++ b/storage/mem/mem.go @@ -3,12 +3,12 @@ package mem import ( "fmt" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" ) func init() { - index.RegisterStorage("memory", - func(opt index.StorageOptions) index.Storage { + pilosa.RegisterStorage("memory", + func(opt pilosa.StorageOptions) pilosa.Storage { return NewStorage() }, ) @@ -16,13 +16,13 @@ func init() { // Storage represents in-memory bitmap storage. type Storage struct { - db map[string]*index.Bitmap + db map[string]*pilosa.Bitmap } // NewStorage returns a new instance of Storage. func NewStorage() *Storage { return &Storage{ - db: make(map[string]*index.Bitmap), + db: make(map[string]*pilosa.Bitmap), } } @@ -36,7 +36,7 @@ func (c *Storage) Close() error { return nil } func (c *Storage) Flush() {} // Fetch retrieves a bitmap by id. -func (c *Storage) Fetch(id uint64, db, frame string, slice int) (*index.Bitmap, uint64) { +func (c *Storage) Fetch(id uint64, db, frame string, slice int) (*pilosa.Bitmap, uint64) { key := fmt.Sprintf("%d:%s:%s:%d", id, db, frame, slice) // Find bitmap by key. @@ -46,14 +46,14 @@ func (c *Storage) Fetch(id uint64, db, frame string, slice int) (*index.Bitmap, } // If the bitmap doesn't exist then create a new one. - b = index.NewBitmap() + b = pilosa.NewBitmap() c.db[key] = b return b, 0 } // Store saves a bitmap to storage. // This is a no-op for in-memory storage because changes are stored in the cache. -func (c *Storage) Store(id uint64, db, frame string, slice int, filter uint64, b *index.Bitmap) error { +func (c *Storage) Store(id uint64, db, frame string, slice int, filter uint64, b *pilosa.Bitmap) error { return nil } diff --git a/index/storage/mem/mem_test.go b/storage/mem/mem_test.go similarity index 92% rename from index/storage/mem/mem_test.go rename to storage/mem/mem_test.go index c524644dc..1c88e0577 100644 --- a/index/storage/mem/mem_test.go +++ b/storage/mem/mem_test.go @@ -3,7 +3,7 @@ package mem_test import ( "testing" - "github.com/umbel/pilosa/index/storage/mem" + "github.com/umbel/pilosa/storage/mem" ) // Ensure a bitmap can be retrieved from storage. diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 000000000..8efbe5c3c --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,7 @@ +package storage + +import ( + _ "github.com/umbel/pilosa/storage/cassandra" + _ "github.com/umbel/pilosa/storage/leveldb" + _ "github.com/umbel/pilosa/storage/mem" +) diff --git a/index/timeframe.go b/timeframe.go similarity index 99% rename from index/timeframe.go rename to timeframe.go index 6af053acf..182adc272 100644 --- a/index/timeframe.go +++ b/timeframe.go @@ -1,4 +1,4 @@ -package index +package pilosa import ( "time" diff --git a/index/timeframe_test.go b/timeframe_test.go similarity index 85% rename from index/timeframe_test.go rename to timeframe_test.go index 51d8112cb..f31fcc32e 100644 --- a/index/timeframe_test.go +++ b/timeframe_test.go @@ -1,14 +1,14 @@ -package index_test +package pilosa_test import ( "testing" "time" - "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa" ) func TestGetRange_1h_0(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-08-11 14:00"), MustParseTime("2014-08-11 16:00"), uint64(1), @@ -18,7 +18,7 @@ func TestGetRange_1h_0(t *testing.T) { } func TestGetRange_1h_1(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 10:03"), MustParseTime("2014-01-02 11:03"), uint64(1), @@ -28,7 +28,7 @@ func TestGetRange_1h_1(t *testing.T) { } func TestGetRange_2h(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 10:03"), MustParseTime("2014-01-02 12:03"), uint64(1), @@ -38,7 +38,7 @@ func TestGetRange_2h(t *testing.T) { } func TestGetRange_24h(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 12:03"), MustParseTime("2014-01-03 12:03"), uint64(1), @@ -48,7 +48,7 @@ func TestGetRange_24h(t *testing.T) { } func TestGetRange_1d(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 00:00"), MustParseTime("2014-01-03 00:00"), uint64(1), @@ -58,7 +58,7 @@ func TestGetRange_1d(t *testing.T) { } func TestGetRange_1d1h(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 00:00"), MustParseTime("2014-01-03 01:00"), uint64(1), @@ -68,7 +68,7 @@ func TestGetRange_1d1h(t *testing.T) { } func TestGetRange_1h1d(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 23:00"), MustParseTime("2014-01-04 00:00"), uint64(1), @@ -78,7 +78,7 @@ func TestGetRange_1h1d(t *testing.T) { } func TestGetRange_1h1d1h(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-02 23:00"), MustParseTime("2014-01-04 01:00"), uint64(1), @@ -88,7 +88,7 @@ func TestGetRange_1h1d1h(t *testing.T) { } func TestGetRange_1y(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-01 00:00"), MustParseTime("2015-01-01 00:00"), uint64(1), @@ -98,7 +98,7 @@ func TestGetRange_1y(t *testing.T) { } func TestGetRange_1h1d1m(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-30 23:00"), MustParseTime("2014-03-01 00:00"), uint64(1), @@ -108,7 +108,7 @@ func TestGetRange_1h1d1m(t *testing.T) { } func TestGetRange_1h1d1m1d1h(t *testing.T) { - if m := index.GetRange( + if m := pilosa.GetRange( MustParseTime("2014-01-30 23:00"), MustParseTime("2014-03-02 01:00"), uint64(1), @@ -118,7 +118,7 @@ func TestGetRange_1h1d1m1d1h(t *testing.T) { } func TestGetTimeIds(t *testing.T) { - _ = index.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), index.YMD) + _ = pilosa.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), pilosa.YMD) } // DefaultTimeLayout is the time layout used by the tests. diff --git a/util/constants.go b/util/constants.go index c7eea49bf..d0a83db06 100644 --- a/util/constants.go +++ b/util/constants.go @@ -1,31 +1,3 @@ package util -import ( - "bytes" - "encoding/binary" -) - const TimeOut = 30 - -func Int64ToByte(data int64) []byte { - buf := new(bytes.Buffer) - binary.Write(buf, binary.BigEndian, data) - return buf.Bytes() -} -func Uint64ToByte(data uint64) []byte { - buf := new(bytes.Buffer) - binary.Write(buf, binary.BigEndian, data) - return buf.Bytes() -} -func ByteToUint64(data []byte) uint64 { - var value uint64 - buf := bytes.NewReader(data) - binary.Read(buf, binary.BigEndian, &value) - return value -} -func ByteToInt64(data []byte) int64 { - var value int64 - buf := bytes.NewReader(data) - binary.Read(buf, binary.BigEndian, &value) - return value -} diff --git a/util/id.go b/util/id.go index fb3fc1deb..f297154a0 100644 --- a/util/id.go +++ b/util/id.go @@ -30,9 +30,6 @@ func init() { type SUUID uint64 -func leftPad(s string, padStr string, pLen int) string { - return strings.Repeat(padStr, pLen) + s -} func Id() SUUID { millis := uint64(time.Now().UTC().UnixNano()) id := millis << (64 - 41) @@ -51,7 +48,7 @@ func Hex_to_SUUID(str string) SUUID { l := len(str) var m string if l < 16 { - m = leftPad(str, "0", 16-l) + m = strings.Repeat("0", 16-l) + str } else { m = str } @@ -130,23 +127,3 @@ func ParseGUID(input string) (GUID, error) { } return u, nil } - -func In(val int, list []int) bool { - for _, v := range list { - if val == v { - return true - } - } - return false -} - -func Difference(a, b []int) []int { - results := make([]int, 0, len(a)) - for _, v := range a { - if !In(v, b) { - results = append(results, v) - } - - } - return results -}