diff --git a/index/bitmap.go b/index/bitmap.go index 563144c47..4a10a8654 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -418,18 +418,19 @@ func (b *Bitmap) ToBytes() []byte { return buf.Bytes() } -func (b *Bitmap) FromBytes(raw []byte) { +func (self *Bitmap) FromBytes(raw []byte) { buf := bytes.NewBuffer(raw) dec := gob.NewDecoder(buf) var size int dec.Decode(&size) - b.nodes = NewRB() + self.nodes = NewRB() for i := 0; i < size; i++ { chunk := &Chunk{} dec.Decode(&chunk) - b.AddChunk(chunk) + self.AddChunk(chunk) } + self.SetCount(BitCount(self)) } func (b *Bitmap) BuildFromBits(bits []uint64) { diff --git a/index/brand.go b/index/brand.go index 7211bcdae..6a7b9f74c 100644 --- a/index/brand.go +++ b/index/brand.go @@ -87,6 +87,7 @@ func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool { self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val)) self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count())) if bm.Count() >= self.threshold_value { + self.Rank() //need to optimize this } } @@ -125,6 +126,35 @@ func packagePairs(r RankList) []Pair { } return res } + +func (self *Brand) Stats() interface{} { + total := uint64(0) + i := uint64(0) + bit_total := uint64(0) + for _, v := range self.bitmap_cache { + total += uint64(v.bitmap.Len()) * uint64(256) + i += 1 + bit_total += v.Count + } + avg_bytes := uint64(0) + avg_bits := uint64(0) + if i > 0 { + avg_bytes = total / i + avg_bits = bit_total / i + } + + stats := map[string]interface{}{ + "total size of cache in bytes": total, + "number of bitmaps": len(self.bitmap_cache), + "avg size of bitmap in space(bytes)": avg_bytes, + "avg size of bitmap in bits": avg_bits, + "rank counter": self.rank_counter, + "threshold_value": self.threshold_value, + "threshold_length": self.threshold_length, + "threshold_idx": self.threshold_idx, + "skip": self.skip} + return stats +} func (self *Brand) Store(bitmap_id uint64, bm IBitmap) { //oldbm:=self.Get(bitmap_id) //nbm = Union(oldbm, bm) diff --git a/index/commands.go b/index/commands.go index 4a2889186..472ab8bee 100644 --- a/index/commands.go +++ b/index/commands.go @@ -218,3 +218,14 @@ func (self *CmdLoader) Execute(f *Fragment) Calculation { f.impl.Store(self.bitmap_id, nbm) return "ok" } + +type CmdStats struct { + *Responder +} + +func NewStats() *CmdStats { + return &CmdStats{NewResponder("Stats")} +} +func (self *CmdStats) Execute(f *Fragment) Calculation { + return f.impl.Stats() +} diff --git a/index/fragment_container.go b/index/fragment_container.go index a17c90082..9ff78c03f 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -4,6 +4,7 @@ import ( "encoding/gob" "errors" "log" + "pilosa/config" . "pilosa/util" "time" @@ -42,6 +43,14 @@ func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) { return c, v } +func (self *FragmentContainer) Stats(frag_id SUUID) interface{} { + if fragment, found := self.GetFragment(frag_id); found { + request := NewStats() + fragment.requestChan <- request + return request.Response().answer + } + return nil +} func (self *FragmentContainer) Empty(frag_id SUUID) (BitmapHandle, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewEmpty() @@ -154,6 +163,7 @@ type Pilosa interface { TopN(b IBitmap, n int) []Pair Clear() bool Store(bitmap_id uint64, bm IBitmap) + Stats() interface{} } type Fragment struct { @@ -168,33 +178,30 @@ type Fragment struct { } func getStorage(db string, slice int, frame string) Storage { - return NewCassStorage("10.87.110.249", "hotbox") - /* - storage_method := config.GetInt("storage") + storage_method := config.GetInt("storage") - switch storage_method { - default: - return NewMemoryStorage() - case 1: - storage_path := config.GetString("kv_base_path") - if storage_path == "" { - storage_path = "/tmp/pilosa" - } - s, _ := NewKVStorage(storage_path, slice, db) - return s - case 2: - host := config.GetString("cass_host") - if host == "" { - host = "localhost" - } - keyspace := config.GetString("cass_keyspace") - if keyspace == "" { - keyspace = "hotbox" - } - return NewCassStorage(host, keyspace) + switch storage_method { + default: + return NewMemoryStorage() + case 1: + storage_path := config.GetString("kv_base_path") + if storage_path == "" { + storage_path = "/tmp/pilosa" } - return nil - */ + s, _ := NewKVStorage(storage_path, slice, db) + return s + case 2: + host := config.GetString("cass_host") + if host == "" { + host = "localhost" + } + keyspace := config.GetString("cass_keyspace") + if keyspace == "" { + keyspace = "hotbox" + } + return NewCassStorage(host, keyspace) + } + return nil } func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {