From 9dbd3fbc6648108fbff00bfe9bd5704b0dbd35ad Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 3 Jan 2014 16:21:48 -0600 Subject: [PATCH] initial brand fragment topn support --- index/brand.go | 63 ++++++++++++++++++++---- index/fragment_container.go | 6 ++- index/fragment_container_test.go | 84 +++++++++++++++++++------------- 3 files changed, 109 insertions(+), 44 deletions(-) diff --git a/index/brand.go b/index/brand.go index 1d70f81ef..58a5ad2b7 100644 --- a/index/brand.go +++ b/index/brand.go @@ -17,19 +17,29 @@ func (p RankList) Len() int { return len(p) } func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count } type Brand struct { - bitmap_cache map[uint64]*Rank - db string - slice int - storage Storage - rankings RankList + bitmap_cache map[uint64]*Rank + db string + slice int + storage Storage + rankings RankList + rank_counter int + threshold_value uint64 + threshold_length int + threshold_idx int + skip int } -func NewBrand(db string, slice int, s Storage) *Brand { +func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand { f := new(Brand) f.bitmap_cache = make(map[uint64]*Rank) f.storage = s f.slice = slice f.db = db + f.rank_counter = 0 + f.threshold_value = 0 + f.threshold_length = threshold_len + f.threshold_idx = threshold + f.skip = skipp return f } @@ -41,10 +51,31 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap { } //I should fetch the category here..need to come up with a good source b := self.storage.Fetch(bitmap_id, self.db, self.slice) - self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, b.Count()}, b} + + //Need to figure out wether to cache or not... + + self.cache_it(b, bitmap_id) + return b } +func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64) { + if bm.Count() >= self.threshold_value { + self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm} + if len(self.bitmap_cache) > self.threshold_length { + self.trim() + } + } +} +func (self *Brand) trim() { + for k, item := range self.bitmap_cache { + if item.bitmap.Count() < self.threshold_value { + delete(self.bitmap_cache, k) + } + } + +} + func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool { bm := self.Get(bitmap_id) change, chunk, address := SetBit(bm, bit_pos) @@ -52,13 +83,21 @@ func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool { val := chunk.Value.Block[address.BlockIndex] self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val)) self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count())) - - self.Rank() //need to optimize this + if bm.Count() >= self.threshold_value { + self.Rank() //need to optimize this + } } return change } func (self *Brand) Rank() { + if self.rank_counter <= 0 { + self.rank_counter = self.skip + } else { + self.rank_counter -= 1 + return //skip + } + var list RankList for k, item := range self.bitmap_cache { if item.bitmap.Count() > 50 { @@ -67,6 +106,12 @@ func (self *Brand) Rank() { } sort.Sort(list) self.rankings = list + if len(list) > self.threshold_idx { + item := list[self.threshold_idx] + self.threshold_value = item.bitmap.Count() + } else { + self.threshold_value = 0 + } } func packagePairs(r RankList) []Pair { diff --git a/index/fragment_container.go b/index/fragment_container.go index 04c1049f9..eb850a138 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -22,6 +22,8 @@ type BitmapHandle uint64 func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) { //lock c, v := self.fragments[frag_id] + //log.Println(self.fragments) + //log.Println(c) return c, v } @@ -116,6 +118,7 @@ func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint6 } func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) { + log.Println("ADD FRAGMENT", frame) f := NewFragment(id, db, slice, frame) self.fragments[id] = f go f.ServeFragment() @@ -140,13 +143,14 @@ type Fragment struct { func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment { var impl Pilosa + log.Println("XXXXXXXXXXXXXXXXXXXXXXXXXXXX", frame) switch frame { default: log.Println("General") impl = NewGeneral(db, slice, NewMemoryStorage()) case "Brand": log.Println("Brand") - impl = NewBrand(db, slice, NewMemoryStorage()) + impl = NewBrand(db, slice, NewMemoryStorage(), 50000, 45000, 100) } f := new(Fragment) diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index 17d84c14d..7b7d8a22b 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -13,26 +13,26 @@ import ( func TestFragment(t *testing.T) { //id := util.Id() - id := util.Hex_to_SUUID("1") - id2 := util.Hex_to_SUUID("2") + general := util.Hex_to_SUUID("1") + brand := util.Hex_to_SUUID("2") dummy := NewFragmentContainer() - dummy.AddFragment("general", "25", 0, id) - dummy.AddFragment("Brand", "25", 0, id2) + dummy.AddFragment("25", "general", 0, general) + dummy.AddFragment("25", "Brand", 0, brand) Convey("Get ", t, func() { - bh, _ := dummy.Get(id, 1234) + bh, _ := dummy.Get(general, 1234) So(bh, ShouldNotEqual, 0) }) Convey("SetBit/Count 1 1", t, func() { // bh, _ := dummy.Get(id, 1234) bi1 := uint64(1234) - changed, _ := dummy.SetBit(id, bi1, 1) + changed, _ := dummy.SetBit(general, bi1, 1) So(changed, ShouldEqual, true) - changed, _ = dummy.SetBit(id, bi1, 1) + changed, _ = dummy.SetBit(general, bi1, 1) So(changed, ShouldEqual, false) - bh, _ := dummy.Get(id, bi1) - num, _ := dummy.Count(id, bh) + bh, _ := dummy.Get(general, bi1) + num, _ := dummy.Count(general, bh) So(num, ShouldEqual, 1) }) @@ -40,42 +40,42 @@ func TestFragment(t *testing.T) { bi1 := uint64(1234) bi2 := uint64(4321) - dummy.SetBit(id, bi2, 2) //set_bit creates the bitmap + dummy.SetBit(general, bi2, 2) //set_bit creates the bitmap - bh1, _ := dummy.Get(id, bi1) - bh2, _ := dummy.Get(id, bi2) + bh1, _ := dummy.Get(general, bi1) + bh2, _ := dummy.Get(general, bi2) handles := []BitmapHandle{bh1, bh2} - result, _ := dummy.Union(id, handles) + result, _ := dummy.Union(general, handles) - num, _ := dummy.Count(id, result) + num, _ := dummy.Count(general, result) So(num, ShouldEqual, 2) - result, _ = dummy.Intersect(id, handles) + result, _ = dummy.Intersect(general, handles) - num, _ = dummy.Count(id, result) + num, _ = dummy.Count(general, result) So(num, ShouldEqual, 0) }) Convey("Bytes", t, func() { bi1 := uint64(1234) - bh1, _ := dummy.Get(id, bi1) - before, _ := dummy.Count(id, bh1) + bh1, _ := dummy.Get(general, bi1) + before, _ := dummy.Count(general, bh1) - bytes, _ := dummy.GetBytes(id, bh1) - bh2, _ := dummy.FromBytes(id, bytes) + bytes, _ := dummy.GetBytes(general, bh1) + bh2, _ := dummy.FromBytes(general, bytes) - after, _ := dummy.Count(id, bh2) + after, _ := dummy.Count(general, bh2) So(before, ShouldEqual, after) }) Convey("Empty ", t, func() { - bh, _ := dummy.Empty(id) - before, _ := dummy.Count(id, bh) + bh, _ := dummy.Empty(general) + before, _ := dummy.Count(general, bh) So(before, ShouldEqual, 0) }) Convey("GetList ", t, func() { - bhs, _ := dummy.GetList(id, []uint64{1234, 4321, 789}) - result, _ := dummy.Union(id, bhs) - num, _ := dummy.Count(id, result) + bhs, _ := dummy.GetList(general, []uint64{1234, 4321, 789}) + result, _ := dummy.Union(general, bhs) + num, _ := dummy.Count(general, result) So(num, ShouldNotEqual, 2) }) @@ -86,22 +86,38 @@ func TestFragment(t *testing.T) { bi4 := uint64(1234) for x := uint64(0); x < 1000; x++ { if x < 100 { - dummy.SetBit(id2, bi1, x) - dummy.SetBit(id2, bi4, x) + dummy.SetBit(brand, bi1, x) + dummy.SetBit(brand, bi4, x) } if x < 500 { - dummy.SetBit(id2, bi2, x) + dummy.SetBit(brand, bi2, x) } if x%3 == 0 { - dummy.SetBit(id2, bi3, x) + dummy.SetBit(brand, bi3, x) } if x > 700 { - dummy.SetBit(id2, bi4, x) + dummy.SetBit(brand, bi4, x) } } - bh1, _ := dummy.Get(id2, bi1) - log.Println(dummy.TopN(id2, bh1, 4)) - // dummy.Rank() + bh1, _ := dummy.Get(brand, bi1) + // dummy.Rank() + log.Println(dummy.TopN(brand, bh1, 4)) So(1, ShouldEqual, 1) }) + /* + Convey("Brand TopN", t, func() { + max_brands := uint64(5000) + for i := uint64(0); i < max_brands; i++ { + for x := uint64(0); x < i; x = x + 1 { + dummy.SetBit(brand, uint64(i), x) + } + + } + bh1, _ := dummy.Get(brand, uint64(4999)) + log.Println(dummy.TopN(brand, bh1, 4)) + + So(1, ShouldEqual, 1) + }) + */ + }