scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

initial brand fragment topn support

Browse source
This commit is contained in:
Todd Gruben 2014-01-03 16:21:48 -06:00
parent b8be246029
commit 9dbd3fbc66
3 changed files with 109 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
index/brand.go
View file

@ -17,19 +17,29 @@ func (p RankList) Len() int { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
type Brand struct {
bitmap_cache map[uint64]*Rank
db string
slice int
storage Storage
rankings RankList
bitmap_cache map[uint64]*Rank
db string
slice int
storage Storage
rankings RankList
rank_counter int
threshold_value uint64
threshold_length int
threshold_idx int
skip int
}
func NewBrand(db string, slice int, s Storage) *Brand {
func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
f := new(Brand)
f.bitmap_cache = make(map[uint64]*Rank)
f.storage = s
f.slice = slice
f.db = db
f.rank_counter = 0
f.threshold_value = 0
f.threshold_length = threshold_len
f.threshold_idx = threshold
f.skip = skipp
return f
}
@ -41,10 +51,31 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap {
}
//I should fetch the category here..need to come up with a good source
b := self.storage.Fetch(bitmap_id, self.db, self.slice)
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, b.Count()}, b}
//Need to figure out wether to cache or not...
self.cache_it(b, bitmap_id)
return b
}
func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64) {
if bm.Count() >= self.threshold_value {
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm}
if len(self.bitmap_cache) > self.threshold_length {
self.trim()
}
}
}
func (self *Brand) trim() {
for k, item := range self.bitmap_cache {
if item.bitmap.Count() < self.threshold_value {
delete(self.bitmap_cache, k)
}
}
}
func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := SetBit(bm, bit_pos)
@ -52,13 +83,21 @@ func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool {
val := chunk.Value.Block[address.BlockIndex]
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count()))
self.Rank() //need to optimize this
if bm.Count() >= self.threshold_value {
self.Rank() //need to optimize this
}
}
return change
}
func (self *Brand) Rank() {
if self.rank_counter <= 0 {
self.rank_counter = self.skip
} else {
self.rank_counter -= 1
return //skip
}
var list RankList
for k, item := range self.bitmap_cache {
if item.bitmap.Count() > 50 {
@ -67,6 +106,12 @@ func (self *Brand) Rank() {
}
sort.Sort(list)
self.rankings = list
if len(list) > self.threshold_idx {
item := list[self.threshold_idx]
self.threshold_value = item.bitmap.Count()
} else {
self.threshold_value = 0
}
}
func packagePairs(r RankList) []Pair {

6
index/fragment_container.go
View file

@ -22,6 +22,8 @@ type BitmapHandle uint64
func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) {
//lock
c, v := self.fragments[frag_id]
//log.Println(self.fragments)
//log.Println(c)
return c, v
}
@ -116,6 +118,7 @@ func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint6
}
func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
log.Println("ADD FRAGMENT", frame)
f := NewFragment(id, db, slice, frame)
self.fragments[id] = f
go f.ServeFragment()
@ -140,13 +143,14 @@ type Fragment struct {
func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
var impl Pilosa
log.Println("XXXXXXXXXXXXXXXXXXXXXXXXXXXX", frame)
switch frame {
default:
log.Println("General")
impl = NewGeneral(db, slice, NewMemoryStorage())
case "Brand":
log.Println("Brand")
impl = NewBrand(db, slice, NewMemoryStorage())
impl = NewBrand(db, slice, NewMemoryStorage(), 50000, 45000, 100)
}
f := new(Fragment)

84
index/fragment_container_test.go
View file

@ -13,26 +13,26 @@ import (
func TestFragment(t *testing.T) {
//id := util.Id()
id := util.Hex_to_SUUID("1")
id2 := util.Hex_to_SUUID("2")
general := util.Hex_to_SUUID("1")
brand := util.Hex_to_SUUID("2")
dummy := NewFragmentContainer()
dummy.AddFragment("general", "25", 0, id)
dummy.AddFragment("Brand", "25", 0, id2)
dummy.AddFragment("25", "general", 0, general)
dummy.AddFragment("25", "Brand", 0, brand)
Convey("Get ", t, func() {
bh, _ := dummy.Get(id, 1234)
bh, _ := dummy.Get(general, 1234)
So(bh, ShouldNotEqual, 0)
})
Convey("SetBit/Count 1 1", t, func() {
// bh, _ := dummy.Get(id, 1234)
bi1 := uint64(1234)
changed, _ := dummy.SetBit(id, bi1, 1)
changed, _ := dummy.SetBit(general, bi1, 1)
So(changed, ShouldEqual, true)
changed, _ = dummy.SetBit(id, bi1, 1)
changed, _ = dummy.SetBit(general, bi1, 1)
So(changed, ShouldEqual, false)
bh, _ := dummy.Get(id, bi1)
num, _ := dummy.Count(id, bh)
bh, _ := dummy.Get(general, bi1)
num, _ := dummy.Count(general, bh)
So(num, ShouldEqual, 1)
})
@ -40,42 +40,42 @@ func TestFragment(t *testing.T) {
bi1 := uint64(1234)
bi2 := uint64(4321)
dummy.SetBit(id, bi2, 2) //set_bit creates the bitmap
dummy.SetBit(general, bi2, 2) //set_bit creates the bitmap
bh1, _ := dummy.Get(id, bi1)
bh2, _ := dummy.Get(id, bi2)
bh1, _ := dummy.Get(general, bi1)
bh2, _ := dummy.Get(general, bi2)
handles := []BitmapHandle{bh1, bh2}
result, _ := dummy.Union(id, handles)
result, _ := dummy.Union(general, handles)
num, _ := dummy.Count(id, result)
num, _ := dummy.Count(general, result)
So(num, ShouldEqual, 2)
result, _ = dummy.Intersect(id, handles)
result, _ = dummy.Intersect(general, handles)
num, _ = dummy.Count(id, result)
num, _ = dummy.Count(general, result)
So(num, ShouldEqual, 0)
})
Convey("Bytes", t, func() {
bi1 := uint64(1234)
bh1, _ := dummy.Get(id, bi1)
before, _ := dummy.Count(id, bh1)
bh1, _ := dummy.Get(general, bi1)
before, _ := dummy.Count(general, bh1)
bytes, _ := dummy.GetBytes(id, bh1)
bh2, _ := dummy.FromBytes(id, bytes)
bytes, _ := dummy.GetBytes(general, bh1)
bh2, _ := dummy.FromBytes(general, bytes)
after, _ := dummy.Count(id, bh2)
after, _ := dummy.Count(general, bh2)
So(before, ShouldEqual, after)
})
Convey("Empty ", t, func() {
bh, _ := dummy.Empty(id)
before, _ := dummy.Count(id, bh)
bh, _ := dummy.Empty(general)
before, _ := dummy.Count(general, bh)
So(before, ShouldEqual, 0)
})
Convey("GetList ", t, func() {
bhs, _ := dummy.GetList(id, []uint64{1234, 4321, 789})
result, _ := dummy.Union(id, bhs)
num, _ := dummy.Count(id, result)
bhs, _ := dummy.GetList(general, []uint64{1234, 4321, 789})
result, _ := dummy.Union(general, bhs)
num, _ := dummy.Count(general, result)
So(num, ShouldNotEqual, 2)
})
@ -86,22 +86,38 @@ func TestFragment(t *testing.T) {
bi4 := uint64(1234)
for x := uint64(0); x < 1000; x++ {
if x < 100 {
dummy.SetBit(id2, bi1, x)
dummy.SetBit(id2, bi4, x)
dummy.SetBit(brand, bi1, x)
dummy.SetBit(brand, bi4, x)
}
if x < 500 {
dummy.SetBit(id2, bi2, x)
dummy.SetBit(brand, bi2, x)
}
if x%3 == 0 {
dummy.SetBit(id2, bi3, x)
dummy.SetBit(brand, bi3, x)
}
if x > 700 {
dummy.SetBit(id2, bi4, x)
dummy.SetBit(brand, bi4, x)
}
}
bh1, _ := dummy.Get(id2, bi1)
log.Println(dummy.TopN(id2, bh1, 4))
// dummy.Rank()
bh1, _ := dummy.Get(brand, bi1)
// dummy.Rank()
log.Println(dummy.TopN(brand, bh1, 4))
So(1, ShouldEqual, 1)
})
/*
Convey("Brand TopN", t, func() {
max_brands := uint64(5000)
for i := uint64(0); i < max_brands; i++ {
for x := uint64(0); x < i; x = x + 1 {
dummy.SetBit(brand, uint64(i), x)
}
}
bh1, _ := dummy.Get(brand, uint64(4999))
log.Println(dummy.TopN(brand, bh1, 4))
So(1, ShouldEqual, 1)
})
*/
}