scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-12 23:51:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into executor

Browse source
This commit is contained in:
travisturner 2014-01-03 17:18:00 -06:00
commit cc1d292c7f
5 changed files with 204 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
hold/hold.go Normal file
View file

@ -0,0 +1,68 @@
package hold
import "github.com/nu7hatch/gouuid"
type holdchan chan interface{}
type gethold struct {
id *uuid.UUID
reply chan holdchan
}
type delhold struct {
id *uuid.UUID
}
type Holder struct {
data map[uuid.UUID]holdchan
getchan chan gethold
delchan chan delhold
}
var Hold Holder
func (self *Holder) DelChan(id *uuid.UUID) {
req := delhold{id}
self.delchan <- req
}
func (self *Holder) GetChan(id *uuid.UUID) holdchan {
reply := make(chan holdchan)
req := gethold{id, reply}
self.getchan <- req
return <-reply
}
func (self *Holder) Get(id *uuid.UUID) interface{} {
ch := self.GetChan(id)
return <-ch
}
func (self *Holder) Set(id *uuid.UUID, value interface{}) {
ch := self.GetChan(id)
go func() {
ch <- value
self.DelChan(id)
}()
}
func (self *Holder) run() {
var greq gethold
var dreq delhold
for {
select {
case greq = <-self.getchan:
item, ok := self.data[*greq.id]
if !ok {
item = make(holdchan)
self.data[*greq.id] = item
}
greq.reply <- item
case dreq = <-self.delchan:
delete(self.data, *dreq.id)
}
}
}
func init() {
Hold = Holder{make(map[uuid.UUID]holdchan), make(chan gethold), make(chan delhold)}
go Hold.run()
}

27
hold/hold_test.go Normal file
View file

@ -0,0 +1,27 @@
package hold
import (
"testing"
"time"
"github.com/nu7hatch/gouuid"
. "github.com/smartystreets/goconvey/convey"
)
func TestHoldChan(t *testing.T) {
Convey("set then get", t, func() {
id, _ := uuid.NewV4()
Hold.Set(id, "derp")
derp := Hold.Get(id)
So(derp, ShouldEqual, "derp")
})
Convey("get then set", t, func() {
id, _ := uuid.NewV4()
go func() {
time.Sleep(time.Second / 10)
Hold.Set(id, "derpsy")
}()
derp := Hold.Get(id)
So(derp, ShouldEqual, "derpsy")
})
}

63
index/brand.go
View file

@ -17,19 +17,29 @@ func (p RankList) Len() int { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
type Brand struct {
bitmap_cache map[uint64]*Rank
db string
slice int
storage Storage
rankings RankList
bitmap_cache map[uint64]*Rank
db string
slice int
storage Storage
rankings RankList
rank_counter int
threshold_value uint64
threshold_length int
threshold_idx int
skip int
}
func NewBrand(db string, slice int, s Storage) *Brand {
func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
f := new(Brand)
f.bitmap_cache = make(map[uint64]*Rank)
f.storage = s
f.slice = slice
f.db = db
f.rank_counter = 0
f.threshold_value = 0
f.threshold_length = threshold_len
f.threshold_idx = threshold
f.skip = skipp
return f
}
@ -41,10 +51,31 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap {
}
//I should fetch the category here..need to come up with a good source
b := self.storage.Fetch(bitmap_id, self.db, self.slice)
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, b.Count()}, b}
//Need to figure out wether to cache or not...
self.cache_it(b, bitmap_id)
return b
}
func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64) {
if bm.Count() >= self.threshold_value {
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm}
if len(self.bitmap_cache) > self.threshold_length {
self.trim()
}
}
}
func (self *Brand) trim() {
for k, item := range self.bitmap_cache {
if item.bitmap.Count() < self.threshold_value {
delete(self.bitmap_cache, k)
}
}
}
func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := SetBit(bm, bit_pos)
@ -52,13 +83,21 @@ func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool {
val := chunk.Value.Block[address.BlockIndex]
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count()))
self.Rank() //need to optimize this
if bm.Count() >= self.threshold_value {
self.Rank() //need to optimize this
}
}
return change
}
func (self *Brand) Rank() {
if self.rank_counter <= 0 {
self.rank_counter = self.skip
} else {
self.rank_counter -= 1
return //skip
}
var list RankList
for k, item := range self.bitmap_cache {
if item.bitmap.Count() > 50 {
@ -67,6 +106,12 @@ func (self *Brand) Rank() {
}
sort.Sort(list)
self.rankings = list
if len(list) > self.threshold_idx {
item := list[self.threshold_idx]
self.threshold_value = item.bitmap.Count()
} else {
self.threshold_value = 0
}
}
func packagePairs(r RankList) []Pair {

6
index/fragment_container.go
View file

@ -22,6 +22,8 @@ type BitmapHandle uint64
func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) {
//lock
c, v := self.fragments[frag_id]
//log.Println(self.fragments)
//log.Println(c)
return c, v
}
@ -116,6 +118,7 @@ func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint6
}
func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
log.Println("ADD FRAGMENT", frame)
f := NewFragment(id, db, slice, frame)
self.fragments[id] = f
go f.ServeFragment()
@ -140,13 +143,14 @@ type Fragment struct {
func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
var impl Pilosa
log.Println("XXXXXXXXXXXXXXXXXXXXXXXXXXXX", frame)
switch frame {
default:
log.Println("General")
impl = NewGeneral(db, slice, NewMemoryStorage())
case "Brand":
log.Println("Brand")
impl = NewBrand(db, slice, NewMemoryStorage())
impl = NewBrand(db, slice, NewMemoryStorage(), 50000, 45000, 100)
}
f := new(Fragment)

84
index/fragment_container_test.go
View file

@ -13,26 +13,26 @@ import (
func TestFragment(t *testing.T) {
//id := util.Id()
id := util.Hex_to_SUUID("1")
id2 := util.Hex_to_SUUID("2")
general := util.Hex_to_SUUID("1")
brand := util.Hex_to_SUUID("2")
dummy := NewFragmentContainer()
dummy.AddFragment("general", "25", 0, id)
dummy.AddFragment("Brand", "25", 0, id2)
dummy.AddFragment("25", "general", 0, general)
dummy.AddFragment("25", "Brand", 0, brand)
Convey("Get ", t, func() {
bh, _ := dummy.Get(id, 1234)
bh, _ := dummy.Get(general, 1234)
So(bh, ShouldNotEqual, 0)
})
Convey("SetBit/Count 1 1", t, func() {
// bh, _ := dummy.Get(id, 1234)
bi1 := uint64(1234)
changed, _ := dummy.SetBit(id, bi1, 1)
changed, _ := dummy.SetBit(general, bi1, 1)
So(changed, ShouldEqual, true)
changed, _ = dummy.SetBit(id, bi1, 1)
changed, _ = dummy.SetBit(general, bi1, 1)
So(changed, ShouldEqual, false)
bh, _ := dummy.Get(id, bi1)
num, _ := dummy.Count(id, bh)
bh, _ := dummy.Get(general, bi1)
num, _ := dummy.Count(general, bh)
So(num, ShouldEqual, 1)
})
@ -40,42 +40,42 @@ func TestFragment(t *testing.T) {
bi1 := uint64(1234)
bi2 := uint64(4321)
dummy.SetBit(id, bi2, 2) //set_bit creates the bitmap
dummy.SetBit(general, bi2, 2) //set_bit creates the bitmap
bh1, _ := dummy.Get(id, bi1)
bh2, _ := dummy.Get(id, bi2)
bh1, _ := dummy.Get(general, bi1)
bh2, _ := dummy.Get(general, bi2)
handles := []BitmapHandle{bh1, bh2}
result, _ := dummy.Union(id, handles)
result, _ := dummy.Union(general, handles)
num, _ := dummy.Count(id, result)
num, _ := dummy.Count(general, result)
So(num, ShouldEqual, 2)
result, _ = dummy.Intersect(id, handles)
result, _ = dummy.Intersect(general, handles)
num, _ = dummy.Count(id, result)
num, _ = dummy.Count(general, result)
So(num, ShouldEqual, 0)
})
Convey("Bytes", t, func() {
bi1 := uint64(1234)
bh1, _ := dummy.Get(id, bi1)
before, _ := dummy.Count(id, bh1)
bh1, _ := dummy.Get(general, bi1)
before, _ := dummy.Count(general, bh1)
bytes, _ := dummy.GetBytes(id, bh1)
bh2, _ := dummy.FromBytes(id, bytes)
bytes, _ := dummy.GetBytes(general, bh1)
bh2, _ := dummy.FromBytes(general, bytes)
after, _ := dummy.Count(id, bh2)
after, _ := dummy.Count(general, bh2)
So(before, ShouldEqual, after)
})
Convey("Empty ", t, func() {
bh, _ := dummy.Empty(id)
before, _ := dummy.Count(id, bh)
bh, _ := dummy.Empty(general)
before, _ := dummy.Count(general, bh)
So(before, ShouldEqual, 0)
})
Convey("GetList ", t, func() {
bhs, _ := dummy.GetList(id, []uint64{1234, 4321, 789})
result, _ := dummy.Union(id, bhs)
num, _ := dummy.Count(id, result)
bhs, _ := dummy.GetList(general, []uint64{1234, 4321, 789})
result, _ := dummy.Union(general, bhs)
num, _ := dummy.Count(general, result)
So(num, ShouldNotEqual, 2)
})
@ -86,22 +86,38 @@ func TestFragment(t *testing.T) {
bi4 := uint64(1234)
for x := uint64(0); x < 1000; x++ {
if x < 100 {
dummy.SetBit(id2, bi1, x)
dummy.SetBit(id2, bi4, x)
dummy.SetBit(brand, bi1, x)
dummy.SetBit(brand, bi4, x)
}
if x < 500 {
dummy.SetBit(id2, bi2, x)
dummy.SetBit(brand, bi2, x)
}
if x%3 == 0 {
dummy.SetBit(id2, bi3, x)
dummy.SetBit(brand, bi3, x)
}
if x > 700 {
dummy.SetBit(id2, bi4, x)
dummy.SetBit(brand, bi4, x)
}
}
bh1, _ := dummy.Get(id2, bi1)
log.Println(dummy.TopN(id2, bh1, 4))
// dummy.Rank()
bh1, _ := dummy.Get(brand, bi1)
// dummy.Rank()
log.Println(dummy.TopN(brand, bh1, 4))
So(1, ShouldEqual, 1)
})
/*
Convey("Brand TopN", t, func() {
max_brands := uint64(5000)
for i := uint64(0); i < max_brands; i++ {
for x := uint64(0); x < i; x = x + 1 {
dummy.SetBit(brand, uint64(i), x)
}
}
bh1, _ := dummy.Get(brand, uint64(4999))
log.Println(dummy.TopN(brand, bh1, 4))
So(1, ShouldEqual, 1)
})
*/
}