diff --git a/index/brand.go b/index/brand.go new file mode 100644 index 000000000..1d70f81ef --- /dev/null +++ b/index/brand.go @@ -0,0 +1,156 @@ +package index + +import "sort" + +type Pair struct { + Key, Count uint64 +} +type Rank struct { + *Pair + bitmap IBitmap +} + +type RankList []*Rank + +func (p RankList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p RankList) Len() int { return len(p) } +func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count } + +type Brand struct { + bitmap_cache map[uint64]*Rank + db string + slice int + storage Storage + rankings RankList +} + +func NewBrand(db string, slice int, s Storage) *Brand { + f := new(Brand) + f.bitmap_cache = make(map[uint64]*Rank) + f.storage = s + f.slice = slice + f.db = db + return f + +} + +func (self *Brand) Get(bitmap_id uint64) IBitmap { + bm, ok := self.bitmap_cache[bitmap_id] + if ok { + return bm.bitmap + } + //I should fetch the category here..need to come up with a good source + b := self.storage.Fetch(bitmap_id, self.db, self.slice) + self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, b.Count()}, b} + return b +} + +func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool { + bm := self.Get(bitmap_id) + change, chunk, address := SetBit(bm, bit_pos) + if change { + val := chunk.Value.Block[address.BlockIndex] + self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val)) + self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count())) + + self.Rank() //need to optimize this + } + return change +} + +func (self *Brand) Rank() { + var list RankList + for k, item := range self.bitmap_cache { + if item.bitmap.Count() > 50 { + list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap}) + } + } + sort.Sort(list) + self.rankings = list +} + +func packagePairs(r RankList) []Pair { + res := make([]Pair, r.Len()) + //for i := 0; i < r.Len(); i++ { + for i, v := range r { + res[i] = Pair{v.Key, v.Count} + } + return res +} + +func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair { + breakout := 500 + var ( + o *Rank + results RankList + ) + counter := 0 + x := 0 + + //needCat := category.Size() > 0 + for i, pair := range self.rankings { + /* + if needCat { + if !category.Contains(pair.category) { + continue + } + } + */ + if counter > n { + break + } + bm := Intersection(src_bitmap, pair.bitmap) + bc := BitCount(bm) + if bc > 0 { + results = append(results, &Rank{&Pair{pair.Key, bc}, bm}) + counter = counter + 1 + } + x = i + } + + sort.Sort(results) + if counter < n { + return packagePairs(results) + } + end := len(results) - 1 + o = results[end] + current_threshold := o.Count + if current_threshold <= 10 { + return packagePairs(results) + } + results = append(results, o) + + for i := x; i < len(self.rankings); i++ { + counter = counter + 1 + o = self.rankings[i] + /* + if needCat { + if !category.Contains(o.category) { + continue + } + } else + */ + if i > breakout { + break + } + + if o.Count < current_threshold { //done + break + + } + + abitmap := Intersection(src_bitmap, o.bitmap) + bc := BitCount(abitmap) + + if bc > current_threshold { + if results[end].Count == current_threshold { + results[end] = &Rank{&Pair{o.Key, bc}, abitmap} + } else { + results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap} + sort.Sort(results) + } + current_threshold = bc + } + } + return packagePairs(results) +} diff --git a/index/commands.go b/index/commands.go index 197a83eca..074f0ea57 100644 --- a/index/commands.go +++ b/index/commands.go @@ -7,16 +7,6 @@ import ( "time" ) -type Rank struct { - Key, Count uint64 -} - -type RankList []Rank - -func (p RankList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p RankList) Len() int { return len(p) } -func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count } - type Result struct { answer Calculation exec_time time.Duration @@ -188,3 +178,16 @@ func (self *CmdGetList) Execute(f *Fragment) Calculation { return ret } + +type CmdTopN struct { + *Responder + bitmap BitmapHandle + n int +} + +func NewTopN(b BitmapHandle, n int) *CmdTopN { + return &CmdTopN{NewResponder("TopN"), b, n} +} +func (self *CmdTopN) Execute(f *Fragment) Calculation { + return f.TopN(self.bitmap, self.n) +} diff --git a/index/fragment_container.go b/index/fragment_container.go index 4fda87cb4..77e83a8f4 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -2,6 +2,7 @@ package index import ( "errors" + "log" . "pilosa/util" "time" @@ -41,6 +42,7 @@ func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (Bitm } return 0, errors.New("Invalid Bitmap Handle") } + func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewUnion(bh) @@ -59,6 +61,15 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl return 0, errors.New("Invalid Bitmap Handle") } +func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int) ([]Pair, error) { + if fragment, found := self.GetFragment(frag_id); found { + request := NewTopN(bh, n) + fragment.requestChan <- request + return request.Response().answer.([]Pair), nil + } + return nil, nil +} + func (self *FragmentContainer) GetList(frag_id SUUID, bitmap_id []uint64) ([]BitmapHandle, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewGetList(bitmap_id) @@ -113,6 +124,7 @@ func (self *FragmentContainer) AddFragment(frame string, db string, slice int, i type Pilosa interface { Get(id uint64) IBitmap SetBit(id uint64, bit_pos uint64) bool + TopN(b IBitmap, n int) []Pair } type Fragment struct { @@ -127,11 +139,21 @@ type Fragment struct { } func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment { + var impl Pilosa + switch frame { + default: + log.Println("General") + impl = NewGeneral(db, slice, NewMemoryStorage()) + case "Brand": + log.Println("Brand") + impl = NewBrand(db, slice, NewMemoryStorage()) + } + f := new(Fragment) f.requestChan = make(chan Command, 64) f.fragment_id = frag_id f.cache = lru.New(10000) - f.impl = NewGeneral(db, slice, NewMemoryStorage()) + f.impl = impl //NewGeneral(db, slice, NewMemoryStorage()) f.slice = slice return f } @@ -141,6 +163,15 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) { return bm.(IBitmap), ok } +func (self *Fragment) TopN(bitmap BitmapHandle, n int) []Pair { + + bm, ok := self.cache.Get(bitmap) + if ok { + return self.impl.TopN(bm.(*Bitmap), n) + } + return nil +} + func (self *Fragment) NewHandle(bitmap_id uint64) BitmapHandle { bm := self.impl.Get(bitmap_id) return self.AllocHandle(bm) diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index 396c1ad45..17d84c14d 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -1,6 +1,7 @@ package index import ( + "log" "testing" // "io/ioutil" @@ -9,12 +10,14 @@ import ( . "github.com/smartystreets/goconvey/convey" ) -func TestServer(t *testing.T) { +func TestFragment(t *testing.T) { //id := util.Id() id := util.Hex_to_SUUID("1") + id2 := util.Hex_to_SUUID("2") dummy := NewFragmentContainer() dummy.AddFragment("general", "25", 0, id) + dummy.AddFragment("Brand", "25", 0, id2) Convey("Get ", t, func() { bh, _ := dummy.Get(id, 1234) @@ -75,4 +78,30 @@ func TestServer(t *testing.T) { num, _ := dummy.Count(id, result) So(num, ShouldNotEqual, 2) }) + + Convey("Brand SetBit", t, func() { + bi1 := uint64(1231) + bi2 := uint64(1232) + bi3 := uint64(1233) + bi4 := uint64(1234) + for x := uint64(0); x < 1000; x++ { + if x < 100 { + dummy.SetBit(id2, bi1, x) + dummy.SetBit(id2, bi4, x) + } + if x < 500 { + dummy.SetBit(id2, bi2, x) + } + if x%3 == 0 { + dummy.SetBit(id2, bi3, x) + } + if x > 700 { + dummy.SetBit(id2, bi4, x) + } + } + bh1, _ := dummy.Get(id2, bi1) + log.Println(dummy.TopN(id2, bh1, 4)) + // dummy.Rank() + So(1, ShouldEqual, 1) + }) } diff --git a/index/general.go b/index/general.go index 88621c9be..7bc642df7 100644 --- a/index/general.go +++ b/index/general.go @@ -39,3 +39,7 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool { } return change } +func (self *General) TopN(b IBitmap, n int) []Pair { + + return nil +}