added initial brand fragment support

index/brand.go

@@ -0,0 +1,156 @@
+package index
+
+import "sort"
+
+type Pair struct {
+	Key, Count uint64
+}
+type Rank struct {
+	*Pair
+	bitmap IBitmap
+}
+
+type RankList []*Rank
+
+func (p RankList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p RankList) Len() int           { return len(p) }
+func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
+
+type Brand struct {
+	bitmap_cache map[uint64]*Rank
+	db           string
+	slice        int
+	storage      Storage
+	rankings     RankList
+}
+
+func NewBrand(db string, slice int, s Storage) *Brand {
+	f := new(Brand)
+	f.bitmap_cache = make(map[uint64]*Rank)
+	f.storage = s
+	f.slice = slice
+	f.db = db
+	return f
+
+}
+
+func (self *Brand) Get(bitmap_id uint64) IBitmap {
+	bm, ok := self.bitmap_cache[bitmap_id]
+	if ok {
+		return bm.bitmap
+	}
+	//I should fetch the category here..need to come up with a good source
+	b := self.storage.Fetch(bitmap_id, self.db, self.slice)
+	self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, b.Count()}, b}
+	return b
+}
+
+func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64) bool {
+	bm := self.Get(bitmap_id)
+	change, chunk, address := SetBit(bm, bit_pos)
+	if change {
+		val := chunk.Value.Block[address.BlockIndex]
+		self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
+		self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count()))
+
+		self.Rank() //need to optimize this
+	}
+	return change
+}
+
+func (self *Brand) Rank() {
+	var list RankList
+	for k, item := range self.bitmap_cache {
+		if item.bitmap.Count() > 50 {
+			list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap})
+		}
+	}
+	sort.Sort(list)
+	self.rankings = list
+}
+
+func packagePairs(r RankList) []Pair {
+	res := make([]Pair, r.Len())
+	//for i := 0; i < r.Len(); i++ {
+	for i, v := range r {
+		res[i] = Pair{v.Key, v.Count}
+	}
+	return res
+}
+
+func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair {
+	breakout := 500
+	var (
+		o       *Rank
+		results RankList
+	)
+	counter := 0
+	x := 0
+
+	//needCat := category.Size() > 0
+	for i, pair := range self.rankings {
+		/*
+			        if needCat {
+					    if !category.Contains(pair.category) {
+					        continue
+					    }
+					}
+		*/
+		if counter > n {
+			break
+		}
+		bm := Intersection(src_bitmap, pair.bitmap)
+		bc := BitCount(bm)
+		if bc > 0 {
+			results = append(results, &Rank{&Pair{pair.Key, bc}, bm})
+			counter = counter + 1
+		}
+		x = i
+	}
+
+	sort.Sort(results)
+	if counter < n {
+		return packagePairs(results)
+	}
+	end := len(results) - 1
+	o = results[end]
+	current_threshold := o.Count
+	if current_threshold <= 10 {
+		return packagePairs(results)
+	}
+	results = append(results, o)
+
+	for i := x; i < len(self.rankings); i++ {
+		counter = counter + 1
+		o = self.rankings[i]
+		/*
+		   if needCat {
+		       if !category.Contains(o.category) {
+		           continue
+		       }
+		   } else
+		*/
+		if i > breakout {
+			break
+		}
+
+		if o.Count < current_threshold { //done
+			break
+
+		}
+
+		abitmap := Intersection(src_bitmap, o.bitmap)
+		bc := BitCount(abitmap)
+
+		if bc > current_threshold {
+			if results[end].Count == current_threshold {
+				results[end] = &Rank{&Pair{o.Key, bc}, abitmap}
+			} else {
+				results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap}
+				sort.Sort(results)
+			}
+			current_threshold = bc
+		}
+	}
+	return packagePairs(results)
+}

index/commands.go

@@ -7,16 +7,6 @@ import (
 	"time"
 )
 
-type Rank struct {
-	Key, Count uint64
-}
-
-type RankList []Rank
-
-func (p RankList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p RankList) Len() int           { return len(p) }
-func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
-
 type Result struct {
 	answer    Calculation
 	exec_time time.Duration
@@ -188,3 +178,16 @@ func (self *CmdGetList) Execute(f *Fragment) Calculation {
 
 	return ret
 }
+
+type CmdTopN struct {
+	*Responder
+	bitmap BitmapHandle
+	n      int
+}
+
+func NewTopN(b BitmapHandle, n int) *CmdTopN {
+	return &CmdTopN{NewResponder("TopN"), b, n}
+}
+func (self *CmdTopN) Execute(f *Fragment) Calculation {
+	return f.TopN(self.bitmap, self.n)
+}

index/fragment_container.go

@@ -2,6 +2,7 @@ package index
 
 import (
 	"errors"
+	"log"
 	. "pilosa/util"
 	"time"
 
@@ -41,6 +42,7 @@ func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (Bitm
 	}
 	return 0, errors.New("Invalid Bitmap Handle")
 }
+
 func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
 	if fragment, found := self.GetFragment(frag_id); found {
 		request := NewUnion(bh)
@@ -59,6 +61,15 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl
 	return 0, errors.New("Invalid Bitmap Handle")
 }
 
+func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int) ([]Pair, error) {
+	if fragment, found := self.GetFragment(frag_id); found {
+		request := NewTopN(bh, n)
+		fragment.requestChan <- request
+		return request.Response().answer.([]Pair), nil
+	}
+	return nil, nil
+}
+
 func (self *FragmentContainer) GetList(frag_id SUUID, bitmap_id []uint64) ([]BitmapHandle, error) {
 	if fragment, found := self.GetFragment(frag_id); found {
 		request := NewGetList(bitmap_id)
@@ -113,6 +124,7 @@ func (self *FragmentContainer) AddFragment(frame string, db string, slice int, i
 type Pilosa interface {
 	Get(id uint64) IBitmap
 	SetBit(id uint64, bit_pos uint64) bool
+	TopN(b IBitmap, n int) []Pair
 }
 
 type Fragment struct {
@@ -127,11 +139,21 @@ type Fragment struct {
 }
 
 func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
+	var impl Pilosa
+	switch frame {
+	default:
+		log.Println("General")
+		impl = NewGeneral(db, slice, NewMemoryStorage())
+	case "Brand":
+		log.Println("Brand")
+		impl = NewBrand(db, slice, NewMemoryStorage())
+	}
+
 	f := new(Fragment)
 	f.requestChan = make(chan Command, 64)
 	f.fragment_id = frag_id
 	f.cache = lru.New(10000)
-	f.impl = NewGeneral(db, slice, NewMemoryStorage())
+	f.impl = impl //NewGeneral(db, slice, NewMemoryStorage())
 	f.slice = slice
 	return f
 }
@@ -141,6 +163,15 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) {
 	return bm.(IBitmap), ok
 }
 
+func (self *Fragment) TopN(bitmap BitmapHandle, n int) []Pair {
+
+	bm, ok := self.cache.Get(bitmap)
+	if ok {
+		return self.impl.TopN(bm.(*Bitmap), n)
+	}
+	return nil
+}
+
 func (self *Fragment) NewHandle(bitmap_id uint64) BitmapHandle {
 	bm := self.impl.Get(bitmap_id)
 	return self.AllocHandle(bm)

index/fragment_container_test.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"log"
 	"testing"
 
 	//	"io/ioutil"
@@ -9,12 +10,14 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
-func TestServer(t *testing.T) {
+func TestFragment(t *testing.T) {
 
 	//id := util.Id()
 	id := util.Hex_to_SUUID("1")
+	id2 := util.Hex_to_SUUID("2")
 	dummy := NewFragmentContainer()
 	dummy.AddFragment("general", "25", 0, id)
+	dummy.AddFragment("Brand", "25", 0, id2)
 
 	Convey("Get ", t, func() {
 		bh, _ := dummy.Get(id, 1234)
@@ -75,4 +78,30 @@ func TestServer(t *testing.T) {
 		num, _ := dummy.Count(id, result)
 		So(num, ShouldNotEqual, 2)
 	})
+
+	Convey("Brand SetBit", t, func() {
+		bi1 := uint64(1231)
+		bi2 := uint64(1232)
+		bi3 := uint64(1233)
+		bi4 := uint64(1234)
+		for x := uint64(0); x < 1000; x++ {
+			if x < 100 {
+				dummy.SetBit(id2, bi1, x)
+				dummy.SetBit(id2, bi4, x)
+			}
+			if x < 500 {
+				dummy.SetBit(id2, bi2, x)
+			}
+			if x%3 == 0 {
+				dummy.SetBit(id2, bi3, x)
+			}
+			if x > 700 {
+				dummy.SetBit(id2, bi4, x)
+			}
+		}
+		bh1, _ := dummy.Get(id2, bi1)
+		log.Println(dummy.TopN(id2, bh1, 4))
+		//		dummy.Rank()
+		So(1, ShouldEqual, 1)
+	})
 }

index/general.go

@@ -39,3 +39,7 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool {
 	}
 	return change
 }
+func (self *General) TopN(b IBitmap, n int) []Pair {
+
+	return nil
+}