slice based popcount

index/bitmap.go

@@ -116,6 +116,11 @@ func BlockArray_copy(a *BlockArray) BlockArray {
 	}
 	return o
 }
+
+func BlockArray_andcount(a *BlockArray, b *BlockArray) uint64 {
+	return popcntAndSliceAsm(a.Block, b.Block)
+}
+
 func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray {
 	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
@@ -172,6 +177,31 @@ func Clone(a_bm IBitmap) IBitmap {
 	return output
 }
 
+func IntersectionCount(a_bm IBitmap, b_bm IBitmap) uint64 {
+	var a = a_bm.Min()
+	var b = b_bm.Min()
+	defer a.Close()
+	defer b.Close()
+	results := uint64(0)
+
+	for {
+		if b.Limit() || a.Limit() {
+			break
+		} else if a.Item().Key < b.Item().Key {
+			a = a.Next()
+		} else if a.Item().Key > b.Item().Key {
+			b = b.Next()
+		} else if a.Item().Key == b.Item().Key {
+			var a_node = a.Item()
+			var b_node = b.Item().Value
+			results += BlockArray_andcount(&a_node.Value, &b_node)
+			a = a.Next()
+			b = b.Next()
+		}
+	}
+	return results
+}
+
 func Intersection(a_bm IBitmap, b_bm IBitmap) IBitmap {
 	var a = a_bm.Min()
 	var b = b_bm.Min()
@@ -429,7 +459,7 @@ func (b *Bitmap) ToBytes() []byte {
 	enc.Encode(b.nodes.Len())
 	for i := b.nodes.Min(); !i.Limit(); i = i.Next() {
 		obj := i.Item().(*Chunk)
-		err := enc.Encode(&obj)
+		err := enc.Encode(obj)
 		if err != nil {
 			log.Println(err)
 		}
@@ -447,7 +477,7 @@ func (self *Bitmap) FromBytes(raw []byte) {
 	for i := 0; i < size; i++ {
 		//chunk := &Chunk{0, BlockArray{make([]uint64, 32, 32)}}
 		var chunk Chunk
-		dec.Decode(chunk)
+		dec.Decode(&chunk)
 		self.AddChunk(&chunk)
 	}
 	self.SetCount(BitCount(self))

index/bitmap_test.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -86,4 +87,31 @@ func TestBitmaps(t *testing.T) {
 		BitCount(all)
 		So(start, ShouldHappenWithin, time.Duration(1)*time.Millisecond, time.Now())
 	})
+
+	Convey("Compressed ", t, func() {
+		all := CreateRBBitmap()
+		for i := uint64(0); i < uint64(4096); i++ {
+			SetBit(all, i)
+		}
+		cs := all.ToCompressString()
+		fmt.Println(cs)
+		bm := CreateRBBitmap()
+		bm.FromCompressString(cs)
+		So(BitCount(all), ShouldEqual, BitCount(bm))
+	})
+
+	Convey("AndCount ", t, func() {
+		a := CreateRBBitmap()
+		for i := uint64(0); i < uint64(4096); i++ {
+			SetBit(a, i)
+		}
+		b := CreateRBBitmap()
+		for i := uint64(0); i < uint64(8192); i++ {
+			SetBit(b, i)
+		}
+		c1 := IntersectionCount(a, b)
+		c := Intersection(a, b)
+		So(c1, ShouldEqual, BitCount(c))
+	})
+
 }

index/brand.go

@@ -240,10 +240,10 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 		if counter > n {
 			break
 		}
-		bm := Intersection(src_bitmap, pair.bitmap)
-		bc := BitCount(bm)
+		bc := IntersectionCount(src_bitmap, pair.bitmap)
+		//bc := BitCount(bm)
 		if bc > 0 {
-			results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category})
+			results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category})
 			counter = counter + 1
 		}
 		x = i
@@ -286,15 +286,15 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 
 		}
 
-		abitmap := Intersection(src_bitmap, o.bitmap)
-		bc := BitCount(abitmap)
+		bc := IntersectionCount(src_bitmap, o.bitmap)
+		//bc := BitCount(abitmap)
 
 		if bc > current_threshold {
 			if results[end-1].Count > bc {
-				results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
+				results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category}
 				current_threshold = bc
 			} else {
-				results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
+				results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category}
 				sort.Sort(results)
 				o = results[end]
 				current_threshold = o.Count

index/fragment_container_test.go

@@ -154,16 +154,15 @@ func TestFragment(t *testing.T) {
 		res, _ := dummy.Clear(general)
 		So(res, ShouldEqual, true)
 	})
-
 	Convey("store ", t, func() {
 		b := uint64(1029)
-		compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA="
+		//	compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA="
 		//		compressed := "H4sIAAAJbogA/2JmYWBS+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGBweN/E+M/RgdGRoUf/6EAk/GbGUQy4AUAAAAA//8BAAD//2vjG9ezAAAA"
-
+		compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA=="
 		dummy.LoadBitmap(brand, b, compressed, 0)
 		bh1, _ := dummy.Get(brand, b)
 		before, _ := dummy.Count(brand, bh1)
-		So(15228, ShouldEqual, before)
+		So(4096, ShouldEqual, before)
 	})
 
 }

index/timeframe_test.go

@@ -34,7 +34,7 @@ func TestTimeFrame(t *testing.T) {
 		getTime(uint64(1), "2014-08-11 14:00")
 		spew.Dump(m)
 
-		So(len(m), ShouldEqual, 1)
+		So(len(m), ShouldEqual, 2)
 	})
 	if true {
 		return