merge fix

index/bitmap.go

@@ -27,6 +27,7 @@ var (
 func init() {
 	var dumb = COUNTERMASK
 	COUNTER_KEY = int64(dumb)
+
 }
 
 //
@@ -80,19 +81,20 @@ func popcount(x uint64) (n uint64) {
 */
 
 type BlockArray struct {
-	Block [32]uint64
+	Block []uint64
 }
 
 func (s *BlockArray) bitcount() uint64 {
-	var sum uint64
+	/*var sum uint64
 	for _, b := range (*s).Block {
 		sum += popcount(b)
 	}
 	return sum
-	//	return popcntSlice(s.Block)
+	*/
+	return popcntSlice(s.Block)
 }
 func BlockArray_union(a *BlockArray, b *BlockArray) BlockArray {
-	var o = BlockArray{}
+	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
 		o.Block[i] = a.Block[i] | b.Block[i]
 	}
@@ -100,7 +102,7 @@ func BlockArray_union(a *BlockArray, b *BlockArray) BlockArray {
 }
 
 func BlockArray_invert(a *BlockArray) BlockArray {
-	var o = BlockArray{}
+	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
 		o.Block[i] = ^a.Block[i]
 	}
@@ -108,14 +110,19 @@ func BlockArray_invert(a *BlockArray) BlockArray {
 }
 
 func BlockArray_copy(a *BlockArray) BlockArray {
-	var o = BlockArray{}
+	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
 		o.Block[i] = a.Block[i]
 	}
 	return o
 }
+
+func BlockArray_andcount(a *BlockArray, b *BlockArray) uint64 {
+	return popcntAndSliceAsm(a.Block, b.Block)
+}
+
 func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray {
-	var o = BlockArray{}
+	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
 		o.Block[i] = a.Block[i] & b.Block[i]
 	}
@@ -123,7 +130,7 @@ func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray {
 }
 
 func BlockArray_difference(a *BlockArray, b *BlockArray) BlockArray {
-	var o = BlockArray{}
+	var o = BlockArray{make([]uint64, 32, 32)}
 	for i, _ := range a.Block {
 		o.Block[i] = a.Block[i] &^ b.Block[i]
 	}
@@ -170,6 +177,31 @@ func Clone(a_bm IBitmap) IBitmap {
 	return output
 }
 
+func IntersectionCount(a_bm IBitmap, b_bm IBitmap) uint64 {
+	var a = a_bm.Min()
+	var b = b_bm.Min()
+	defer a.Close()
+	defer b.Close()
+	results := uint64(0)
+
+	for {
+		if b.Limit() || a.Limit() {
+			break
+		} else if a.Item().Key < b.Item().Key {
+			a = a.Next()
+		} else if a.Item().Key > b.Item().Key {
+			b = b.Next()
+		} else if a.Item().Key == b.Item().Key {
+			var a_node = a.Item()
+			var b_node = b.Item().Value
+			results += BlockArray_andcount(&a_node.Value, &b_node)
+			a = a.Next()
+			b = b.Next()
+		}
+	}
+	return results
+}
+
 func Intersection(a_bm IBitmap, b_bm IBitmap) IBitmap {
 	var a = a_bm.Min()
 	var b = b_bm.Min()
@@ -357,7 +389,7 @@ func (r *RBNodeIterator) Item() *Chunk {
 	return nil
 }
 func GetChunk(bm IBitmap, ChunkKey uint64) *Chunk {
-	look := &Chunk{ChunkKey, BlockArray{}}
+	look := &Chunk{ChunkKey, BlockArray{make([]uint64, 32, 32)}}
 	return bm.Get(look)
 }
 
@@ -427,7 +459,10 @@ func (b *Bitmap) ToBytes() []byte {
 	enc.Encode(b.nodes.Len())
 	for i := b.nodes.Min(); !i.Limit(); i = i.Next() {
 		obj := i.Item().(*Chunk)
-		enc.Encode(obj)
+		err := enc.Encode(obj)
+		if err != nil {
+			log.Println(err)
+		}
 	}
 	return buf.Bytes()
 }
@@ -440,7 +475,6 @@ func (self *Bitmap) FromBytes(raw []byte) {
 	dec.Decode(&size)
 	self.nodes = NewRB()
 	for i := 0; i < size; i++ {
-		//chunk := &Chunk{}
 		var chunk Chunk
 		dec.Decode(&chunk)
 		self.AddChunk(&chunk)
@@ -509,7 +543,7 @@ func SetBit(b IBitmap, position uint64) (bool, *Chunk, Address) {
 	item := GetChunk(b, address.ChunkKey)
 	var node *Chunk
 	if item == nil {
-		node = &Chunk{address.ChunkKey, BlockArray{}}
+		node = &Chunk{address.ChunkKey, BlockArray{make([]uint64, 32, 32)}}
 		b.AddChunk(node)
 	} else {
 		node = item

index/bitmap_test.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -86,4 +87,31 @@ func TestBitmaps(t *testing.T) {
 		BitCount(all)
 		So(start, ShouldHappenWithin, time.Duration(1)*time.Millisecond, time.Now())
 	})
+
+	Convey("Compressed ", t, func() {
+		all := CreateRBBitmap()
+		for i := uint64(0); i < uint64(4096); i++ {
+			SetBit(all, i)
+		}
+		cs := all.ToCompressString()
+		fmt.Println(cs)
+		bm := CreateRBBitmap()
+		bm.FromCompressString(cs)
+		So(BitCount(all), ShouldEqual, BitCount(bm))
+	})
+
+	Convey("AndCount ", t, func() {
+		a := CreateRBBitmap()
+		for i := uint64(0); i < uint64(4096); i++ {
+			SetBit(a, i)
+		}
+		b := CreateRBBitmap()
+		for i := uint64(0); i < uint64(8192); i++ {
+			SetBit(b, i)
+		}
+		c1 := IntersectionCount(a, b)
+		c := Intersection(a, b)
+		So(c1, ShouldEqual, BitCount(c))
+	})
+
 }

index/brand.go

@@ -240,10 +240,10 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 		if counter > n {
 			break
 		}
-		bm := Intersection(src_bitmap, pair.bitmap)
-		bc := BitCount(bm)
+		bc := IntersectionCount(src_bitmap, pair.bitmap)
+		//bc := BitCount(bm)
 		if bc > 0 {
-			results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category})
+			results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category})
 			counter = counter + 1
 		}
 		x = i
@@ -286,15 +286,15 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 
 		}
 
-		abitmap := Intersection(src_bitmap, o.bitmap)
-		bc := BitCount(abitmap)
+		bc := IntersectionCount(src_bitmap, o.bitmap)
+		//bc := BitCount(abitmap)
 
 		if bc > current_threshold {
 			if results[end-1].Count > bc {
-				results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
+				results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category}
 				current_threshold = bc
 			} else {
-				results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
+				results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category}
 				sort.Sort(results)
 				o = results[end]
 				current_threshold = o.Count

index/fragment_container_test.go

@@ -154,16 +154,15 @@ func TestFragment(t *testing.T) {
 		res, _ := dummy.Clear(general)
 		So(res, ShouldEqual, true)
 	})
-
 	Convey("store ", t, func() {
 		b := uint64(1029)
-		compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA="
+		//	compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA="
 		//		compressed := "H4sIAAAJbogA/2JmYWBS+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGBweN/E+M/RgdGRoUf/6EAk/GbGUQy4AUAAAAA//8BAAD//2vjG9ezAAAA"
-
+		compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA=="
 		dummy.LoadBitmap(brand, b, compressed, 0)
 		bh1, _ := dummy.Get(brand, b)
 		before, _ := dummy.Count(brand, bh1)
-		So(15228, ShouldEqual, before)
+		So(4096, ShouldEqual, before)
 	})
 
 }

index/storage_leveldb.go

@@ -86,7 +86,7 @@ func (self *LevelDBStorage) Fetch(bitmap_id uint64, db string, frame string, sli
 		block, filter = decodeValue(iter.Value())
 		if chunk_key != COUNTERMASK {
 			if chunk_key != last_key {
-				chunk = &Chunk{chunk_key, BlockArray{}}
+				chunk = &Chunk{chunk_key, BlockArray{make([]uint64, 32, 32)}}
 				bitmap.AddChunk(chunk)
 			}
 			chunk.Value.Block[block_index] = block