diff --git a/index/bitmap.go b/index/bitmap.go index 5aa9d2c43..eb0d4db41 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -27,6 +27,7 @@ var ( func init() { var dumb = COUNTERMASK COUNTER_KEY = int64(dumb) + } // @@ -80,19 +81,20 @@ func popcount(x uint64) (n uint64) { */ type BlockArray struct { - Block [32]uint64 + Block []uint64 } func (s *BlockArray) bitcount() uint64 { - var sum uint64 + /*var sum uint64 for _, b := range (*s).Block { sum += popcount(b) } return sum - // return popcntSlice(s.Block) + */ + return popcntSlice(s.Block) } func BlockArray_union(a *BlockArray, b *BlockArray) BlockArray { - var o = BlockArray{} + var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { o.Block[i] = a.Block[i] | b.Block[i] } @@ -100,7 +102,7 @@ func BlockArray_union(a *BlockArray, b *BlockArray) BlockArray { } func BlockArray_invert(a *BlockArray) BlockArray { - var o = BlockArray{} + var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { o.Block[i] = ^a.Block[i] } @@ -108,14 +110,19 @@ func BlockArray_invert(a *BlockArray) BlockArray { } func BlockArray_copy(a *BlockArray) BlockArray { - var o = BlockArray{} + var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { o.Block[i] = a.Block[i] } return o } + +func BlockArray_andcount(a *BlockArray, b *BlockArray) uint64 { + return popcntAndSliceAsm(a.Block, b.Block) +} + func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray { - var o = BlockArray{} + var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { o.Block[i] = a.Block[i] & b.Block[i] } @@ -123,7 +130,7 @@ func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray { } func BlockArray_difference(a *BlockArray, b *BlockArray) BlockArray { - var o = BlockArray{} + var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { o.Block[i] = a.Block[i] &^ b.Block[i] } @@ -170,6 +177,31 @@ func Clone(a_bm IBitmap) IBitmap { return output } +func IntersectionCount(a_bm IBitmap, b_bm IBitmap) uint64 { + var a = a_bm.Min() + var b = b_bm.Min() + defer a.Close() + defer b.Close() + results := uint64(0) + + for { + if b.Limit() || a.Limit() { + break + } else if a.Item().Key < b.Item().Key { + a = a.Next() + } else if a.Item().Key > b.Item().Key { + b = b.Next() + } else if a.Item().Key == b.Item().Key { + var a_node = a.Item() + var b_node = b.Item().Value + results += BlockArray_andcount(&a_node.Value, &b_node) + a = a.Next() + b = b.Next() + } + } + return results +} + func Intersection(a_bm IBitmap, b_bm IBitmap) IBitmap { var a = a_bm.Min() var b = b_bm.Min() @@ -357,7 +389,7 @@ func (r *RBNodeIterator) Item() *Chunk { return nil } func GetChunk(bm IBitmap, ChunkKey uint64) *Chunk { - look := &Chunk{ChunkKey, BlockArray{}} + look := &Chunk{ChunkKey, BlockArray{make([]uint64, 32, 32)}} return bm.Get(look) } @@ -427,7 +459,10 @@ func (b *Bitmap) ToBytes() []byte { enc.Encode(b.nodes.Len()) for i := b.nodes.Min(); !i.Limit(); i = i.Next() { obj := i.Item().(*Chunk) - enc.Encode(obj) + err := enc.Encode(obj) + if err != nil { + log.Println(err) + } } return buf.Bytes() } @@ -440,7 +475,6 @@ func (self *Bitmap) FromBytes(raw []byte) { dec.Decode(&size) self.nodes = NewRB() for i := 0; i < size; i++ { - //chunk := &Chunk{} var chunk Chunk dec.Decode(&chunk) self.AddChunk(&chunk) @@ -509,7 +543,7 @@ func SetBit(b IBitmap, position uint64) (bool, *Chunk, Address) { item := GetChunk(b, address.ChunkKey) var node *Chunk if item == nil { - node = &Chunk{address.ChunkKey, BlockArray{}} + node = &Chunk{address.ChunkKey, BlockArray{make([]uint64, 32, 32)}} b.AddChunk(node) } else { node = item diff --git a/index/bitmap_test.go b/index/bitmap_test.go index 2915ea21d..cc2a41db2 100644 --- a/index/bitmap_test.go +++ b/index/bitmap_test.go @@ -1,6 +1,7 @@ package index import ( + "fmt" "testing" "time" @@ -86,4 +87,31 @@ func TestBitmaps(t *testing.T) { BitCount(all) So(start, ShouldHappenWithin, time.Duration(1)*time.Millisecond, time.Now()) }) + + Convey("Compressed ", t, func() { + all := CreateRBBitmap() + for i := uint64(0); i < uint64(4096); i++ { + SetBit(all, i) + } + cs := all.ToCompressString() + fmt.Println(cs) + bm := CreateRBBitmap() + bm.FromCompressString(cs) + So(BitCount(all), ShouldEqual, BitCount(bm)) + }) + + Convey("AndCount ", t, func() { + a := CreateRBBitmap() + for i := uint64(0); i < uint64(4096); i++ { + SetBit(a, i) + } + b := CreateRBBitmap() + for i := uint64(0); i < uint64(8192); i++ { + SetBit(b, i) + } + c1 := IntersectionCount(a, b) + c := Intersection(a, b) + So(c1, ShouldEqual, BitCount(c)) + }) + } diff --git a/index/brand.go b/index/brand.go index ba9954fde..4474f7489 100644 --- a/index/brand.go +++ b/index/brand.go @@ -240,10 +240,10 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { if counter > n { break } - bm := Intersection(src_bitmap, pair.bitmap) - bc := BitCount(bm) + bc := IntersectionCount(src_bitmap, pair.bitmap) + //bc := BitCount(bm) if bc > 0 { - results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category}) + results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category}) counter = counter + 1 } x = i @@ -286,15 +286,15 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { } - abitmap := Intersection(src_bitmap, o.bitmap) - bc := BitCount(abitmap) + bc := IntersectionCount(src_bitmap, o.bitmap) + //bc := BitCount(abitmap) if bc > current_threshold { if results[end-1].Count > bc { - results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category} + results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category} current_threshold = bc } else { - results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category} + results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category} sort.Sort(results) o = results[end] current_threshold = o.Count diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index 585f79adf..0c04f5030 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -154,16 +154,15 @@ func TestFragment(t *testing.T) { res, _ := dummy.Clear(general) So(res, ShouldEqual, true) }) - Convey("store ", t, func() { b := uint64(1029) - compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA=" + // compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA=" // compressed := "H4sIAAAJbogA/2JmYWBS+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGBweN/E+M/RgdGRoUf/6EAk/GbGUQy4AUAAAAA//8BAAD//2vjG9ezAAAA" - + compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" dummy.LoadBitmap(brand, b, compressed, 0) bh1, _ := dummy.Get(brand, b) before, _ := dummy.Count(brand, bh1) - So(15228, ShouldEqual, before) + So(4096, ShouldEqual, before) }) } diff --git a/index/storage_leveldb.go b/index/storage_leveldb.go index 06d3d7f87..3a50decd4 100644 --- a/index/storage_leveldb.go +++ b/index/storage_leveldb.go @@ -86,7 +86,7 @@ func (self *LevelDBStorage) Fetch(bitmap_id uint64, db string, frame string, sli block, filter = decodeValue(iter.Value()) if chunk_key != COUNTERMASK { if chunk_key != last_key { - chunk = &Chunk{chunk_key, BlockArray{}} + chunk = &Chunk{chunk_key, BlockArray{make([]uint64, 32, 32)}} bitmap.AddChunk(chunk) } chunk.Value.Block[block_index] = block