From cf120e9f021a759b0a1aa5966e135e74846dd072 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 18 Aug 2014 21:08:38 +0000 Subject: [PATCH] slice based popcount --- index/bitmap.go | 34 ++++++++++++++++++++++++++++++-- index/bitmap_test.go | 28 ++++++++++++++++++++++++++ index/brand.go | 14 ++++++------- index/fragment_container_test.go | 7 +++---- index/timeframe_test.go | 2 +- 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/index/bitmap.go b/index/bitmap.go index f2ad791cf..723b767e8 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -116,6 +116,11 @@ func BlockArray_copy(a *BlockArray) BlockArray { } return o } + +func BlockArray_andcount(a *BlockArray, b *BlockArray) uint64 { + return popcntAndSliceAsm(a.Block, b.Block) +} + func BlockArray_intersection(a *BlockArray, b *BlockArray) BlockArray { var o = BlockArray{make([]uint64, 32, 32)} for i, _ := range a.Block { @@ -172,6 +177,31 @@ func Clone(a_bm IBitmap) IBitmap { return output } +func IntersectionCount(a_bm IBitmap, b_bm IBitmap) uint64 { + var a = a_bm.Min() + var b = b_bm.Min() + defer a.Close() + defer b.Close() + results := uint64(0) + + for { + if b.Limit() || a.Limit() { + break + } else if a.Item().Key < b.Item().Key { + a = a.Next() + } else if a.Item().Key > b.Item().Key { + b = b.Next() + } else if a.Item().Key == b.Item().Key { + var a_node = a.Item() + var b_node = b.Item().Value + results += BlockArray_andcount(&a_node.Value, &b_node) + a = a.Next() + b = b.Next() + } + } + return results +} + func Intersection(a_bm IBitmap, b_bm IBitmap) IBitmap { var a = a_bm.Min() var b = b_bm.Min() @@ -429,7 +459,7 @@ func (b *Bitmap) ToBytes() []byte { enc.Encode(b.nodes.Len()) for i := b.nodes.Min(); !i.Limit(); i = i.Next() { obj := i.Item().(*Chunk) - err := enc.Encode(&obj) + err := enc.Encode(obj) if err != nil { log.Println(err) } @@ -447,7 +477,7 @@ func (self *Bitmap) FromBytes(raw []byte) { for i := 0; i < size; i++ { //chunk := &Chunk{0, BlockArray{make([]uint64, 32, 32)}} var chunk Chunk - dec.Decode(chunk) + dec.Decode(&chunk) self.AddChunk(&chunk) } self.SetCount(BitCount(self)) diff --git a/index/bitmap_test.go b/index/bitmap_test.go index 2915ea21d..cc2a41db2 100644 --- a/index/bitmap_test.go +++ b/index/bitmap_test.go @@ -1,6 +1,7 @@ package index import ( + "fmt" "testing" "time" @@ -86,4 +87,31 @@ func TestBitmaps(t *testing.T) { BitCount(all) So(start, ShouldHappenWithin, time.Duration(1)*time.Millisecond, time.Now()) }) + + Convey("Compressed ", t, func() { + all := CreateRBBitmap() + for i := uint64(0); i < uint64(4096); i++ { + SetBit(all, i) + } + cs := all.ToCompressString() + fmt.Println(cs) + bm := CreateRBBitmap() + bm.FromCompressString(cs) + So(BitCount(all), ShouldEqual, BitCount(bm)) + }) + + Convey("AndCount ", t, func() { + a := CreateRBBitmap() + for i := uint64(0); i < uint64(4096); i++ { + SetBit(a, i) + } + b := CreateRBBitmap() + for i := uint64(0); i < uint64(8192); i++ { + SetBit(b, i) + } + c1 := IntersectionCount(a, b) + c := Intersection(a, b) + So(c1, ShouldEqual, BitCount(c)) + }) + } diff --git a/index/brand.go b/index/brand.go index ba9954fde..4474f7489 100644 --- a/index/brand.go +++ b/index/brand.go @@ -240,10 +240,10 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { if counter > n { break } - bm := Intersection(src_bitmap, pair.bitmap) - bc := BitCount(bm) + bc := IntersectionCount(src_bitmap, pair.bitmap) + //bc := BitCount(bm) if bc > 0 { - results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category}) + results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category}) counter = counter + 1 } x = i @@ -286,15 +286,15 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { } - abitmap := Intersection(src_bitmap, o.bitmap) - bc := BitCount(abitmap) + bc := IntersectionCount(src_bitmap, o.bitmap) + //bc := BitCount(abitmap) if bc > current_threshold { if results[end-1].Count > bc { - results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category} + results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category} current_threshold = bc } else { - results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category} + results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category} sort.Sort(results) o = results[end] current_threshold = o.Count diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index 585f79adf..0c04f5030 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -154,16 +154,15 @@ func TestFragment(t *testing.T) { res, _ := dummy.Clear(general) So(res, ShouldEqual, true) }) - Convey("store ", t, func() { b := uint64(1029) - compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA=" + // compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA=" // compressed := "H4sIAAAJbogA/2JmYWBS+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGBweN/E+M/RgdGRoUf/6EAk/GbGUQy4AUAAAAA//8BAAD//2vjG9ezAAAA" - + compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" dummy.LoadBitmap(brand, b, compressed, 0) bh1, _ := dummy.Get(brand, b) before, _ := dummy.Count(brand, bh1) - So(15228, ShouldEqual, before) + So(4096, ShouldEqual, before) }) } diff --git a/index/timeframe_test.go b/index/timeframe_test.go index 77125e1b1..f5e00869a 100644 --- a/index/timeframe_test.go +++ b/index/timeframe_test.go @@ -34,7 +34,7 @@ func TestTimeFrame(t *testing.T) { getTime(uint64(1), "2014-08-11 14:00") spew.Dump(m) - So(len(m), ShouldEqual, 1) + So(len(m), ShouldEqual, 2) }) if true { return