From 3b32db998dd857afed5d56d7523e5199d848ced9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 7 Jul 2016 11:22:15 -0500 Subject: [PATCH 1/8] ignore checksum on single node clusters --- fragment.go | 8 ++++++-- handler.go | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index 4c9be2782..db934841e 100644 --- a/fragment.go +++ b/fragment.go @@ -1189,13 +1189,17 @@ func (s *FragmentSyncer) isClosing() bool { func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. nodes := s.Cluster.FragmentNodes(s.Fragment.DB(), s.Fragment.Slice()) - + if len(nodes) == 1 { + //fmt.Println("no place to replicate", s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.Slice()) + return nil + } // Create a set of blocks. blockSets := make([][]FragmentBlock, 0, len(nodes)) for _, node := range nodes { // Read local blocks. if node.Host == s.Host { - blockSets = append(blockSets, s.Fragment.Blocks()) + b := s.Fragment.Blocks() + blockSets = append(blockSets, b) continue } diff --git a/handler.go b/handler.go index 2c1510d53..fe4cecd85 100644 --- a/handler.go +++ b/handler.go @@ -59,6 +59,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { pprof.Profile(w, r) case "/debug/pprof/symbol": pprof.Symbol(w, r) + case "/debug/pprof/trace": + pprof.Trace(w, r) default: pprof.Index(w, r) } From 13f8ce673ea08904d06bfc8c90ecf6a0402311bb Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 12 Aug 2016 15:10:50 -0500 Subject: [PATCH 2/8] cache loading --- fragment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 2f35e2c95..6268ab729 100644 --- a/fragment.go +++ b/fragment.go @@ -234,7 +234,7 @@ func (f *Fragment) openCache() error { // Read in all bitmaps by ID. // This will cause them to be added to the cache. for _, bitmapID := range pb.GetBitmapIDs() { - bm := f.bitmap(bitmapID) + f.bitmap(bitmapID) } return nil From d4067ef34ead169cfd0a16f13941064df46ba7dc Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 30 Aug 2016 17:36:00 -0500 Subject: [PATCH 3/8] corrected ClearBit --- bitmap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitmap.go b/bitmap.go index d5f1fc9a7..40881245a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -127,7 +127,7 @@ func (b *Bitmap) SetBit(i uint64) (changed bool) { // ClearBit clears the i-th bit of the bitmap. func (b *Bitmap) ClearBit(i uint64) (changed bool) { - return b.createSegmentIfNotExists(i / SliceWidth).SetBit(i) + return b.createSegmentIfNotExists(i / SliceWidth).ClearBit(i) } func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment { From b7780518cc3d0945b7287f34f5b54bcd253eb2ca Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 15 Sep 2016 13:37:10 -0500 Subject: [PATCH 4/8] cache update on import adjustment --- fragment.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index e38867474..1369a0143 100644 --- a/fragment.go +++ b/fragment.go @@ -883,7 +883,8 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { } // Write to storage. - if _, err := f.storage.Add(pos); err != nil { + changed, err := f.storage.Add(pos) + if err != nil { return err } @@ -898,13 +899,14 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { bmCounter = bitmap.Count() lastID = bitmapID } - if bitmap.SetBit(profileID) { + if changed { bmCounter += 1 } // Invalidate block checksum. delete(f.checksums, int(bitmapID/HashBlockSize)) } + f.cache.Add(lastID, bmCounter) f.cache.Invalidate() return nil From 57810f8ab6cde05314e5b950138d025b20186dab Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 22 Sep 2016 03:05:53 -0500 Subject: [PATCH 5/8] merge --- cmd/pilosactl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 507b920a8..ee43eb968 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -233,7 +233,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand Stdout: stdout, Stderr: stderr, - BufferSize: 1000000, + BufferSize: 10000000, } } From 415507ffe3a959faa9b4eb516d23595f2605077f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 22 Sep 2016 16:47:39 -0500 Subject: [PATCH 6/8] fix for intersect and updated tests --- roaring/roaring.go | 40 ++++++++++++++++++++++++++++++---------- roaring/roaring_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ba4bfbf87..7a98de3d8 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -275,15 +275,36 @@ func (b *Bitmap) container(key uint64) *container { } return b.containers[i] } +func insertU64(original []uint64, position int, value uint64) []uint64 { + l := len(original) + target := original + if cap(original) == l { + target = make([]uint64, l+1, l+524288) + copy(target, original[:position]) + } else { + target = append(target, 0) + } + copy(target[position+1:], original[position:]) + target[position] = value + return target +} +func insertContainer(original []*container, position int, value *container) []*container { + l := len(original) + target := original + if cap(original) == l { + target = make([]*container, l+1, l+524288) + copy(target, original[:position]) + } else { + target = append(target, nil) + } + copy(target[position+1:], original[position:]) + target[position] = value + return target +} func (b *Bitmap) insertAt(key uint64, c *container, i int) { - b.keys = append(b.keys, 0) - copy(b.keys[i+1:], b.keys[i:]) - b.keys[i] = key - - b.containers = append(b.containers, nil) - copy(b.containers[i+1:], b.containers[i:]) - b.containers[i] = c + b.keys = insertU64(b.keys, i, key) + b.containers = insertContainer(b.containers, i, c) } // IntersectionCount returns the number of intersections between b and other. @@ -309,7 +330,6 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { ki, ci := b.keys, b.containers kj, cj := other.keys, other.containers - for { var key uint64 var container *container @@ -327,10 +347,10 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { key, container = ki[0], intersect(ci[0], cj[0]) ki, ci = ki[1:], ci[1:] kj, cj = kj[1:], cj[1:] + output.keys = append(output.keys, key) + output.containers = append(output.containers, container) } - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) } return output diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index cb03eb7d3..1493afa2e 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -80,6 +80,42 @@ func TestBitmap_Max(t *testing.T) { } } +func TestBitmap_Intersection(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + + result := bm0.Intersect(bm1) + if n := result.Count(); n != 1 { + t.Fatalf("unexpected n: %d", n) + } + +} + +func TestBitmap_Difference(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + result := bm0.Difference(bm1) + //expect to have just 0 + if n := result.Count(); n != 1 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_Union(t *testing.T) { + bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + result := bm0.Union(bm1) + if n := result.Count(); n != 5 { + t.Fatalf("unexpected n: %d", n) + } +} + // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) From 11a2013f2da1140cbd1f9078c30be17a75351fdb Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 23 Sep 2016 13:27:35 -0500 Subject: [PATCH 7/8] fix roaring.Difference crash --- roaring/roaring.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 7a98de3d8..b581bd195 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -406,16 +406,18 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { } else if nj == 0 || ki[0] < kj[0] { // eof(j) or i < j key, container = ki[0], ci[0].clone() ki, ci = ki[1:], ci[1:] + output.keys = append(output.keys, key) + output.containers = append(output.containers, container) } else if nj > 0 && ki[0] > kj[0] { // i > j kj, cj = kj[1:], cj[1:] } else { // i == j key, container = ki[0], difference(ci[0], cj[0]) ki, ci = ki[1:], ci[1:] kj, cj = kj[1:], cj[1:] + output.keys = append(output.keys, key) + output.containers = append(output.containers, container) } - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) } return output From d41f998932e57064f3bc6e2fd31bfd311a1dec42 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 23 Sep 2016 16:04:22 -0500 Subject: [PATCH 8/8] cleaned up constant --- roaring/roaring.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b581bd195..6c65cc3ed 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -20,6 +20,9 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 + + // manual allocation size tuned to our average client data + manualAlloc = 524288 ) // Bitmap represents a roaring bitmap. @@ -279,7 +282,7 @@ func insertU64(original []uint64, position int, value uint64) []uint64 { l := len(original) target := original if cap(original) == l { - target = make([]uint64, l+1, l+524288) + target = make([]uint64, l+1, l+manualAlloc) copy(target, original[:position]) } else { target = append(target, 0) @@ -293,7 +296,7 @@ func insertContainer(original []*container, position int, value *container) []*c l := len(original) target := original if cap(original) == l { - target = make([]*container, l+1, l+524288) + target = make([]*container, l+1, l+manualAlloc) copy(target, original[:position]) } else { target = append(target, nil)