diff --git a/roaring/roaring.go b/roaring/roaring.go index 3a4050cea..9c4274df9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1903,16 +1903,25 @@ func intersectionCount(a, b *Container) int32 { func intersectionCountArrayArray(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayArray") - na, nb := len(a.array), len(b.array) - for i, j := 0, 0; i < na && j < nb; { - va, vb := a.array[i], b.array[j] - if va < vb { - i++ - } else if va > vb { + ca, cb := a.array, b.array + na, nb := len(ca), len(cb) + if na == 0 || nb == 0 { + return 0 + } + if na > nb { + ca, cb = cb, ca + na, nb = nb, na // nolint: ineffassign + } + j := 0 + for _, va := range ca { + for cb[j] < va { j++ - } else { + if j >= nb { + return n + } + } + if cb[j] == va { n++ - i, j = i+1, j+1 } } return n @@ -2102,12 +2111,12 @@ func intersectRunRun(a, b *Container) *Container { return output } -// intersectBitmapRun returns an array container if the run container's -// cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container. +// intersectBitmapRun returns an array container if either container's +// cardinality is <= ArrayMaxSize. Otherwise it returns a bitmap container. func intersectBitmapRun(a, b *Container) *Container { statsHit("intersect/BitmapRun") var output *Container - if b.n < ArrayMaxSize { + if b.n <= ArrayMaxSize || a.n <= ArrayMaxSize { // output is array container output = &Container{containerType: containerArray} for _, iv := range b.runs { @@ -2161,9 +2170,6 @@ func intersectBitmapRun(a, b *Container) *Container { valast = vastart + 63 } } - if output.n < ArrayMaxSize { - output.bitmapToArray() - } } return output } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 266bf39e7..78b686371 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -165,8 +165,8 @@ func TestRunCountRange(t *testing.T) { } c.add(17) - c.add(18) c.add(19) + c.add(18) cnt = c.runCountRange(1, 22) if cnt != 10 { @@ -180,6 +180,11 @@ func TestRunCountRange(t *testing.T) { if cnt != 9 { t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt) } + // verify that the disparate ops resulted in three separate runs + cnt = c.countRuns() + if cnt != 3 { + t.Fatalf("should get 3 total runs, but got: %v [%v]", cnt, c.runs) + } } func TestRunContains(t *testing.T) { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 64cb45e81..f5c016ea3 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -411,13 +411,29 @@ func TestBitmap_Intersection_Empty(t *testing.T) { } func TestBitmap_IntersectArrayArray(t *testing.T) { - bm0 := roaring.NewFileBitmap(0, 1, 2683, 5005) + bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005) bm1 := roaring.NewFileBitmap(0, 2683, 2684, 5000) + expected := []uint64{0, 2683} result := bm0.Intersect(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) } + for _, e := range expected { + if !result.Contains(e) { + t.Fatalf("missing value %d", e) + } + } + // confirm that it also works going the other way + result = bm1.Intersect(bm0) + if n := result.Count(); n != 2 { + t.Fatalf("unexpected n: %d", n) + } + for _, e := range expected { + if !result.Contains(e) { + t.Fatalf("missing value %d", e) + } + } } func TestBitmap_IntersectBitmapBitmap(t *testing.T) { @@ -689,10 +705,10 @@ func TestBitmap_Flip_After(t *testing.T) { } -// Ensure bitmap can return the number of intersecting bits in two bitmaps. +// Ensure bitmap can return the number of intersecting bits in two arrays. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { - bm0 := roaring.NewFileBitmap(0, 1, 1000001, 1000002, 1000003) - bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewFileBitmap(0, 50000, 999998, 999999, 1000000, 1000001, 1000002) if n := bm0.IntersectionCount(bm1); n != 3 { t.Fatalf("unexpected n: %d", n) @@ -1056,19 +1072,39 @@ func TestBitmapBufIterator(t *testing.T) { } -var benchmarkBitmapIntersectionCountData struct { - a, b, r *roaring.Bitmap +// this data is used to test various operations across +// different types. +type benchmarkSampleData struct { + a1, a2, b, r1, r2 *roaring.Bitmap } -func getBenchData() *struct{ a, b, r *roaring.Bitmap } { - data := &benchmarkBitmapIntersectionCountData - if data.a == nil { +var sampleData benchmarkSampleData + +func isAllType(b *roaring.Bitmap, typ string) bool { + bi := b.Info() + for _, c := range bi.Containers { + if c.Type != typ { + return false + } + } + return true +} + +func getBenchData(b *testing.B) *benchmarkSampleData { + data := &sampleData + if data.a1 == nil { const max = (1 << 24) / 64 // Build bitmap with array container. - data.a = roaring.NewFileBitmap() - for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ { - data.a.Add(uint64(rand.Intn(max))) + data.a1 = roaring.NewFileBitmap() + data.a2 = roaring.NewFileBitmap() + // two lists of different lengths + for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { + data.a1.Add(uint64(rand.Intn(max))) + data.a2.Add(uint64(rand.Intn(max))) + } + for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { + data.a1.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. @@ -1078,12 +1114,42 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } { } // build bitmap with run container - data.r = roaring.NewFileBitmap() + data.r1 = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { - data.r.Add(uint64(i)) + data.r1.Add(uint64(i)) } + // build bitmap with multiple runs + data.r2 = roaring.NewFileBitmap() + for i, n := 0, MaxContainerVal; i < n; i++ { + data.r2.Add(uint64(i)) + // break the runs up, this should produce 16 runs, which + // is small enough to make RLE tempting + if i&0xfff == 0xfff { + i += 5 + } + } + data.a1.Optimize() + data.a2.Optimize() + data.b.Optimize() + data.r1.Optimize() + data.r2.Optimize() } + if !isAllType(data.a1, "array") { + b.Fatalf("expected data.a1 to be an array, it wasn't.") + } + if !isAllType(data.a2, "array") { + b.Fatalf("expected data.a2 to be an array, it wasn't.") + } + if !isAllType(data.b, "bitmap") { + b.Fatalf("expected data.b to be a bitmap, it wasn't.") + } + if !isAllType(data.r1, "run") { + b.Fatalf("expected data.r1 to be RLE, it wasn't.") + } + if !isAllType(data.r2, "run") { + b.Fatalf("expected data.r2 to be RLE, it wasn't.") + } return data } @@ -1138,30 +1204,65 @@ func TestBitmap_Intersect(t *testing.T) { } } +func BenchmarkGetBenchData(b *testing.B) { + for i := 0; i < b.N; i++ { + sampleData = benchmarkSampleData{} + getBenchData(b) + } +} + func BenchmarkBitmap_IntersectionCount_ArrayRun(b *testing.B) { - data := getBenchData() + data := getBenchData(b) // Reset timer & benchmark. b.ResetTimer() for i := 0; i < b.N; i++ { - data.a.IntersectionCount(data.r) + data.a1.IntersectionCount(data.r1) + } +} + +func BenchmarkBitmap_IntersectionCount_ArrayRuns(b *testing.B) { + data := getBenchData(b) + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.a1.IntersectionCount(data.r2) } } func BenchmarkBitmap_IntersectionCount_BitmapRun(b *testing.B) { - data := getBenchData() + data := getBenchData(b) // Reset timer & benchmark. b.ResetTimer() for i := 0; i < b.N; i++ { - data.b.IntersectionCount(data.r) + data.b.IntersectionCount(data.r1) + } +} + +func BenchmarkBitmap_IntersectionCount_BitmapRuns(b *testing.B) { + data := getBenchData(b) + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.b.IntersectionCount(data.r2) + } +} + +func BenchmarkBitmap_IntersectionCount_ArrayArray(b *testing.B) { + data := getBenchData(b) + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.a1.IntersectionCount(data.a2) + data.a2.IntersectionCount(data.a1) } } func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) { - data := getBenchData() + data := getBenchData(b) // Reset timer & benchmark. b.ResetTimer() for i := 0; i < b.N; i++ { - data.a.IntersectionCount(data.b) + data.a1.IntersectionCount(data.b) } }