scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-09 12:27:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into translate-file-size

Browse source
This commit is contained in:
Travis Turner 2018-11-19 11:43:42 -06:00 • committed by GitHub
commit c8ac958360
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 147 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
roaring/roaring.go
View file

@ -1903,16 +1903,25 @@ func intersectionCount(a, b *Container) int32 {
func intersectionCountArrayArray(a, b *Container) (n int32) {
statsHit("intersectionCount/ArrayArray")
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.array[j]
if va < vb {
i++
} else if va > vb {
ca, cb := a.array, b.array
na, nb := len(ca), len(cb)
if na == 0 || nb == 0 {
return 0
}
if na > nb {
ca, cb = cb, ca
na, nb = nb, na // nolint: ineffassign
}
j := 0
for _, va := range ca {
for cb[j] < va {
j++
} else {
if j >= nb {
return n
}
}
if cb[j] == va {
n++
i, j = i+1, j+1
}
}
return n
@ -2102,12 +2111,12 @@ func intersectRunRun(a, b *Container) *Container {
return output
}
// intersectBitmapRun returns an array container if the run container's
// cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container.
// intersectBitmapRun returns an array container if either container's
// cardinality is <= ArrayMaxSize. Otherwise it returns a bitmap container.
func intersectBitmapRun(a, b *Container) *Container {
statsHit("intersect/BitmapRun")
var output *Container
if b.n < ArrayMaxSize {
if b.n <= ArrayMaxSize || a.n <= ArrayMaxSize {
// output is array container
output = &Container{containerType: containerArray}
for _, iv := range b.runs {
@ -2161,9 +2170,6 @@ func intersectBitmapRun(a, b *Container) *Container {
valast = vastart + 63
}
}
if output.n < ArrayMaxSize {
output.bitmapToArray()
}
}
return output
}

7
roaring/roaring_internal_test.go
View file

@ -165,8 +165,8 @@ func TestRunCountRange(t *testing.T) {
}
c.add(17)
c.add(18)
c.add(19)
c.add(18)
cnt = c.runCountRange(1, 22)
if cnt != 10 {
@ -180,6 +180,11 @@ func TestRunCountRange(t *testing.T) {
if cnt != 9 {
t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt)
}
// verify that the disparate ops resulted in three separate runs
cnt = c.countRuns()
if cnt != 3 {
t.Fatalf("should get 3 total runs, but got: %v [%v]", cnt, c.runs)
}
}
func TestRunContains(t *testing.T) {

141
roaring/roaring_test.go
View file

@ -411,13 +411,29 @@ func TestBitmap_Intersection_Empty(t *testing.T) {
}
func TestBitmap_IntersectArrayArray(t *testing.T) {
bm0 := roaring.NewFileBitmap(0, 1, 2683, 5005)
bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005)
bm1 := roaring.NewFileBitmap(0, 2683, 2684, 5000)
expected := []uint64{0, 2683}
result := bm0.Intersect(bm1)
if n := result.Count(); n != 2 {
t.Fatalf("unexpected n: %d", n)
}
for _, e := range expected {
if !result.Contains(e) {
t.Fatalf("missing value %d", e)
}
}
// confirm that it also works going the other way
result = bm1.Intersect(bm0)
if n := result.Count(); n != 2 {
t.Fatalf("unexpected n: %d", n)
}
for _, e := range expected {
if !result.Contains(e) {
t.Fatalf("missing value %d", e)
}
}
}
func TestBitmap_IntersectBitmapBitmap(t *testing.T) {
@ -689,10 +705,10 @@ func TestBitmap_Flip_After(t *testing.T) {
}
// Ensure bitmap can return the number of intersecting bits in two bitmaps.
// Ensure bitmap can return the number of intersecting bits in two arrays.
func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) {
bm0 := roaring.NewFileBitmap(0, 1, 1000001, 1000002, 1000003)
bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
bm1 := roaring.NewFileBitmap(0, 50000, 999998, 999999, 1000000, 1000001, 1000002)
if n := bm0.IntersectionCount(bm1); n != 3 {
t.Fatalf("unexpected n: %d", n)
@ -1056,19 +1072,39 @@ func TestBitmapBufIterator(t *testing.T) {
}
var benchmarkBitmapIntersectionCountData struct {
a, b, r *roaring.Bitmap
// this data is used to test various operations across
// different types.
type benchmarkSampleData struct {
a1, a2, b, r1, r2 *roaring.Bitmap
}
func getBenchData() *struct{ a, b, r *roaring.Bitmap } {
data := &benchmarkBitmapIntersectionCountData
if data.a == nil {
var sampleData benchmarkSampleData
func isAllType(b *roaring.Bitmap, typ string) bool {
bi := b.Info()
for _, c := range bi.Containers {
if c.Type != typ {
return false
}
}
return true
}
func getBenchData(b *testing.B) *benchmarkSampleData {
data := &sampleData
if data.a1 == nil {
const max = (1 << 24) / 64
// Build bitmap with array container.
data.a = roaring.NewFileBitmap()
for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ {
data.a.Add(uint64(rand.Intn(max)))
data.a1 = roaring.NewFileBitmap()
data.a2 = roaring.NewFileBitmap()
// two lists of different lengths
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
data.a1.Add(uint64(rand.Intn(max)))
data.a2.Add(uint64(rand.Intn(max)))
}
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
data.a1.Add(uint64(rand.Intn(max)))
}
// Build bitmap with bitmap container.
@ -1078,12 +1114,42 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } {
}
// build bitmap with run container
data.r = roaring.NewFileBitmap()
data.r1 = roaring.NewFileBitmap()
for i, n := 0, MaxContainerVal; i < n; i++ {
data.r.Add(uint64(i))
data.r1.Add(uint64(i))
}
// build bitmap with multiple runs
data.r2 = roaring.NewFileBitmap()
for i, n := 0, MaxContainerVal; i < n; i++ {
data.r2.Add(uint64(i))
// break the runs up, this should produce 16 runs, which
// is small enough to make RLE tempting
if i&0xfff == 0xfff {
i += 5
}
}
data.a1.Optimize()
data.a2.Optimize()
data.b.Optimize()
data.r1.Optimize()
data.r2.Optimize()
}
if !isAllType(data.a1, "array") {
b.Fatalf("expected data.a1 to be an array, it wasn't.")
}
if !isAllType(data.a2, "array") {
b.Fatalf("expected data.a2 to be an array, it wasn't.")
}
if !isAllType(data.b, "bitmap") {
b.Fatalf("expected data.b to be a bitmap, it wasn't.")
}
if !isAllType(data.r1, "run") {
b.Fatalf("expected data.r1 to be RLE, it wasn't.")
}
if !isAllType(data.r2, "run") {
b.Fatalf("expected data.r2 to be RLE, it wasn't.")
}
return data
}
@ -1138,30 +1204,65 @@ func TestBitmap_Intersect(t *testing.T) {
}
}
func BenchmarkGetBenchData(b *testing.B) {
for i := 0; i < b.N; i++ {
sampleData = benchmarkSampleData{}
getBenchData(b)
}
}
func BenchmarkBitmap_IntersectionCount_ArrayRun(b *testing.B) {
data := getBenchData()
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a.IntersectionCount(data.r)
data.a1.IntersectionCount(data.r1)
}
}
func BenchmarkBitmap_IntersectionCount_ArrayRuns(b *testing.B) {
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a1.IntersectionCount(data.r2)
}
}
func BenchmarkBitmap_IntersectionCount_BitmapRun(b *testing.B) {
data := getBenchData()
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.b.IntersectionCount(data.r)
data.b.IntersectionCount(data.r1)
}
}
func BenchmarkBitmap_IntersectionCount_BitmapRuns(b *testing.B) {
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.b.IntersectionCount(data.r2)
}
}
func BenchmarkBitmap_IntersectionCount_ArrayArray(b *testing.B) {
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a1.IntersectionCount(data.a2)
data.a2.IntersectionCount(data.a1)
}
}
func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) {
data := getBenchData()
data := getBenchData(b)
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a.IntersectionCount(data.b)
data.a1.IntersectionCount(data.b)
}
}