mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
Merge pull request #258 from kuba--/intersect-inplace
The first implementation of intersect in place
This commit is contained in:
Kuba Podgórski
GitHub
commit
5578eb8daa
5 changed files with 1114 additions and 81 deletions
|
|
@ -77,8 +77,7 @@ func (sc *sliceContainers) Remove(key uint64) {
|
|||
return
|
||||
}
|
||||
if key == sc.lastKey {
|
||||
sc.lastKey = ^uint64(0)
|
||||
sc.lastContainer = nil
|
||||
sc.invalidateCache()
|
||||
}
|
||||
sc.keys = append(sc.keys[:i], sc.keys[i+1:]...)
|
||||
sc.containers = append(sc.containers[:i], sc.containers[i+1:]...)
|
||||
|
|
@ -159,8 +158,7 @@ func (sc *sliceContainers) Count() uint64 {
|
|||
func (sc *sliceContainers) Reset() {
|
||||
sc.keys = sc.keys[:0]
|
||||
sc.containers = sc.containers[:0]
|
||||
sc.lastContainer = nil
|
||||
sc.lastKey = ^uint64(0)
|
||||
sc.invalidateCache()
|
||||
}
|
||||
|
||||
func (sc *sliceContainers) ResetN(n int) {
|
||||
|
|
@ -171,8 +169,7 @@ func (sc *sliceContainers) ResetN(n int) {
|
|||
sc.keys = sc.keys[:0]
|
||||
sc.containers = sc.containers[:0]
|
||||
}
|
||||
sc.lastContainer = nil
|
||||
sc.lastKey = ^uint64(0)
|
||||
sc.invalidateCache()
|
||||
}
|
||||
|
||||
func (sc *sliceContainers) seek(key uint64) (int, bool) {
|
||||
|
|
@ -187,13 +184,28 @@ func (sc *sliceContainers) seek(key uint64) (int, bool) {
|
|||
|
||||
func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
|
||||
i, found := sc.seek(key)
|
||||
return &sliceIterator{e: sc, i: i}, found
|
||||
return &sliceIterator{e: sc, i: i, index: i}, found
|
||||
}
|
||||
|
||||
// Repair tries to repair all containers,
|
||||
// results in nil and empty containers getting dropped from the slice.
|
||||
// For instance, that has to happen for writing the roaring format,
|
||||
// which can't represent an empty container (c.N() == 0).
|
||||
func (sc *sliceContainers) Repair() {
|
||||
for _, c := range sc.containers {
|
||||
n := 0
|
||||
for i, c := range sc.containers {
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
c.Repair()
|
||||
sc.containers[n] = c
|
||||
sc.keys[n] = sc.keys[i]
|
||||
n++
|
||||
}
|
||||
sc.containers = sc.containers[:n]
|
||||
sc.keys = sc.keys[:n]
|
||||
|
||||
sc.invalidateCache()
|
||||
}
|
||||
|
||||
// Update calls fn (existing-container, existed), and expects
|
||||
|
|
@ -228,27 +240,35 @@ func (sc *sliceContainers) UpdateEvery(fn func(uint64, *Container, bool) (*Conta
|
|||
sc.containers[i] = nc
|
||||
}
|
||||
}
|
||||
// invalidate cache.
|
||||
|
||||
sc.invalidateCache()
|
||||
}
|
||||
|
||||
func (sc *sliceContainers) invalidateCache() {
|
||||
sc.lastKey = ^uint64(0)
|
||||
sc.lastContainer = nil
|
||||
}
|
||||
|
||||
type sliceIterator struct {
|
||||
e *sliceContainers
|
||||
i int
|
||||
key uint64
|
||||
value *Container
|
||||
i int // next e's index to get key, value
|
||||
index int // current e's index of key, value
|
||||
key uint64 // current key
|
||||
value *Container // current value
|
||||
}
|
||||
|
||||
func (si *sliceIterator) Next() bool {
|
||||
if si.e == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// discard nil containers from iteration. we don't always
|
||||
// actually remove them because copying is expensive.
|
||||
for si.i < len(si.e.keys) {
|
||||
si.key = si.e.keys[si.i]
|
||||
si.value = si.e.containers[si.i]
|
||||
// keep the current index of key, value
|
||||
si.index = si.i
|
||||
si.i++
|
||||
if si.value != nil {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -99,3 +99,89 @@ func testContainersIterator(cs Containers, t *testing.T) {
|
|||
t.Fatalf("itr should be done, but got true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSliceContainers(t *testing.T) {
|
||||
const size = 10
|
||||
n := size
|
||||
sc := newSliceContainers()
|
||||
|
||||
// Add n keys
|
||||
for i := 0; i < n; i++ {
|
||||
key, set := uint64(i), []uint16{uint16(i)}
|
||||
sc.Put(key, NewContainerArray(set))
|
||||
}
|
||||
|
||||
t.Run("Get n keys", func(t *testing.T) {
|
||||
for i := 0; i < n; i++ {
|
||||
key, set := uint64(i), []uint16{uint16(i)}
|
||||
c := sc.Get(key)
|
||||
if c == nil {
|
||||
t.Fatalf("Get(%d) returned nil container", key)
|
||||
}
|
||||
if c.data[0] != set[0] {
|
||||
t.Fatalf("Get(%d): expected: %v, got: %v", key, set[0], c.data[0])
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Last key/container", func(t *testing.T) {
|
||||
key, c := sc.Last()
|
||||
if key != uint64(n-1) || c.data[0] != uint16(n-1) {
|
||||
t.Fatalf("Last: expected: %v, got: %d, %v", n-1, key, c.data)
|
||||
}
|
||||
})
|
||||
|
||||
// Remove odd keys
|
||||
for i := 1; i < size; i += 2 {
|
||||
key := uint64(i)
|
||||
sc.Remove(key)
|
||||
n--
|
||||
}
|
||||
|
||||
t.Run("Try to Get removed containers", func(t *testing.T) {
|
||||
for i := 1; i < size; i += 2 {
|
||||
key := uint64(i)
|
||||
c := sc.Get(key)
|
||||
if c != nil {
|
||||
t.Fatalf("Get(for non existing key %d): found container: %v", key, c.data)
|
||||
}
|
||||
}
|
||||
|
||||
// Test - Last key/container
|
||||
key, c := sc.Last()
|
||||
if key != uint64(size-2) || c.data[0] != uint16(size-2) {
|
||||
t.Fatalf("Last: expected: %v, got: %d, %v", size-2, key, c.data)
|
||||
}
|
||||
|
||||
if sc.Size() != n {
|
||||
t.Fatalf("Size: expected: %d, got: %d", n, sc.Size())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Nil containers and repair them", func(t *testing.T) {
|
||||
// Remove half of even containers
|
||||
for i := range sc.containers {
|
||||
if i%2 == 0 {
|
||||
sc.containers[i] = nil
|
||||
n--
|
||||
}
|
||||
}
|
||||
sc.Repair()
|
||||
|
||||
if sc.Size() != n {
|
||||
t.Fatalf("Size: expected: %d, got: %d", n, sc.Size())
|
||||
}
|
||||
|
||||
for i, key := range sc.keys {
|
||||
if sc.containers[i] == nil {
|
||||
t.Fatalf("Found nil container for key: %d at index: %d", key, i)
|
||||
} else {
|
||||
if sc.containers[i].data[0] != uint16(key) {
|
||||
t.Fatalf("Invalid container data for key: %d at index: %d - expected: %d, got: %d",
|
||||
key, i, uint16(key), sc.containers[i].data[0],
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -694,6 +694,474 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
|
|||
return output
|
||||
}
|
||||
|
||||
type mutableContainersIterator struct {
|
||||
c Containers
|
||||
|
||||
cit ContainerIterator
|
||||
sit *sliceIterator
|
||||
}
|
||||
|
||||
func newMutableContainersIterator(cs Containers, key uint64) *mutableContainersIterator {
|
||||
it := &mutableContainersIterator{c: cs}
|
||||
|
||||
if sc, ok := cs.(*sliceContainers); ok {
|
||||
if i, found := sc.seek(key); found {
|
||||
it.sit = &sliceIterator{e: sc, i: i, index: i}
|
||||
return it
|
||||
}
|
||||
}
|
||||
|
||||
it.cit, _ = cs.Iterator(key)
|
||||
return it
|
||||
}
|
||||
|
||||
func (it *mutableContainersIterator) update(key uint64, newCont *Container) {
|
||||
if it.sit != nil {
|
||||
if key == it.sit.key {
|
||||
it.sit.e.containers[it.sit.index] = newCont
|
||||
}
|
||||
}
|
||||
|
||||
it.c.Update(key, func(_ *Container, _ bool) (*Container, bool) {
|
||||
return newCont, true
|
||||
})
|
||||
}
|
||||
|
||||
func (it *mutableContainersIterator) Next() bool {
|
||||
if it.sit != nil {
|
||||
return it.sit.Next()
|
||||
}
|
||||
|
||||
return it.cit.Next()
|
||||
}
|
||||
|
||||
func (it *mutableContainersIterator) Value() (uint64, *Container) {
|
||||
if it.sit != nil {
|
||||
return it.sit.Value()
|
||||
}
|
||||
|
||||
return it.cit.Value()
|
||||
}
|
||||
|
||||
// IntersectInPlace returns the bitwise intersection of b and others,
|
||||
// modifying b in place.
|
||||
func (b *Bitmap) IntersectInPlace(others ...*Bitmap) {
|
||||
var bSize int
|
||||
if bSize = b.Size(); bSize == 0 {
|
||||
// If b doesn't have any containers then return early.
|
||||
return
|
||||
}
|
||||
|
||||
otherIters := make(handledIters, 0, len(others))
|
||||
for _, other := range others {
|
||||
it, _ := other.Containers.Iterator(0)
|
||||
if !it.Next() {
|
||||
// An empty bitmap - reset all
|
||||
b.Containers.Reset()
|
||||
return
|
||||
}
|
||||
|
||||
otherIters = append(otherIters, handledIter{
|
||||
iter: it,
|
||||
hasNext: true,
|
||||
})
|
||||
}
|
||||
|
||||
bIter := newMutableContainersIterator(b.Containers, 0)
|
||||
for bIter.Next() {
|
||||
bKey, bCont := bIter.Value()
|
||||
if bCont.N() == 0 {
|
||||
// No point in intersecting things from an empty container.
|
||||
bIter.update(bKey, nil)
|
||||
continue
|
||||
}
|
||||
|
||||
// Loop until every iters current value has been handled.
|
||||
for _, otherIter := range otherIters {
|
||||
if !otherIter.hasNext {
|
||||
continue
|
||||
}
|
||||
|
||||
otherKey, otherCont := otherIter.iter.Value()
|
||||
for otherKey < bKey {
|
||||
otherIter.hasNext = otherIter.iter.Next()
|
||||
if !otherIter.hasNext {
|
||||
break
|
||||
}
|
||||
otherKey, otherCont = otherIter.iter.Value()
|
||||
}
|
||||
|
||||
if bKey == otherKey {
|
||||
// Note: a nil container is valid, and has N == 0.
|
||||
if otherCont.N() != 0 {
|
||||
if bCont.frozen() {
|
||||
bCont = bCont.Clone()
|
||||
b.Containers.Put(bKey, bCont)
|
||||
}
|
||||
bCont = bCont.intersectInPlace(otherCont)
|
||||
bIter.update(bKey, bCont)
|
||||
if bCont == nil || bCont.N() == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
otherIter.hasNext = otherIter.iter.Next()
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
bIter.update(bKey, nil)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
b.Containers.Repair()
|
||||
}
|
||||
|
||||
func (c *Container) intersectInPlace(other *Container) *Container {
|
||||
// short-circuit the trivial cases
|
||||
if c == nil || other == nil || c.N() == 0 || other.N() == 0 {
|
||||
c = nil
|
||||
return c
|
||||
}
|
||||
cFull, otherFull := (c.N() == maxContainerVal+1), (other.N() == maxContainerVal+1)
|
||||
if cFull && otherFull {
|
||||
return c
|
||||
}
|
||||
if cFull {
|
||||
return c.copyInPlace(other)
|
||||
}
|
||||
if otherFull {
|
||||
return c
|
||||
}
|
||||
|
||||
switch c.typ() {
|
||||
case containerArray:
|
||||
switch other.typ() {
|
||||
case containerArray:
|
||||
return intersectArrayArrayInPlace(c, other)
|
||||
case containerBitmap:
|
||||
return intersectArrayBitmapInPlace(c, other)
|
||||
case containerRun:
|
||||
return intersectArrayRunInPlace(c, other)
|
||||
}
|
||||
|
||||
case containerBitmap:
|
||||
switch other.typ() {
|
||||
case containerArray:
|
||||
return intersectBitmapArrayInPlace(c, other)
|
||||
case containerBitmap:
|
||||
return intersectBitmapBitmapInPlace(c, other)
|
||||
case containerRun:
|
||||
return intersectBitmapRunInPlace(c, other)
|
||||
}
|
||||
|
||||
case containerRun:
|
||||
switch other.typ() {
|
||||
case containerArray:
|
||||
return intersectRunArrayInPlace(c, other)
|
||||
case containerBitmap:
|
||||
return intersectRunBitmapInPlace(c, other)
|
||||
case containerRun:
|
||||
return intersectRunRunInPlace(c, other)
|
||||
}
|
||||
}
|
||||
|
||||
if roaringParanoia {
|
||||
panic(fmt.Sprintf("invalid intersect op: unknown types %d/%d", c.typ(), other.typ()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Container) copyInPlace(other *Container) *Container {
|
||||
switch other.typ() {
|
||||
case containerArray:
|
||||
c.setTyp(containerArray)
|
||||
c.setArrayMaybeCopy(other.array(), true)
|
||||
c.setN(other.N())
|
||||
|
||||
case containerBitmap:
|
||||
bmp := make([]uint64, bitmapN)
|
||||
copy(bmp, other.bitmap())
|
||||
c.setTyp(containerBitmap)
|
||||
c.setBitmap(bmp)
|
||||
c.setN(other.N())
|
||||
|
||||
case containerRun:
|
||||
c.setTyp(containerRun)
|
||||
c.setRunsMaybeCopy(other.runs(), true)
|
||||
c.setN(other.N())
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func intersectArrayArrayInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/ArrayArray")
|
||||
|
||||
a = a.Thaw()
|
||||
aa, ba := a.array(), b.array()
|
||||
an, bn := len(aa), len(ba)
|
||||
n := 0
|
||||
for i, j := 0, 0; i < an && j < bn; {
|
||||
va, vb := aa[i], ba[j]
|
||||
|
||||
if va < vb {
|
||||
i++
|
||||
} else if va > vb {
|
||||
j++
|
||||
} else {
|
||||
aa[n] = va
|
||||
n, i, j = n+1, i+1, j+1
|
||||
}
|
||||
}
|
||||
aa = aa[:n]
|
||||
a.setArray(aa)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectArrayRunInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/ArrayRun")
|
||||
|
||||
a = a.Thaw()
|
||||
aa, br := a.array(), b.runs()
|
||||
an, bn := len(aa), len(br)
|
||||
|
||||
n := 0
|
||||
for i, j := 0, 0; i < an && j < bn; {
|
||||
va, vb := aa[i], br[j]
|
||||
if va < vb.start {
|
||||
i++
|
||||
} else if va > vb.last {
|
||||
j++
|
||||
} else {
|
||||
aa[n] = va
|
||||
n++
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
aa = aa[:n]
|
||||
a.setArray(aa)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectArrayBitmapInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/ArrayBitmap")
|
||||
|
||||
a = a.Thaw()
|
||||
aa := a.array()
|
||||
bb := b.bitmap()
|
||||
|
||||
n := 0
|
||||
for _, va := range aa {
|
||||
b := bb[va>>6]
|
||||
bidx := va % 64
|
||||
mask := uint64(1) << bidx
|
||||
if b&mask > 0 {
|
||||
aa[n] = va
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
aa = aa[:n]
|
||||
a.setArray(aa)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectBitmapBitmapInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/BitmapBitmap")
|
||||
|
||||
a = a.Thaw()
|
||||
ab := a.bitmap()[:bitmapN]
|
||||
bb := b.bitmap()[:bitmapN]
|
||||
|
||||
n := int32(0)
|
||||
for i := 0; i < bitmapN; i += 4 {
|
||||
// unrolling is still effective in go
|
||||
ptr := (*[4]uint64)(unsafe.Pointer(&bb[i]))
|
||||
ab[i] &= ptr[0]
|
||||
ab[i+1] &= ptr[1]
|
||||
ab[i+2] &= ptr[2]
|
||||
ab[i+3] &= ptr[3]
|
||||
|
||||
n += int32(popcount(ab[i])) +
|
||||
int32(popcount(ab[i+1])) +
|
||||
int32(popcount(ab[i+2])) +
|
||||
int32(popcount(ab[i+3]))
|
||||
}
|
||||
a.setN(n)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectBitmapArrayInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/BitmapArray")
|
||||
|
||||
a = a.Thaw()
|
||||
ab := a.bitmap()
|
||||
ba := b.array()
|
||||
|
||||
bn := len(ba)
|
||||
array := make([]uint16, bn)
|
||||
n := int32(0)
|
||||
for _, vb := range ba {
|
||||
i := vb >> 6
|
||||
mask := uint64(1) << uint(vb%64)
|
||||
if ab[i]&mask > 0 {
|
||||
array[n] = vb
|
||||
n++
|
||||
}
|
||||
}
|
||||
array = array[:n]
|
||||
a.setTyp(containerArray)
|
||||
a.setArray(array)
|
||||
a.setN(n)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectBitmapRunInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/BitmapRun")
|
||||
|
||||
a = a.Thaw()
|
||||
ab := a.bitmap()
|
||||
br := b.runs()
|
||||
an := len(ab)
|
||||
bitmap := make([]uint64, an)
|
||||
|
||||
n := int32(0)
|
||||
for _, vb := range br {
|
||||
i := vb.start >> 6 // index into a
|
||||
|
||||
vastart := i << 6
|
||||
valast := vastart + 63
|
||||
for valast >= vb.start && vastart <= vb.last && int(i) < an {
|
||||
if vastart >= vb.start && valast <= vb.last { // a within b
|
||||
bitmap[i] = ab[i]
|
||||
n += int32(popcount(ab[i]))
|
||||
} else if vb.start >= vastart && vb.last <= valast { // b within a
|
||||
var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
|
||||
bits := ab[i] & mask
|
||||
bitmap[i] |= bits
|
||||
n += int32(popcount(bits))
|
||||
} else if vastart < vb.start { // a overlaps front of b
|
||||
offset := 64 - (1 + valast - vb.start)
|
||||
bits := (ab[i] >> offset) << offset
|
||||
bitmap[i] |= bits
|
||||
n += int32(popcount(bits))
|
||||
} else if vb.start < vastart { // b overlaps front of a
|
||||
offset := 64 - (1 + vb.last - vastart)
|
||||
bits := (ab[i] << offset) >> offset
|
||||
bitmap[i] |= bits
|
||||
n += int32(popcount(bits))
|
||||
}
|
||||
|
||||
i++
|
||||
vastart = i << 6
|
||||
valast = vastart + 63
|
||||
}
|
||||
}
|
||||
a.setBitmap(bitmap)
|
||||
a.setN(n)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectRunRunInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/RunRun")
|
||||
|
||||
a = a.Thaw()
|
||||
ar, br := a.runs(), b.runs()
|
||||
an, bn := len(ar), len(br)
|
||||
|
||||
var runs []interval16
|
||||
if an > bn {
|
||||
runs = make([]interval16, 0, an)
|
||||
} else {
|
||||
runs = make([]interval16, 0, bn)
|
||||
}
|
||||
|
||||
n := int32(0)
|
||||
for i, j := 0, 0; i < an && j < bn; {
|
||||
va, vb := ar[i], br[j]
|
||||
|
||||
if va.last < vb.start {
|
||||
// |--va--| |--vb--|
|
||||
i++
|
||||
} else if vb.last < va.start {
|
||||
// |--vb--| |--va--|
|
||||
j++
|
||||
} else if va.last > vb.last && va.start >= vb.start {
|
||||
// |--vb-|-|-va--|
|
||||
runs = append(runs, interval16{start: va.start, last: vb.last})
|
||||
n += int32(vb.last-va.start) + 1
|
||||
j++
|
||||
} else if va.last > vb.last && va.start < vb.start {
|
||||
// |--va|--vb--|--|
|
||||
runs = append(runs, vb)
|
||||
n += int32(vb.last-vb.start) + 1
|
||||
j++
|
||||
} else if va.last <= vb.last && va.start >= vb.start {
|
||||
// |--vb|--va--|--|
|
||||
runs = append(runs, va)
|
||||
n += int32(va.last-va.start) + 1
|
||||
i++
|
||||
} else if va.last <= vb.last && va.start < vb.start {
|
||||
// |--va-|-|-vb--|
|
||||
runs = append(runs, interval16{start: vb.start, last: va.last})
|
||||
n += int32(va.last-vb.start) + 1
|
||||
i++
|
||||
}
|
||||
}
|
||||
a.setRuns(runs)
|
||||
a.setN(n)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectRunArrayInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/RunArray")
|
||||
|
||||
a = a.Thaw()
|
||||
ar, ba := a.runs(), b.array()
|
||||
an, bn := len(ar), len(ba)
|
||||
|
||||
array := make([]uint16, bn)
|
||||
n := 0
|
||||
for i, j := 0, 0; i < an && j < bn; {
|
||||
va, vb := ar[i], ba[j]
|
||||
if vb < va.start {
|
||||
j++
|
||||
} else if vb > va.last {
|
||||
i++
|
||||
} else {
|
||||
array[n] = vb
|
||||
n++
|
||||
j++
|
||||
}
|
||||
}
|
||||
|
||||
array = array[:n]
|
||||
a.setTyp(containerArray)
|
||||
a.setArray(array)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func intersectRunBitmapInPlace(a, b *Container) *Container {
|
||||
statsHit("intersectInPlace/RunBitmap")
|
||||
|
||||
// TODO(@kuba--):
|
||||
// Figure out how to efficiently intersect dense runs with bitmaps.
|
||||
// So far we convert run to bitmap and intersect bitmaps - it's much faster
|
||||
// than naive O(n^2) algorithm.
|
||||
a = a.runToBitmap()
|
||||
return intersectBitmapBitmapInPlace(a, b)
|
||||
}
|
||||
|
||||
// Union returns the bitwise union of b and others as a new bitmap.
|
||||
func (b *Bitmap) Union(others ...*Bitmap) *Bitmap {
|
||||
if len(others) == 1 {
|
||||
|
|
|
|||
|
|
@ -2754,6 +2754,15 @@ func differenceInPlaceWrapper(a, b *Container) *Container {
|
|||
return out.Containers.Get(0)
|
||||
}
|
||||
|
||||
func intersectInPlaceWrapper(a, b *Container) *Container {
|
||||
out := NewBitmap()
|
||||
out.Containers.Put(0, a.Clone())
|
||||
B := NewBitmap()
|
||||
B.Containers.Put(0, b)
|
||||
out.IntersectInPlace(B)
|
||||
return out.Containers.Get(0)
|
||||
}
|
||||
|
||||
func TestContainerCombinations(t *testing.T) {
|
||||
|
||||
cts := setupContainerTests()
|
||||
|
|
@ -2872,6 +2881,109 @@ func TestContainerCombinations(t *testing.T) {
|
|||
{intersect, "evenBitsSet", "oddBitsSet", "empty"},
|
||||
{intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"},
|
||||
|
||||
// intersect in place
|
||||
{intersectInPlaceWrapper, "empty", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "full", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "firstBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "lastBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "firstBitUnset", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "lastBitUnset", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "innerBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "outerBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "oddBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "empty", "evenBitsSet", "empty"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "full", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "full", "full", "full"},
|
||||
{intersectInPlaceWrapper, "full", "firstBitSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "full", "lastBitSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "full", "firstBitUnset", "firstBitUnset"},
|
||||
{intersectInPlaceWrapper, "full", "lastBitUnset", "lastBitUnset"},
|
||||
{intersectInPlaceWrapper, "full", "innerBitsSet", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "full", "outerBitsSet", "outerBitsSet"},
|
||||
{intersectInPlaceWrapper, "full", "oddBitsSet", "oddBitsSet"},
|
||||
{intersectInPlaceWrapper, "full", "evenBitsSet", "evenBitsSet"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "firstBitSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "full", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "firstBitSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "lastBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "firstBitUnset", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "lastBitUnset", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "innerBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "outerBitsSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "oddBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitSet", "evenBitsSet", "firstBitSet"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "lastBitSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "full", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "firstBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "lastBitSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "firstBitUnset", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "lastBitUnset", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "innerBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "outerBitsSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "oddBitsSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitSet", "evenBitsSet", "empty"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "full", "firstBitUnset"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "firstBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "lastBitSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "firstBitUnset", "firstBitUnset"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "lastBitUnset", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "innerBitsSet", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "outerBitsSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "firstBitUnset", "oddBitsSet", "oddBitsSet"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "full", "lastBitUnset"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "firstBitSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "lastBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "firstBitUnset", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "lastBitUnset", "lastBitUnset"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "innerBitsSet", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "outerBitsSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "lastBitUnset", "evenBitsSet", "evenBitsSet"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "full", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "firstBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "lastBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "firstBitUnset", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "lastBitUnset", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "innerBitsSet", "innerBitsSet"},
|
||||
{intersectInPlaceWrapper, "innerBitsSet", "outerBitsSet", "empty"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "full", "outerBitsSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "firstBitSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "lastBitSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "firstBitUnset", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "lastBitUnset", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "innerBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "outerBitsSet", "outerBitsSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "oddBitsSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "outerBitsSet", "evenBitsSet", "firstBitSet"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "full", "oddBitsSet"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "firstBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "lastBitSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "firstBitUnset", "oddBitsSet"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "outerBitsSet", "lastBitSet"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "oddBitsSet", "oddBitsSet"},
|
||||
{intersectInPlaceWrapper, "oddBitsSet", "evenBitsSet", "empty"},
|
||||
//
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "empty", "empty"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "full", "evenBitsSet"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "firstBitSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "lastBitSet", "empty"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "lastBitUnset", "evenBitsSet"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "outerBitsSet", "firstBitSet"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "oddBitsSet", "empty"},
|
||||
{intersectInPlaceWrapper, "evenBitsSet", "evenBitsSet", "evenBitsSet"},
|
||||
|
||||
// union
|
||||
{union, "empty", "empty", "empty"},
|
||||
{union, "empty", "full", "full"},
|
||||
|
|
|
|||
|
|
@ -481,6 +481,288 @@ func TestBitmap_Intersection(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectionInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 2683177)
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
for i := uint64(628); i < 2683301; i++ {
|
||||
if _, err := bm1.Add(i); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 1 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectionInPlace_Empty(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 2683177)
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 0 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
bm1.IntersectInPlace(roaring.NewFileBitmap(0, 2683177))
|
||||
if n := bm1.Count(); n != 0 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectArrayArrayInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005)
|
||||
bm1 := roaring.NewFileBitmap(0, 2683, 2684, 5000)
|
||||
expected := []uint64{0, 2683}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 2 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm0.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
|
||||
// confirm that it also works going the other way
|
||||
bm1.IntersectInPlace(bm0)
|
||||
if n := bm1.Count(); n != 2 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm1.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectArrayBitmapInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005)
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
for i := uint64(1); i < 65536; i += 3 {
|
||||
if _, err := bm1.Add(i); err != nil {
|
||||
t.Fatalf("adding bits(%d): %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
expected := []uint64{1, 7, 2683, 5005}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
|
||||
if n := bm0.Count(); n != uint64(len(expected)) {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm0.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectArrayRunInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005)
|
||||
bm1 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11, 13)
|
||||
bm1.Optimize() // convert to runs
|
||||
|
||||
expected := []uint64{7, 9, 11}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
|
||||
if n := bm0.Count(); n != uint64(len(expected)) {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm0.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectBitmapBitmapInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap()
|
||||
for i := uint64(0); i < 65536; i += 2 {
|
||||
if _, err := bm0.Add(i); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
for i := uint64(0); i < 65536; i += 3 {
|
||||
if _, err := bm1.Add(i); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 10923 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectBitmapArrayInPlace(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 7, 9, 11, 2683, 5005)
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
for i := uint64(1); i < 65536; i += 3 {
|
||||
if _, err := bm1.Add(i); err != nil {
|
||||
t.Fatalf("adding bits(%d): %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
expected := []uint64{1, 7, 2683, 5005}
|
||||
|
||||
bm1.IntersectInPlace(bm0)
|
||||
|
||||
if n := bm1.Count(); n != uint64(len(expected)) {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm1.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
|
||||
bm11 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11, 13, 2683)
|
||||
bm12 := roaring.NewFileBitmap(6, 7, 10, 13, 2683)
|
||||
|
||||
bm0.IntersectInPlace(bm11, bm12)
|
||||
|
||||
expected = []uint64{7, 2683}
|
||||
if n := bm0.Count(); n != uint64(len(expected)) {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm0.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectBitmapRunInPlace(t *testing.T) {
|
||||
// Intersect two runs that result in a bitmap.
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
runLen := uint64(25)
|
||||
spaceLen := uint64(8)
|
||||
offset := (runLen / 2) + spaceLen
|
||||
for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm1.Add(offset + i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
bm1.Optimize() // convert to runs
|
||||
|
||||
bm0 := roaring.NewFileBitmap()
|
||||
runLen = uint64(32)
|
||||
spaceLen = uint64(1)
|
||||
for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm0.Add(i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 47628 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectRunRunInPlace(t *testing.T) {
|
||||
// Intersect two runs that result in an array.
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15)
|
||||
bm0.Optimize() // convert to runs
|
||||
bm1 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11, 13)
|
||||
bm1.Optimize() // convert to runs
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 4 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
// Intersect two runs that result in a bitmap.
|
||||
bm2 := roaring.NewFileBitmap()
|
||||
runLen := uint64(25)
|
||||
spaceLen := uint64(8)
|
||||
offset := (runLen / 2) + spaceLen
|
||||
for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm2.Add(offset + i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
bm2.Optimize() // convert to runs
|
||||
bm3 := roaring.NewFileBitmap()
|
||||
runLen = uint64(32)
|
||||
spaceLen = uint64(1)
|
||||
for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm3.Add(i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
bm3.Optimize() // convert to runs
|
||||
bm2.IntersectInPlace(bm3)
|
||||
if n := bm2.Count(); n != 47628 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectRunArrayInPlace(t *testing.T) {
|
||||
// Intersect two runs that result in an array.
|
||||
bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15)
|
||||
bm0.Optimize() // convert to runs
|
||||
bm1 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11, 13)
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
|
||||
expected := []uint64{5, 10, 11, 13}
|
||||
if n := bm0.Count(); n != uint64(len(expected)) {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
for _, e := range expected {
|
||||
if !bm0.Contains(e) {
|
||||
t.Fatalf("missing value %d", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_IntersectRunBitmapInPlace(t *testing.T) {
|
||||
// Intersect two runs that result in a bitmap.
|
||||
bm0 := roaring.NewFileBitmap()
|
||||
runLen := uint64(25)
|
||||
spaceLen := uint64(8)
|
||||
offset := (runLen / 2) + spaceLen
|
||||
for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm0.Add(offset + i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
bm0.Optimize() // convert to runs
|
||||
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
runLen = uint64(32)
|
||||
spaceLen = uint64(1)
|
||||
for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) {
|
||||
for j := uint64(0); j < runLen; j++ {
|
||||
if _, err := bm1.Add(i + j); err != nil {
|
||||
t.Fatalf("adding bits: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bm0.IntersectInPlace(bm1)
|
||||
if n := bm0.Count(); n != 47628 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Union1(t *testing.T) {
|
||||
bm0 := roaring.NewFileBitmap(0, 2683177)
|
||||
bm1 := roaring.NewFileBitmap()
|
||||
|
|
@ -1193,9 +1475,13 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) }
|
||||
func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000, false) }
|
||||
func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000, false) }
|
||||
func TestBitmap_Marshal_Quick_Array1(t *testing.T) {
|
||||
testBitmapMarshalQuick(t, 1000, 1000, 2000, false)
|
||||
}
|
||||
func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000, false) }
|
||||
func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) {
|
||||
testBitmapMarshalQuick(t, 10000, 0, 10000, false)
|
||||
}
|
||||
func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) {
|
||||
testBitmapMarshalQuick(t, 10000, 10000, 20000, false)
|
||||
}
|
||||
|
|
@ -1403,55 +1689,55 @@ func isAllType(b *roaring.Bitmap, typ string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// getBenchData yields some sample data
|
||||
func getBenchData(tb testing.TB) *benchmarkSampleData {
|
||||
data := &sampleData
|
||||
if data.a1 == nil {
|
||||
// throughout this, we ignore any errors from bitmap adds,
|
||||
// because errors in those should result in the Optimize
|
||||
// pass producing the wrong values, so we can just check there.
|
||||
const max = (1 << 24) / 64
|
||||
func newBenchData(tb testing.TB) *benchmarkSampleData {
|
||||
tb.Helper()
|
||||
|
||||
// Build bitmap with array container.
|
||||
data.a1 = roaring.NewFileBitmap()
|
||||
data.a2 = roaring.NewFileBitmap()
|
||||
// two lists of different lengths
|
||||
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
|
||||
_, _ = data.a1.Add(uint64(rand.Intn(max)))
|
||||
_, _ = data.a2.Add(uint64(rand.Intn(max)))
|
||||
}
|
||||
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
|
||||
_, _ = data.a1.Add(uint64(rand.Intn(max)))
|
||||
}
|
||||
var data benchmarkSampleData
|
||||
|
||||
// Build bitmap with bitmap container.
|
||||
data.b = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal/3; i < n; i++ {
|
||||
_, _ = data.b.Add(uint64(i * 3))
|
||||
}
|
||||
|
||||
// build bitmap with run container
|
||||
data.r1 = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal; i < n; i++ {
|
||||
_, _ = data.r1.Add(uint64(i))
|
||||
}
|
||||
// build bitmap with multiple runs
|
||||
data.r2 = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal; i < n; i++ {
|
||||
_, _ = data.r2.Add(uint64(i))
|
||||
// break the runs up, this should produce 16 runs, which
|
||||
// is small enough to make RLE tempting
|
||||
if i&0xfff == 0xfff {
|
||||
i += 5
|
||||
}
|
||||
}
|
||||
data.a1.Optimize()
|
||||
data.a2.Optimize()
|
||||
data.b.Optimize()
|
||||
data.r1.Optimize()
|
||||
data.r2.Optimize()
|
||||
// throughout this, we ignore any errors from bitmap adds,
|
||||
// because errors in those should result in the Optimize
|
||||
// pass producing the wrong values, so we can just check there.
|
||||
const max = (1 << 24) / 64
|
||||
|
||||
// Build bitmap with array container.
|
||||
data.a1 = roaring.NewFileBitmap()
|
||||
data.a2 = roaring.NewFileBitmap()
|
||||
// two lists of different lengths
|
||||
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
|
||||
_, _ = data.a1.Add(uint64(rand.Intn(max)))
|
||||
_, _ = data.a2.Add(uint64(rand.Intn(max)))
|
||||
}
|
||||
for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ {
|
||||
_, _ = data.a1.Add(uint64(rand.Intn(max)))
|
||||
}
|
||||
|
||||
// Build bitmap with bitmap container.
|
||||
data.b = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal/3; i < n; i++ {
|
||||
_, _ = data.b.Add(uint64(i * 3))
|
||||
}
|
||||
|
||||
// build bitmap with run container
|
||||
data.r1 = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal; i < n; i++ {
|
||||
_, _ = data.r1.Add(uint64(i))
|
||||
}
|
||||
// build bitmap with multiple runs
|
||||
data.r2 = roaring.NewFileBitmap()
|
||||
for i, n := 0, MaxContainerVal; i < n; i++ {
|
||||
_, _ = data.r2.Add(uint64(i))
|
||||
// break the runs up, this should produce 16 runs, which
|
||||
// is small enough to make RLE tempting
|
||||
if i&0xfff == 0xfff {
|
||||
i += 5
|
||||
}
|
||||
}
|
||||
data.a1.Optimize()
|
||||
data.a2.Optimize()
|
||||
data.b.Optimize()
|
||||
data.r1.Optimize()
|
||||
data.r2.Optimize()
|
||||
|
||||
if !isAllType(data.a1, "array") {
|
||||
tb.Fatalf("expected data.a1 to be an array, it wasn't.")
|
||||
}
|
||||
|
|
@ -1467,6 +1753,34 @@ func getBenchData(tb testing.TB) *benchmarkSampleData {
|
|||
if !isAllType(data.r2, "run") {
|
||||
tb.Fatalf("expected data.r2 to be RLE, it wasn't.")
|
||||
}
|
||||
|
||||
return &data
|
||||
}
|
||||
|
||||
// getBenchData yields some sample data
|
||||
func getBenchData(tb testing.TB) *benchmarkSampleData {
|
||||
tb.Helper()
|
||||
|
||||
data := &sampleData
|
||||
if data.a1 == nil {
|
||||
data = newBenchData(tb)
|
||||
} else {
|
||||
if !isAllType(data.a1, "array") {
|
||||
tb.Fatalf("expected data.a1 to be an array, it wasn't.")
|
||||
}
|
||||
if !isAllType(data.a2, "array") {
|
||||
tb.Fatalf("expected data.a2 to be an array, it wasn't.")
|
||||
}
|
||||
if !isAllType(data.b, "bitmap") {
|
||||
tb.Fatalf("expected data.b to be a bitmap, it wasn't.")
|
||||
}
|
||||
if !isAllType(data.r1, "run") {
|
||||
tb.Fatalf("expected data.r1 to be RLE, it wasn't.")
|
||||
}
|
||||
if !isAllType(data.r2, "run") {
|
||||
tb.Fatalf("expected data.r2 to be RLE, it wasn't.")
|
||||
}
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
|
|
@ -1726,23 +2040,11 @@ func BenchmarkSliceDescendingStriped(b *testing.B) {
|
|||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnion(b *testing.B) {
|
||||
data := getBenchData(b)
|
||||
for n := 0; n < b.N; n++ {
|
||||
data.a1.
|
||||
Union(data.a2).
|
||||
Union(data.b).
|
||||
Union(data.r1).
|
||||
Union(data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnionBulk(b *testing.B) {
|
||||
data := getBenchData(b)
|
||||
for n := 0; n < b.N; n++ {
|
||||
bm := roaring.NewBitmap()
|
||||
bm.
|
||||
UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1789,13 +2091,58 @@ func TestBitmap_DifferenceInPlace(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func BenchmarkDifferencInPlace(b *testing.B) {
|
||||
data := getBenchData(b)
|
||||
func BenchmarkUnion(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := roaring.NewBitmap()
|
||||
bm.
|
||||
UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
|
||||
bm.
|
||||
DifferenceInPlace(data.a1, data.r2, data.b, data.r1)
|
||||
bm.Union(data.a1).Union(data.a2).Union(data.b).Union(data.r1).Union(data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnionInPlace(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDifference(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
|
||||
bm.Difference(data.r2).Difference(data.b).Difference(data.r1)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDifferenceInPlace(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
|
||||
bm.DifferenceInPlace(data.r2, data.b, data.r1)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIntersect(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
|
||||
bm.Intersect(data.a2).Intersect(data.b).Intersect(data.r1).Intersect(data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIntersectInPlace(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
data := getBenchData(b)
|
||||
bm := data.a1.Clone()
|
||||
bm.UnionInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
|
||||
bm.IntersectInPlace(data.a2, data.b, data.r1, data.r2)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue