diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index fe357d933..db5c9946d 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -93,8 +93,8 @@ type updater struct { mapped bool } -func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - a := updater{key, int32(n), containerType, mapped} +func (btc *bTreeContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) { + a := updater{key, int32(n), typ, mapped} btc.tree.Put(key, a.update) } @@ -111,7 +111,7 @@ func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container { btc.lastKey = key v, ok := btc.tree.Get(key) if !ok { - cont := roaring.NewContainer() + cont := roaring.NewContainerArray(nil) btc.tree.Set(key, cont) btc.lastContainer = cont return cont diff --git a/roaring/container_slice.go b/roaring/container_slice.go new file mode 100644 index 000000000..9cd1a7d1a --- /dev/null +++ b/roaring/container_slice.go @@ -0,0 +1,182 @@ +package roaring + +import ( + "unsafe" +) + +// Container represents a Container for uint16 integers. +// +// These are used for storing the low bits of numbers in larger sets of uint64. +// The high bits are stored in a Container's key which is tracked by a separate +// data structure. Integers in a Container can be encoded in one of three ways - +// the encoding used is usually whichever is most compact, though any Container +// type should be able to encode any set of integers safely. For containers with +// less than 4,096 values, an array is often used. Containers with long runs of +// integers would use run length encoding, and more random data usually uses +// bitmap encoding. +type Container struct { + oneSlice []uint16 // and here is where the magic happens + n int32 // number of integers in container + mapped bool // mapped directly to a byte slice when true + typ byte // array, bitmap, or run +} + +// NewContainer returns a new instance of container. This trivial function +// may later become more interesting. +func NewContainer() *Container { + statsHit("NewContainer") + return &Container{typ: containerArray} +} + +// NewContainerBitmap makes a bitmap container using the provided bitmap, or +// an empty one if provided bitmap is nil. If the provided bitmap is too short, +// it will be padded. +func NewContainerBitmap(n int32, bitmap []uint64) *Container { + if bitmap == nil { + bitmap = make([]uint64, bitmapN) + } + // pad to required length + if len(bitmap) < bitmapN { + bm2 := make([]uint64, bitmapN) + copy(bm2, bitmap) + bitmap = bm2 + } + return &Container{typ: containerBitmap, n: n, oneSlice: *(*[]uint16)(unsafe.Pointer(&bitmap))} +} + +// NewContainerArray returns an array using the provided set of values. It's +// okay if the slice is nil; that's a length of zero. +func NewContainerArray(set []uint16) *Container { + return &Container{typ: containerArray, n: int32(len(set)), oneSlice: *(*[]uint16)(unsafe.Pointer(&set))} + +} + +// NewContainerRun creates a new run array using a provided (possibly nil) +// slice of intervals. +func NewContainerRun(set []interval16) *Container { + c := &Container{typ: containerRun, oneSlice: *(*[]uint16)(unsafe.Pointer(&set))} + for _, run := range set { + c.n += int32(run.last-run.start) + 1 + } + return c +} + +// array yields the data viewed as a slice of intervals. +func (c *Container) array() []uint16 { + if roaringParanoia { + if c.typ != containerArray { + panic("attempt to read non-array's array") + } + } + return *(*[]uint16)(unsafe.Pointer(&c.oneSlice)) +} + +// setArray stores a set of uint16s as data. +func (c *Container) setArray(array []uint16) { + if roaringParanoia { + if c.typ != containerArray { + panic("attempt to write non-array's array") + } + } + c.oneSlice = *(*[]uint16)(unsafe.Pointer(&array)) +} + +// bitmap yields the data viewed as a slice of uint64s holding bits. +func (c *Container) bitmap() []uint64 { + if roaringParanoia { + if c.typ != containerBitmap { + panic("attempt to read non-bitmap's bitmap") + } + } + return *(*[]uint64)(unsafe.Pointer(&c.oneSlice)) +} + +// setBitmap stores a set of uint64s as data. +func (c *Container) setBitmap(bitmap []uint64) { + if roaringParanoia { + if c.typ != containerBitmap { + panic("attempt to write non-bitmap's bitmap") + } + } + c.oneSlice = *(*[]uint16)(unsafe.Pointer(&bitmap)) +} + +// runs yields the data viewed as a slice of intervals. +func (c *Container) runs() []interval16 { + if roaringParanoia { + if c.typ != containerRun { + panic("attempt to read non-run's runs") + } + } + return *(*[]interval16)(unsafe.Pointer(&c.oneSlice)) +} + +// setRuns stores a set of intervals as data. +func (c *Container) setRuns(runs []interval16) { + if roaringParanoia { + if c.typ != containerRun { + panic("attempt to write non-run's runs") + } + } + c.oneSlice = *(*[]uint16)(unsafe.Pointer(&runs)) +} + +// Mapped returns true if the container is mapped directly to a byte slice +func (c *Container) Mapped() bool { + return c.mapped +} + +// N returns the cached bit count of the container +func (c *Container) N() int32 { + return c.n +} + +// Update updates the container +func (c *Container) Update(typ byte, n int32, mapped bool) { + c.typ = typ + c.n = n + c.mapped = mapped + // we don't know that any existing slice is usable, so let's ditch it + c.oneSlice = nil +} + +// isArray returns true if the container is an array container. +func (c *Container) isArray() bool { + return c.typ == containerArray +} + +// isBitmap returns true if the container is a bitmap container. +func (c *Container) isBitmap() bool { + return c.typ == containerBitmap +} + +// isRun returns true if the container is a run-length-encoded container. +func (c *Container) isRun() bool { + return c.typ == containerRun +} + +// unmap creates copies of the containers data in the heap. +// +// This is performed when altering the container since its contents could be +// pointing at a read-only mmap. +func (c *Container) unmap() { + if !c.mapped { + return + } + + switch c.typ { + case containerArray: + tmp := make([]uint16, len(c.array())) + copy(tmp, c.array()) + c.setArray(tmp) + case containerBitmap: + tmp := make([]uint64, len(c.bitmap())) + copy(tmp, c.bitmap()) + c.setBitmap(tmp) + case containerRun: + tmp := make([]interval16, len(c.runs())) + copy(tmp, c.runs()) + c.setRuns(tmp) + } + c.mapped = false +} diff --git a/roaring/containers.go b/roaring/containers.go index b9928823a..a2f7b2045 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -46,17 +46,17 @@ func (sc *sliceContainers) Put(key uint64, c *Container) { } -func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (sc *sliceContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) { i := search64(sc.keys, key) if i < 0 { c := NewContainer() - c.containerType = containerType + c.typ = typ c.n = int32(n) c.mapped = mapped sc.insertAt(key, c, -i-1) } else { c := sc.containers[i] - c.containerType = containerType + c.typ = typ c.n = int32(n) c.mapped = mapped } @@ -93,7 +93,7 @@ func (sc *sliceContainers) GetOrCreate(key uint64) *Container { sc.lastKey = key i := search64(sc.keys, key) if i < 0 { - c := NewContainer() + c := NewContainerArray(nil) sc.insertAt(key, c, -i-1) sc.lastContainer = c return c diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 95066fe03..b6b1ec6ab 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -72,24 +72,26 @@ func (btc *bTreeContainers) Put(key uint64, c *Container) { func (u updater) update(oldV *Container, exists bool) (*Container, bool) { // update the existing container if exists { - oldV.Update(u.containerType, u.n, u.mapped) + oldV.Update(u.typ, u.n, u.mapped) return oldV, false } cont := NewContainer() - cont.Update(u.containerType, u.n, u.mapped) + cont.typ = u.typ + cont.n = u.n + cont.mapped = u.mapped return cont, true } // this struct is added to prevent the closure locals from being escaped out to the heap type updater struct { - key uint64 - n int32 - containerType byte - mapped bool + key uint64 + n int32 + typ byte + mapped bool } -func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - a := updater{key, int32(n), containerType, mapped} +func (btc *bTreeContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) { + a := updater{key, int32(n), typ, mapped} btc.tree.Put(key, a.update) } @@ -106,7 +108,7 @@ func (btc *bTreeContainers) GetOrCreate(key uint64) *Container { btc.lastKey = key v, ok := btc.tree.Get(key) if !ok { - cont := NewContainer() + cont := NewContainerArray(nil) btc.tree.Set(key, cont) btc.lastContainer = cont return cont diff --git a/roaring/containers_test.go b/roaring/containers_test.go index ea3106a96..9c8bedd22 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -32,8 +32,8 @@ func testContainersIterator(cs Containers, t *testing.T) { t.Fatal("Next() should be false for empty btc") } - cs.Put(1, &Container{n: 1}) - cs.Put(2, &Container{n: 2}) + cs.Put(1, NewContainerArray([]uint16{1})) + cs.Put(2, NewContainerArray([]uint16{1, 2})) itr, found = cs.Iterator(0) if found { @@ -57,9 +57,9 @@ func testContainersIterator(cs Containers, t *testing.T) { t.Fatalf("itr should be done, but got true") } - cs.Put(3, &Container{n: 3}) - cs.Put(5, &Container{n: 5}) - cs.Put(6, &Container{n: 6}) + cs.Put(3, NewContainerArray([]uint16{1, 2, 3})) + cs.Put(5, NewContainerArray([]uint16{1, 2, 3, 4, 5})) + cs.Put(6, NewContainerArray([]uint16{1, 2, 3, 4, 5, 6})) itr, found = cs.Iterator(3) if !itr.Next() { diff --git a/roaring/roaring.go b/roaring/roaring.go index f12d3fc38..2eecac0e2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -52,15 +52,6 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - // containerArray indicates a container of bit position values - containerArray = byte(1) - - // containerBitmap indicates a container of bits packed in a uint64 array block - containerBitmap = byte(2) - - // containerRun indicates a container of run encoded bits - containerRun = byte(3) - maxContainerVal = 0xffff // maxContainerKey is the key representing the last container in a full row. @@ -68,6 +59,19 @@ const ( maxContainerKey = (1 << 48) - 1 ) +const ( + containerArray byte = iota + 1 // slice of bit position values + containerBitmap // slice of 1024 uint64s + containerRun // container of run-encoded bits +) + +// map used for a more descriptive print +var containerTypeNames = map[byte]string{ + containerArray: "array", + containerBitmap: "bitmap", + containerRun: "run", +} + type Containers interface { // Get returns nil if the key does not exist. Get(key uint64) *Container @@ -78,7 +82,7 @@ type Containers interface { // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. // TODO(2.0) make n int32 - PutContainerValues(key uint64, containerType byte, n int, mapped bool) + PutContainerValues(key uint64, typ byte, n int, mapped bool) // Remove takes the container at key out. Remove(key uint64) @@ -695,11 +699,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // range that a container can store, so instead of calculating a // union we can generate an RLE container that represents the entire // range. - tContainer = &Container{ - runs: []interval16{{start: 0, last: maxContainerVal}}, - containerType: containerRun, - n: maxContainerVal + 1, - } + tContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}) target.Containers.Put(iKey, tContainer) bitmapIters.markItersWithKeyAsHandled(i, iKey) continue @@ -726,11 +726,11 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // first other container, but for some cases, that will // result in cloning a non-bitmap, then converting it // to a bitmap, and this will be expensive... - if expectedN >= 512 && iContainer.containerType != containerBitmap { + if expectedN >= 512 && iContainer.typ != containerBitmap { // copying the non-bitmap, then converting it, // is expensive. statsHit("unionInPlace/newBitmap") - tContainer = &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + tContainer = NewContainerBitmap(0, nil) itersToUnion = bitmapIters[i:] } else { // either N will be small or iContainer is a @@ -745,9 +745,9 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // convert it preemptively, because union into a // bitmap is nearly always faster. itersToUnion = bitmapIters[i:] - if expectedN >= 512 && tContainer.containerType != containerBitmap { + if expectedN >= 512 && tContainer.typ != containerBitmap { statsHit("unionInPlace/convertToBitmap") - switch tContainer.containerType { + switch tContainer.typ { case containerArray: tContainer.arrayToBitmap() case containerRun: @@ -866,8 +866,7 @@ func (b *Bitmap) Shift(n int) (*Bitmap, error) { // As long as the carry wasn't from the max container, // append a new container and add the carried bit. if lastCarry && lastKey != maxContainerKey { - extra := NewContainer() - extra.add(0) + extra := NewContainerArray([]uint16{0}) output.Containers.Put(lastKey+1, extra) } @@ -943,6 +942,13 @@ func (ew *errWriter) WriteUint64(b []byte, v uint64) { // WriteTo writes b to w. func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { b.Optimize() + return b.writeToUnoptimized(w) +} + +// writeToUnoptimized is a WriteTo without the Optimize path. We need +// this because otherwise we can't do some of our marshal/unmarshal tests +// safely. +func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { // Remove empty containers before persisting. //b.removeEmptyContainers() @@ -953,7 +959,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { byte8 := make([]byte, 8) // Build header before writing individual container blocks. - // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(containerType)+sizeof(cardinality) + sizeof(file offset) + // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(type)+sizeof(cardinality) + sizeof(file offset) + // Type is stored as 2 bytes, even though it's only got values 1..3. // Cookie header section. ew := &errWriter{ w: w, @@ -974,7 +981,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) if c.n > 0 { ew.WriteUint64(byte8, key) - ew.WriteUint16(byte2, uint16(c.containerType)) + ew.WriteUint16(byte2, uint16(c.typ)) ew.WriteUint16(byte2, uint16(c.n-1)) } @@ -1058,23 +1065,17 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.containerType { + switch c.typ { case containerRun: - c.array = nil - c.bitmap = nil runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) - c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount] - opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size + c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]) + opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size case containerArray: - c.runs = nil - c.bitmap = nil - c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] - opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]) + opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) case containerBitmap: - c.array = nil - c.runs = nil - c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] - opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64) + c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]) + opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64) } } @@ -1223,11 +1224,11 @@ func (itr *Iterator) Seek(seek uint64) { lb := lowbits(seek) if itr.c.isArray() { // Find index in the container. - itr.j = search32(itr.c.array, lb) + itr.j = search32(itr.c.array(), lb) if itr.j < 0 { itr.j = -itr.j - 1 } - if itr.j < int32(len(itr.c.array)) { + if itr.j < int32(len(itr.c.array())) { itr.j-- return } @@ -1247,10 +1248,10 @@ func (itr *Iterator) Seek(seek uint64) { itr.j, itr.k = 0, -1 } - j, contains := binSearchRuns(lb, itr.c.runs) + j, contains := binSearchRuns(lb, itr.c.runs()) if contains { itr.j = j - itr.k = int32(lb) - int32(itr.c.runs[j].start) - 1 + itr.k = int32(lb) - int32(itr.c.runs()[j].start) - 1 } else { // Set iterator to next value in the Bitmap. itr.j = j @@ -1300,7 +1301,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } // If the container is empty, move to the next container. - if len(itr.c.runs) == 0 { + if len(itr.c.runs()) == 0 { if !itr.citer.Next() { itr.c = nil return 0, true @@ -1310,7 +1311,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { continue } - r := itr.c.runs[itr.j] + r := itr.c.runs()[itr.j] runLength := int32(r.last - r.start) if itr.k >= runLength { @@ -1318,7 +1319,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { itr.j, itr.k = itr.j+1, -1 } - if itr.j >= int32(len(itr.c.runs)) { + if itr.j >= int32(len(itr.c.runs())) { // Reached end of runs, move to the next container. if !itr.citer.Next() { itr.c = nil @@ -1339,7 +1340,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Find first non-zero bit in current bitmap, if possible. hb := itr.j >> 6 - if hb >= int32(len(itr.c.bitmap)) { + if hb >= int32(len(itr.c.bitmap())) { if !itr.citer.Next() { itr.c = nil return 0, true @@ -1348,16 +1349,16 @@ func (itr *Iterator) Next() (v uint64, eof bool) { itr.j = -1 continue } - lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64) + lb := itr.c.bitmap()[hb] >> (uint(itr.j) % 64) if lb != 0 { itr.j = itr.j + int32(trailingZeroN(lb)) return itr.peek(), false } // Otherwise iterate through remaining bitmaps to find next bit. - for hb++; hb < int32(len(itr.c.bitmap)); hb++ { - if itr.c.bitmap[hb] != 0 { - itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap[hb])) + for hb++; hb < int32(len(itr.c.bitmap())); hb++ { + if itr.c.bitmap()[hb] != 0 { + itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap()[hb])) return itr.peek(), false } } @@ -1378,10 +1379,10 @@ func (itr *Iterator) peek() uint64 { return 0 } if itr.c.isArray() { - return itr.key<<16 | uint64(itr.c.array[itr.j]) + return itr.key<<16 | uint64(itr.c.array()[itr.j]) } if itr.c.isRun() { - return itr.key<<16 | uint64(itr.c.runs[itr.j].start+uint16(itr.k)) + return itr.key<<16 | uint64(itr.c.runs()[itr.j].start+uint16(itr.k)) } return itr.key<<16 | uint64(itr.j) } @@ -1392,25 +1393,6 @@ const ArrayMaxSize = 4096 // runMaxSize represents the maximum size of run length encoded containers. const runMaxSize = 2048 -// Container represents a Container for uint16 integers. -// -// These are used for storing the low bits of numbers in larger sets of uint64. -// The high bits are stored in a Container's key which is tracked by a separate -// data structure. Integers in a Container can be encoded in one of three ways - -// the encoding used is usually whichever is most compact, though any Container -// type should be able to encode any set of integers safely. For containers with -// less than 4,096 values, an array is often used. Containers with long runs of -// integers would use run length encoding, and more random data usually uses -// bitmap encoding. -type Container struct { - mapped bool // mapped directly to a byte slice when true - containerType byte // array, bitmap, or run - n int32 // number of integers in container - array []uint16 // used for array containers - bitmap []uint64 // used for bitmap containers - runs []interval16 // used for RLE containers -} - type interval16 struct { start uint16 last uint16 @@ -1421,70 +1403,6 @@ func (iv interval16) runlen() int32 { return 1 + int32(iv.last-iv.start) } -// newContainer returns a new instance of container. -func NewContainer() *Container { - statsHit("NewContainer") - return &Container{containerType: containerArray} -} - -// Mapped returns true if the container is mapped directly to a byte slice -func (c *Container) Mapped() bool { - return c.mapped -} - -// N returns the cached bit count of the container -func (c *Container) N() int32 { - return c.n -} - -// Update updates the container -func (c *Container) Update(containerType byte, n int32, mapped bool) { - c.containerType = containerType - c.n = n - c.mapped = mapped -} - -// isArray returns true if the container is an array container. -func (c *Container) isArray() bool { - return c.containerType == containerArray -} - -// isBitmap returns true if the container is a bitmap container. -func (c *Container) isBitmap() bool { - return c.containerType == containerBitmap -} - -// isRun returns true if the container is a run-length-encoded container. -func (c *Container) isRun() bool { - return c.containerType == containerRun -} - -// unmap creates copies of the containers data in the heap. -// -// This is performed when altering the container since its contents could be -// pointing at a read-only mmap. -func (c *Container) unmap() { - if !c.mapped { - return - } - - switch c.containerType { - case containerArray: - tmp := make([]uint16, len(c.array)) - copy(tmp, c.array) - c.array = tmp - case containerBitmap: - tmp := make([]uint64, len(c.bitmap)) - copy(tmp, c.bitmap) - c.bitmap = tmp - case containerRun: - tmp := make([]interval16, len(c.runs)) - copy(tmp, c.runs) - c.runs = tmp - } - c.mapped = false -} - // count counts all bits in the container. func (c *Container) count() (n int32) { return c.countRange(0, maxContainerVal+1) @@ -1501,9 +1419,10 @@ func (c *Container) countRange(start, end int32) (n int32) { } func (c *Container) arrayCountRange(start, end int32) (n int32) { - i := int32(sort.Search(len(c.array), func(i int) bool { return int32(c.array[i]) >= start })) - for ; i < int32(len(c.array)); i++ { - v := int32(c.array[i]) + array := c.array() + i := int32(sort.Search(len(array), func(i int) bool { return int32(array[i]) >= start })) + for ; i < int32(len(array)); i++ { + v := int32(array[i]) if v >= end { break } @@ -1516,34 +1435,36 @@ func (c *Container) bitmapCountRange(start, end int32) int32 { var n uint64 i, j := start/64, end/64 // Special case when start and end fall in the same word. + bitmap := c.bitmap() if i == j { offi, offj := uint(start%64), uint(64-end%64) - n += popcount((c.bitmap[i] >> offi) << (offj + offi)) + n += popcount((bitmap[i] >> offi) << (offj + offi)) return int32(n) } // Count partial starting word. if off := uint(start) % 64; off != 0 { - n += popcount(c.bitmap[i] >> off) + n += popcount(bitmap[i] >> off) i++ } // Count words in between. for ; i < j; i++ { - n += popcount(c.bitmap[i]) + n += popcount(bitmap[i]) } // Count partial ending word. - if j < int32(len(c.bitmap)) { + if j < int32(len(bitmap)) { off := 64 - (uint(end) % 64) - n += popcount(c.bitmap[j] << off) + n += popcount(bitmap[j] << off) } return int32(n) } func (c *Container) runCountRange(start, end int32) (n int32) { - for _, iv := range c.runs { + runs := c.runs() + for _, iv := range runs { // iv is before range if int32(iv.last) < start { continue @@ -1590,15 +1511,17 @@ func (c *Container) add(v uint16) (added bool) { func (c *Container) arrayAdd(v uint16) bool { // Optimize appending to the end of an array container. - if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v { + array := c.array() + if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && array[c.n-1] < v { statsHit("arrayAdd/append") c.unmap() - c.array = append(c.array, v) + array = append(c.array(), v) + c.setArray(array) return true } // Find index of the integer in the container. Exit if it already exists. - i := search32(c.array, v) + i := search32(array, v) if i >= 0 { return false } @@ -1614,9 +1537,10 @@ func (c *Container) arrayAdd(v uint16) bool { statsHit("arrayAdd/insert") c.unmap() i = -i - 1 - c.array = append(c.array, 0) - copy(c.array[i+1:], c.array[i:]) - c.array[i] = v + array = append(c.array(), 0) + copy(array[i+1:], array[i:]) + array[i] = v + c.setArray(array) return true } @@ -1626,53 +1550,58 @@ func (c *Container) bitmapAdd(v uint16) bool { return false } c.unmap() - c.bitmap[v/64] |= (1 << uint64(v%64)) + c.bitmap()[v/64] |= (1 << uint64(v%64)) return true } func (c *Container) runAdd(v uint16) bool { - if len(c.runs) == 0 { + runs := c.runs() + + if len(runs) == 0 { c.unmap() - c.runs = []interval16{{start: v, last: v}} + c.setRuns([]interval16{{start: v, last: v}}) return true } - i := sort.Search(len(c.runs), - func(i int) bool { return c.runs[i].last >= v }) + i := sort.Search(len(runs), + func(i int) bool { return runs[i].last >= v }) - if i == len(c.runs) { + if i == len(runs) { i-- } - iv := c.runs[i] + iv := runs[i] if v >= iv.start && iv.last >= v { return false } c.unmap() + runs = c.runs() if iv.last < v { if iv.last == v-1 { - c.runs[i].last++ + runs[i].last++ } else { - c.runs = append(c.runs, interval16{start: v, last: v}) + runs = append(runs, interval16{start: v, last: v}) } } else if v+1 == iv.start { // combining two intervals - if i > 0 && c.runs[i-1].last == v-1 { - c.runs[i-1].last = iv.last - c.runs = append(c.runs[:i], c.runs[i+1:]...) + if i > 0 && runs[i-1].last == v-1 { + runs[i-1].last = iv.last + runs = append(runs[:i], runs[i+1:]...) + c.setRuns(runs) return true } // just before an interval - c.runs[i].start-- - } else if i > 0 && v-1 == c.runs[i-1].last { + runs[i].start-- + } else if i > 0 && v-1 == runs[i-1].last { // just after an interval - c.runs[i-1].last++ + runs[i-1].last++ } else { // alone newIv := interval16{start: v, last: v} - c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...) + runs = append(runs[:i], append([]interval16{newIv}, runs[i:]...)...) } + c.setRuns(runs) return true } @@ -1688,18 +1617,22 @@ func (c *Container) Contains(v uint16) bool { } func (c *Container) bitmapCountRuns() (r int32) { + return bitmapCountRuns(c.bitmap()) +} + +func bitmapCountRuns(bitmap []uint64) (r int32) { for i := 0; i < 1023; i++ { - v, v1 := c.bitmap[i], c.bitmap[i+1] + v, v1 := bitmap[i], bitmap[i+1] r = r + int32(popcount((v<<1)&^v)+((v>>63)&^v1)) } - vl := c.bitmap[len(c.bitmap)-1] + vl := bitmap[len(bitmap)-1] r = r + int32(popcount((vl<<1)&^vl)+vl>>63) return r } -func (c *Container) arrayCountRuns() (r int32) { +func arrayCountRuns(array []uint16) (r int32) { prev := int32(-2) - for _, v := range c.array { + for _, v := range array { if prev+1 != int32(v) { r++ } @@ -1708,13 +1641,17 @@ func (c *Container) arrayCountRuns() (r int32) { return r } +func (c *Container) arrayCountRuns() (r int32) { + return arrayCountRuns(c.array()) +} + func (c *Container) countRuns() (r int32) { if c.isArray() { return c.arrayCountRuns() } else if c.isBitmap() { return c.bitmapCountRuns() } else if c.isRun() { - return int32(len(c.runs)) + return int32(len(c.runs())) } // sure hope this never happens @@ -1777,9 +1714,9 @@ func (c *Container) optimize() { // to be used when running a sequence of unions, after which you should // call Repair(). (As of this writing, that only matters for bitmaps.) func (c *Container) unionInPlace(other *Container) { - switch c.containerType { + switch c.typ { case containerBitmap: - switch other.containerType { + switch other.typ { case containerBitmap: unionBitmapBitmapInPlace(c, other) case containerArray: @@ -1789,7 +1726,7 @@ func (c *Container) unionInPlace(other *Container) { } case containerArray: - switch other.containerType { + switch other.typ { case containerBitmap: c.arrayToBitmap() unionBitmapBitmapInPlace(c, other) @@ -1800,7 +1737,7 @@ func (c *Container) unionInPlace(other *Container) { unionBitmapRunInPlace(c, other) } case containerRun: - switch other.containerType { + switch other.typ { case containerBitmap: c.runToBitmap() unionBitmapBitmapInPlace(c, other) @@ -1815,11 +1752,11 @@ func (c *Container) unionInPlace(other *Container) { } func (c *Container) arrayContains(v uint16) bool { - return search32(c.array, v) >= 0 + return search32(c.array(), v) >= 0 } func (c *Container) bitmapContains(v uint16) bool { - return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0 + return (c.bitmap()[v/64] & (1 << uint64(v%64))) != 0 } // binSearchRuns returns the index of the run containing v, and true, when v is contained; @@ -1837,7 +1774,7 @@ func binSearchRuns(v uint16, a []interval16) (int32, bool) { // runContains determines if v is in the container assuming c is a run // container. func (c *Container) runContains(v uint16) bool { - _, found := binSearchRuns(v, c.runs) + _, found := binSearchRuns(v, c.runs()) return found } @@ -1850,20 +1787,20 @@ func (c *Container) remove(v uint16) (removed bool) { } else { removed = c.bitmapRemove(v) } - if removed { - c.n-- - } return removed } func (c *Container) arrayRemove(v uint16) bool { - i := search32(c.array, v) + array := c.array() + i := search32(array, v) if i < 0 { return false } c.unmap() - c.array = append(c.array[:i], c.array[i+1:]...) + array = append(array[:i], array[i+1:]...) + c.n-- + c.setArray(array) return true } @@ -1874,8 +1811,8 @@ func (c *Container) bitmapRemove(v uint16) bool { c.unmap() // Lower count and remove element. - // c.n-- // TODO removed this - test it - c.bitmap[v/64] &^= (uint64(1) << uint(v%64)) + c.bitmap()[v/64] &^= (uint64(1) << uint(v%64)) + c.n-- // Convert to array if we go below the threshold. if c.n == ArrayMaxSize { @@ -1887,22 +1824,28 @@ func (c *Container) bitmapRemove(v uint16) bool { // runRemove removes v from a run container, and returns true if v was removed. func (c *Container) runRemove(v uint16) bool { - i, contains := binSearchRuns(v, c.runs) + runs := c.runs() + i, contains := binSearchRuns(v, runs) if !contains { return false } c.unmap() - if v == c.runs[i].last && v == c.runs[i].start { - c.runs = append(c.runs[:i], c.runs[i+1:]...) - } else if v == c.runs[i].last { - c.runs[i].last-- - } else if v == c.runs[i].start { - c.runs[i].start++ - } else if v > c.runs[i].start { - last := c.runs[i].last - c.runs[i].last = v - 1 - c.runs = append(c.runs[:i+1], append([]interval16{{start: v + 1, last: last}}, c.runs[i+1:]...)...) + if v == runs[i].last && v == runs[i].start { + runs = append(runs[:i], runs[i+1:]...) + } else if v == runs[i].last { + runs[i].last-- + } else if v == runs[i].start { + runs[i].start++ + } else if v > runs[i].start { + last := runs[i].last + runs[i].last = v - 1 + runs = append(runs, interval16{}) + copy(runs[i+2:], runs[i+1:]) + runs[i+1] = interval16{start: v + 1, last: last} + // runs = append(runs[:i+1], append([]interval16{{start: v + 1, last: last}}, runs[i+1:]...)...) } + c.n-- + c.setRuns(runs) return true } @@ -1918,17 +1861,19 @@ func (c *Container) max() uint16 { } func (c *Container) arrayMax() uint16 { - if len(c.array) == 0 { + array := c.array() + if len(array) == 0 { return 0 // probably hiding some ugly bug but it prevents a crash } - return c.array[len(c.array)-1] + return array[len(array)-1] } func (c *Container) bitmapMax() uint16 { // Search bitmap in reverse order. - for i := len(c.bitmap); i > 0; i-- { + bitmap := c.bitmap() + for i := len(bitmap); i > 0; i-- { // If value is zero then skip. - v := c.bitmap[i-1] + v := bitmap[i-1] if v != 0 { r := bits.LeadingZeros64(v) return uint16((i-1)*64 + 63 - r) @@ -1939,102 +1884,117 @@ func (c *Container) bitmapMax() uint16 { } func (c *Container) runMax() uint16 { - if len(c.runs) == 0 { + runs := c.runs() + if len(runs) == 0 { return 0 } - return c.runs[len(c.runs)-1].last + return runs[len(runs)-1].last } // bitmapToArray converts from bitmap format to array format. func (c *Container) bitmapToArray() { statsHit("bitmapToArray") - c.array = make([]uint16, 0, c.n) - c.containerType = containerArray + bitmap := c.bitmap() + c.setBitmap(nil) + c.typ = containerArray + array := make([]uint16, c.n) + c.setArray(array) + c.mapped = false // return early if empty if c.n == 0 { - c.bitmap = nil - c.mapped = false return } + n := int32(0) - for i, bitmap := range c.bitmap { - for bitmap != 0 { - t := bitmap & -bitmap - c.array = append(c.array, uint16((i*64 + int(popcount(t-1))))) - bitmap ^= t + for i, word := range bitmap { + for word != 0 { + t := word & -word + if roaringParanoia { + if n >= c.n { + panic("bitmap has more bits set than container.n") + } + } + array[n] = uint16((i*64 + int(popcount(t-1)))) + n++ + word ^= t + } + } + if roaringParanoia { + if n != c.n { + panic("bitmap has fewer bits set than container.n") } } - c.bitmap = nil - c.mapped = false } // arrayToBitmap converts from array format to bitmap format. func (c *Container) arrayToBitmap() { statsHit("arrayToBitmap") - c.bitmap = make([]uint64, bitmapN) - c.containerType = containerBitmap + array := c.array() + c.setArray(nil) + c.typ = containerBitmap + bitmap := make([]uint64, bitmapN) + c.setBitmap(bitmap) + c.mapped = false // return early if empty if c.n == 0 { - c.array = nil - c.mapped = false return } - for _, v := range c.array { - c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64)) + for _, v := range array { + bitmap[int(v)/64] |= (uint64(1) << uint(v%64)) } - c.array = nil - c.mapped = false } // runToBitmap converts from RLE format to bitmap format. func (c *Container) runToBitmap() { statsHit("runToBitmap") - c.bitmap = make([]uint64, bitmapN) - c.containerType = containerBitmap + runs := c.runs() + c.setRuns(nil) + bitmap := make([]uint64, bitmapN) + c.typ = containerBitmap + c.setBitmap(bitmap) + + c.mapped = false // return early if empty if c.n == 0 { - c.runs = nil - c.mapped = false return } - for _, r := range c.runs { + for _, r := range runs { // TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits //note v must be int or will overflow for v := int(r.start); v <= int(r.last); v++ { - c.bitmap[v/64] |= (uint64(1) << uint(v%64)) + bitmap[v/64] |= (uint64(1) << uint(v%64)) } } - c.runs = nil - c.mapped = false } // bitmapToRun converts from bitmap format to RLE format. func (c *Container) bitmapToRun() { statsHit("bitmapToRun") - c.containerType = containerRun + bitmap := c.bitmap() + c.setBitmap(nil) + c.mapped = false + c.typ = containerRun // return early if empty if c.n == 0 { - c.runs = make([]interval16, 0) - c.bitmap = nil - c.mapped = false + c.setRuns(make([]interval16, 0)) return } - numRuns := c.bitmapCountRuns() - c.runs = make([]interval16, 0, numRuns) + numRuns := bitmapCountRuns(bitmap) + runs := make([]interval16, 0, numRuns) - current := c.bitmap[0] + current := bitmap[0] var i, start, last uint16 for { // skip while empty for current == 0 && i < bitmapN-1 { i++ - current = c.bitmap[i] + current = bitmap[i] } if current == 0 { @@ -2049,97 +2009,105 @@ func (c *Container) bitmapToRun() { // find next 0 for current == maxBitmap && i < bitmapN-1 { i++ - current = c.bitmap[i] + current = bitmap[i] } if current == maxBitmap { // bitmap[1023] == maxBitmap - c.runs = append(c.runs, interval16{start, maxContainerVal}) + runs = append(runs, interval16{start, maxContainerVal}) break } currentLast := uint16(trailingZeroN(^current)) last = 64*i + currentLast - c.runs = append(c.runs, interval16{start, last - 1}) + runs = append(runs, interval16{start, last - 1}) // pad LSBs with 0s current = current & (current + 1) } - - c.bitmap = nil - c.mapped = false + c.setRuns(runs) } // arrayToRun converts from array format to RLE format. func (c *Container) arrayToRun() { statsHit("arrayToRun") - c.containerType = containerRun + array := c.array() + c.setArray(nil) + c.typ = containerRun + c.mapped = false // return early if empty if c.n == 0 { - c.runs = make([]interval16, 0) - c.array = nil - c.mapped = false + c.setRuns(make([]interval16, 0)) return } - numRuns := c.arrayCountRuns() - c.runs = make([]interval16, 0, numRuns) - start := c.array[0] - for i, v := range c.array[1:] { - if v-c.array[i] > 1 { + numRuns := arrayCountRuns(array) + runs := make([]interval16, 0, numRuns) + start := array[0] + for i, v := range array[1:] { + if v-array[i] > 1 { // if current-previous > 1, one run ends and another begins - c.runs = append(c.runs, interval16{start, c.array[i]}) + runs = append(runs, interval16{start, array[i]}) start = v } } // append final run - c.runs = append(c.runs, interval16{start, c.array[c.n-1]}) - c.array = nil - c.mapped = false + runs = append(runs, interval16{start, array[c.n-1]}) + c.setRuns(runs) } // runToArray converts from RLE format to array format. func (c *Container) runToArray() { statsHit("runToArray") - c.containerType = containerArray - c.array = make([]uint16, 0, c.n) + array := make([]uint16, 0, c.n) + runs := c.runs() + c.setRuns(nil) + c.typ = containerArray + c.mapped = false // return early if empty if c.n == 0 { - c.runs = nil - c.mapped = false + c.setArray(array) return } - for _, r := range c.runs { + for _, r := range runs { for v := int(r.start); v <= int(r.last); v++ { - c.array = append(c.array, uint16(v)) + array = append(array, uint16(v)) } } - c.runs = nil - c.mapped = false + c.setArray(array) } // Clone returns a copy of c. -func (c *Container) Clone() *Container { +func (c *Container) Clone() (out *Container) { statsHit("Container/Clone") - other := &Container{n: c.n, containerType: c.containerType} - - switch c.containerType { + switch c.typ { case containerArray: statsHit("Container/Clone/Array") - other.array = make([]uint16, len(c.array)) - copy(other.array, c.array) + cArray := c.array() + array := make([]uint16, len(cArray)) + copy(array, cArray) + out = NewContainerArray(array) case containerBitmap: statsHit("Container/Clone/Bitmap") - other.bitmap = make([]uint64, len(c.bitmap)) - copy(other.bitmap, c.bitmap) + other := NewContainerBitmap(c.n, nil) + copy(other.bitmap(), c.bitmap()) + out = other case containerRun: statsHit("Container/Clone/Run") - other.runs = make([]interval16, len(c.runs)) - copy(other.runs, c.runs) + cRuns := c.runs() + runs := make([]interval16, len(cRuns)) + copy(runs, cRuns) + out = NewContainerRun(runs) } - return other + if roaringParanoia { + if out.n != out.count() { + panic("cloned container has wrong n") + } + } + // this should probably never happen + return out } // WriteTo writes c to w. @@ -2155,7 +2123,8 @@ func (c *Container) WriteTo(w io.Writer) (n int64, err error) { func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { statsHit("Container/arrayWriteTo") - if len(c.array) == 0 { + array := c.array() + if len(array) == 0 { return 0, nil } @@ -2166,40 +2135,42 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { //} // Write sizeof(uint16) * cardinality bytes. - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&array[0]))[:2*c.n]) return int64(nn), err } func (c *Container) bitmapWriteTo(w io.Writer) (n int64, err error) { statsHit("Container/bitmapWriteTo") + bitmap := c.bitmap() // Write sizeof(uint64) * bitmapN bytes. - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)]) + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&bitmap[0]))[:(8 * bitmapN)]) return int64(nn), err } func (c *Container) runWriteTo(w io.Writer) (n int64, err error) { statsHit("Container/runWriteTo") - if len(c.runs) == 0 { + runs := c.runs() + if len(runs) == 0 { return 0, nil } var byte2 [2]byte - binary.LittleEndian.PutUint16(byte2[:], uint16(len(c.runs))) + binary.LittleEndian.PutUint16(byte2[:], uint16(len(runs))) _, err = w.Write(byte2[:]) if err != nil { return 0, err } - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval16Size*len(c.runs)]) + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&runs[0]))[:interval16Size*len(runs)]) return int64(runCountHeaderSize + nn), err } // size returns the encoded size of the container, in bytes. func (c *Container) size() int { if c.isArray() { - return len(c.array) * 2 // sizeof(uint16) + return len(c.array()) * 2 // sizeof(uint16) } else if c.isRun() { - return len(c.runs)*interval16Size + runCountHeaderSize + return len(c.runs())*interval16Size + runCountHeaderSize } else { - return len(c.bitmap) * 8 // sizeof(uint64) + return len(c.bitmap()) * 8 // sizeof(uint64) } } @@ -2209,22 +2180,22 @@ func (c *Container) info() containerInfo { if c.isArray() { info.Type = "array" - info.Alloc = len(c.array) * 2 // sizeof(uint16) + info.Alloc = len(c.array()) * 2 // sizeof(uint16) } else if c.isRun() { info.Type = "run" - info.Alloc = len(c.runs)*interval16Size + runCountHeaderSize + info.Alloc = len(c.runs())*interval16Size + runCountHeaderSize } else { info.Type = "bitmap" - info.Alloc = len(c.bitmap) * 8 // sizeof(uint64) + info.Alloc = len(c.bitmap()) * 8 // sizeof(uint64) } if c.mapped { if c.isArray() { - info.Pointer = unsafe.Pointer(&c.array[0]) + info.Pointer = unsafe.Pointer(&c.array()[0]) } else if c.isRun() { - info.Pointer = unsafe.Pointer(&c.runs[0]) + info.Pointer = unsafe.Pointer(&c.runs()[0]) } else { - info.Pointer = unsafe.Pointer(&c.bitmap[0]) + info.Pointer = unsafe.Pointer(&c.bitmap()[0]) } } @@ -2236,8 +2207,9 @@ func (c *Container) check() error { var a ErrorList if c.isArray() { - if int32(len(c.array)) != c.n { - a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n)) + array := c.array() + if int32(len(array)) != c.n { + a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.n)) } } else if c.isRun() { n := c.runCountRange(0, maxContainerVal+1) @@ -2274,11 +2246,12 @@ func (c *Container) bitmapRepair() { // Manually unroll loop to make it a little faster. // TODO(rartoul): Can probably make this a few x faster using // SIMD instructions. + bitmap := c.bitmap() for i := 0; i < bitmapN; i += 4 { - n += int32(popcount(c.bitmap[i])) - n += int32(popcount(c.bitmap[i+1])) - n += int32(popcount(c.bitmap[i+2])) - n += int32(popcount(c.bitmap[i+3])) + n += int32(popcount(bitmap[i])) + n += int32(popcount(bitmap[i+1])) + n += int32(popcount(bitmap[i+2])) + n += int32(popcount(bitmap[i+3])) } c.n = n } @@ -2314,10 +2287,11 @@ func flipArray(b *Container) *Container { func flipBitmap(b *Container) *Container { statsHit("flipBitmap") - other := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} - - for i, bitmap := range b.bitmap { - other.bitmap[i] = ^bitmap + other := NewContainerBitmap(0, nil) + bitmap := b.bitmap() + otherBitmap := other.bitmap() + for i, word := range bitmap { + otherBitmap[i] = ^word } other.n = other.count() @@ -2362,7 +2336,7 @@ func intersectionCount(a, b *Container) int32 { func intersectionCountArrayArray(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayArray") - ca, cb := a.array, b.array + ca, cb := a.array(), b.array() na, nb := len(ca), len(cb) if na == 0 || nb == 0 { return 0 @@ -2388,9 +2362,10 @@ func intersectionCountArrayArray(a, b *Container) (n int32) { func intersectionCountArrayRun(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayRun") - na, nb := len(a.array), len(b.runs) + array, runs := a.array(), b.runs() + na, nb := len(array), len(runs) for i, j := 0, 0; i < na && j < nb; { - va, vb := a.array[i], b.runs[j] + va, vb := array[i], runs[j] if va < vb.start { i++ } else if va >= vb.start && va <= vb.last { @@ -2405,9 +2380,10 @@ func intersectionCountArrayRun(a, b *Container) (n int32) { func intersectionCountRunRun(a, b *Container) (n int32) { statsHit("intersectionCount/RunRun") - na, nb := len(a.runs), len(b.runs) + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) for i, j := 0, 0; i < na && j < nb; { - va, vb := a.runs[i], b.runs[j] + va, vb := ra[i], rb[j] if va.last < vb.start { // |--va--| |--vb--| i++ @@ -2437,7 +2413,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) { func intersectionCountBitmapRun(a, b *Container) (n int32) { statsHit("intersectionCount/BitmapRun") - for _, iv := range b.runs { + for _, iv := range b.runs() { n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1) } return n @@ -2445,21 +2421,22 @@ func intersectionCountBitmapRun(a, b *Container) (n int32) { func intersectionCountArrayBitmap(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayBitmap") - ln := len(b.bitmap) - for _, val := range a.array { + bitmap := b.bitmap() + ln := len(bitmap) + for _, val := range a.array() { i := int(val >> 6) if i >= ln { break } off := val % 64 - n += int32(b.bitmap[i]>>off) & 1 + n += int32(bitmap[i]>>off) & 1 } return n } func intersectionCountBitmapBitmap(a, b *Container) (n int32) { statsHit("intersectionCount/BitmapBitmap") - return int32(popcountAndSlice(a.bitmap, b.bitmap)) + return int32(popcountAndSlice(a.bitmap(), b.bitmap())) } func intersect(a, b *Container) *Container { @@ -2492,21 +2469,21 @@ func intersect(a, b *Container) *Container { func intersectArrayArray(a, b *Container) *Container { statsHit("intersect/ArrayArray") - output := &Container{containerType: containerArray} - na, nb := len(a.array), len(b.array) + aa, ab := a.array(), b.array() + na, nb := len(aa), len(ab) + output := make([]uint16, 0, na) for i, j := 0, 0; i < na && j < nb; { - va, vb := a.array[i], b.array[j] + va, vb := aa[i], ab[j] if va < vb { i++ } else if va > vb { j++ } else { - output.array = append(output.array, va) + output = append(output, va) i, j = i+1, j+1 } } - output.n = int32(len(output.array)) - return output + return NewContainerArray(output) } // intersectArrayRun computes the intersect of an array container and a run @@ -2514,30 +2491,31 @@ func intersectArrayArray(a, b *Container) *Container { // be low-cardinality) func intersectArrayRun(a, b *Container) *Container { statsHit("intersect/ArrayRun") - output := &Container{containerType: containerArray} - na, nb := len(a.array), len(b.runs) + aa, rb := a.array(), b.runs() + na, nb := len(aa), len(rb) + var output []uint16 for i, j := 0, 0; i < na && j < nb; { - va, vb := a.array[i], b.runs[j] + va, vb := aa[i], rb[j] if va < vb.start { i++ } else if va > vb.last { j++ } else { - output.array = append(output.array, va) + output = append(output, va) i++ } } - output.n = int32(len(output.array)) - return output + return NewContainerArray(output) } // intersectRunRun computes the intersect of two run containers. func intersectRunRun(a, b *Container) *Container { statsHit("intersect/RunRun") - output := &Container{containerType: containerRun} - na, nb := len(a.runs), len(b.runs) + output := NewContainerRun(nil) + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) for i, j := 0, 0; i < na && j < nb; { - va, vb := a.runs[i], b.runs[j] + va, vb := ra[i], rb[j] if va.last < vb.start { // |--va--| |--vb--| i++ @@ -2562,9 +2540,10 @@ func intersectRunRun(a, b *Container) *Container { i++ } } - if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { + runs := output.runs() + if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { output.runToArray() - } else if len(output.runs) > runMaxSize { + } else if len(runs) > runMaxSize { output.runToBitmap() } return output @@ -2575,13 +2554,14 @@ func intersectRunRun(a, b *Container) *Container { func intersectBitmapRun(a, b *Container) *Container { statsHit("intersect/BitmapRun") var output *Container + runs := b.runs() if b.n <= ArrayMaxSize || a.n <= ArrayMaxSize { // output is array container - output = &Container{containerType: containerArray} - for _, iv := range b.runs { + array := make([]uint16, 0, b.n) + for _, iv := range runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { - output.array = append(output.array, i) + array = append(array, i) } // If the run ends the container, break to avoid an infinite loop. if i == 65535 { @@ -2589,38 +2569,38 @@ func intersectBitmapRun(a, b *Container) *Container { } } } - output.n = int32(len(output.array)) + + output = NewContainerArray(array) } else { // right now this iterates through the runs and sets integers in the // bitmap that are in the runs. alternately, we could zero out ranges in // the bitmap which are between runs. - output = &Container{ - bitmap: make([]uint64, bitmapN), - containerType: containerBitmap, - } - for j := 0; j < len(b.runs); j++ { - vb := b.runs[j] + output = NewContainerBitmap(0, nil) + bitmap := output.bitmap() + aBitmap := a.bitmap() + for j := 0; j < len(runs); j++ { + vb := runs[j] i := vb.start >> 6 // index into a vastart := i << 6 valast := vastart + 63 for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b - output.bitmap[i] = a.bitmap[i] - output.n += int32(popcount(a.bitmap[i])) + bitmap[i] = aBitmap[i] + output.n += int32(popcount(aBitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) - bits := a.bitmap[i] & mask - output.bitmap[i] |= bits + bits := aBitmap[i] & mask + bitmap[i] |= bits output.n += int32(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) - bits := (a.bitmap[i] >> offset) << offset - output.bitmap[i] |= bits + bits := (aBitmap[i] >> offset) << offset + bitmap[i] |= bits output.n += int32(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) - bits := (a.bitmap[i] << offset) >> offset - output.bitmap[i] |= bits + bits := (aBitmap[i] << offset) >> offset + bitmap[i] |= bits output.n += int32(popcount(bits)) } // update loop vars @@ -2635,18 +2615,18 @@ func intersectBitmapRun(a, b *Container) *Container { func intersectArrayBitmap(a, b *Container) *Container { statsHit("intersect/ArrayBitmap") - output := &Container{containerType: containerArray} - for _, va := range a.array { + array := make([]uint16, 0) + bBitmap := b.bitmap() + for _, va := range a.array() { bmidx := va / 64 bidx := va % 64 mask := uint64(1) << bidx - b := b.bitmap[bmidx] + b := bBitmap[bmidx] if b&mask > 0 { - output.array = append(output.array, va) + array = append(array, va) } } - output.n = int32(len(output.array)) - return output + return NewContainerArray(array) } func intersectBitmapBitmap(a, b *Container) *Container { @@ -2654,22 +2634,17 @@ func intersectBitmapBitmap(a, b *Container) *Container { // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] - buf = make([]uint64, bitmapN) - ob = buf[:bitmapN] - n int32 + ab = a.bitmap() + bb = b.bitmap() + ob = make([]uint64, bitmapN) + n int32 ) for i := 0; i < bitmapN; i++ { ob[i] = ab[i] & bb[i] n += int32(popcount(ob[i])) } - output := &Container{ - bitmap: ob, - n: n, - containerType: containerBitmap, - } + output := NewContainerBitmap(n, ob) return output } @@ -2703,22 +2678,23 @@ func union(a, b *Container) *Container { func unionArrayArray(a, b *Container) *Container { statsHit("union/ArrayArray") - output := &Container{containerType: containerArray} - na, nb := len(a.array), len(b.array) + output := NewContainerArray(nil) + aa, ab := a.array(), b.array() + na, nb := len(aa), len(ab) for i, j := 0, 0; ; { if i >= na && j >= nb { break } else if i < na && j >= nb { - output.add(a.array[i]) + output.add(aa[i]) i++ continue } else if i >= na && j < nb { - output.add(b.array[j]) + output.add(ab[j]) j++ continue } - va, vb := a.array[i], b.array[j] + va, vb := aa[i], ab[j] if va < vb { output.add(va) i++ @@ -2738,23 +2714,24 @@ func unionArrayArray(a, b *Container) *Container { // of a good array size, so it could be up to twice that size, temporarily. func unionArrayArrayInPlace(a, b *Container) { statsHit("union/ArrayArrayInPlace") - na, nb := len(a.array), len(b.array) + aa, ab := a.array(), b.array() + na, nb := len(aa), len(ab) output := make([]uint16, na+nb) outN := 0 for i, j := 0, 0; ; { if i >= na && j >= nb { break } else if i < na && j >= nb { - copy(output[outN:], a.array[i:]) + copy(output[outN:], aa[i:]) outN += na - i break } else if i >= na && j < nb { - copy(output[outN:], b.array[j:]) + copy(output[outN:], ab[j:]) outN += nb - j break } - va, vb := a.array[i], b.array[j] + va, vb := aa[i], ab[j] if va < vb { output[outN] = va outN++ @@ -2770,7 +2747,7 @@ func unionArrayArrayInPlace(a, b *Container) { j++ } } - a.array = output[:outN] + a.setArray(output[:outN]) a.n = int32(outN) if a.n > ArrayMaxSize { a.optimize() @@ -2784,16 +2761,17 @@ func unionArrayRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.Clone() } - output := &Container{containerType: containerRun} - na, nb := len(a.array), len(b.runs) + output := NewContainerRun(nil) + aa, rb := a.array(), b.runs() + na, nb := len(aa), len(rb) var vb interval16 var va uint16 for i, j := 0, 0; i < na || j < nb; { if i < na { - va = a.array[i] + va = aa[i] } if j < nb { - vb = b.runs[j] + vb = rb[j] } if i < na && (j >= nb || va < vb.start) { output.n += output.runAppendInterval(interval16{start: va, last: va}) @@ -2805,7 +2783,7 @@ func unionArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > runMaxSize { + } else if len(output.runs()) > runMaxSize { output.runToBitmap() } return output @@ -2818,20 +2796,24 @@ func unionArrayRun(a, b *Container) *Container { // Its return value is the amount by which the cardinality of the container was // increased. func (c *Container) runAppendInterval(v interval16) int32 { - if len(c.runs) == 0 { - c.runs = append(c.runs, v) + runs := c.runs() + if len(runs) == 0 { + runs = append(runs, v) + c.setRuns(runs) return int32(v.last-v.start) + 1 } - last := c.runs[len(c.runs)-1] + last := runs[len(runs)-1] if last.last == maxContainerVal { //protect against overflow return 0 } if last.last+1 >= v.start && v.last > last.last { - c.runs[len(c.runs)-1].last = v.last + runs[len(runs)-1].last = v.last + c.setRuns(runs) return int32(v.last - last.last) } else if last.last+1 < v.start { - c.runs = append(c.runs, v) + runs = append(runs, v) + c.setRuns(runs) return int32(v.last-v.start) + 1 } return 0 @@ -2845,18 +2827,16 @@ func unionRunRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.Clone() } - na, nb := len(a.runs), len(b.runs) - output := &Container{ - runs: make([]interval16, 0, na+nb), - containerType: containerRun, - } + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) + output := NewContainerRun(make([]interval16, 0, na+nb)) var va, vb interval16 for i, j := 0, 0; i < na || j < nb; { if i < na { - va = a.runs[i] + va = ra[i] } if j < nb { - vb = b.runs[j] + vb = rb[j] } if i < na && (j >= nb || va.start < vb.start) { output.n += output.runAppendInterval(va) @@ -2866,7 +2846,7 @@ func unionRunRun(a, b *Container) *Container { j++ } } - if len(output.runs) > runMaxSize { + if len(output.runs()) > runMaxSize { output.runToBitmap() } return output @@ -2881,8 +2861,8 @@ func unionBitmapRun(a, b *Container) *Container { return a.Clone() } output := a.Clone() - for j := 0; j < len(b.runs); j++ { - output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + for _, run := range b.runs() { + output.bitmapSetRange(uint64(run.start), uint64(run.last)+1) } return output } @@ -2892,8 +2872,8 @@ func unionBitmapRun(a, b *Container) *Container { func unionBitmapRunInPlace(a, b *Container) { a.unmap() statsHit("union/BitmapRun") - for j := 0; j < len(b.runs); j++ { - a.bitmapSetRangeIgnoreN(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + for _, run := range b.runs() { + a.bitmapSetRangeIgnoreN(uint64(run.start), uint64(run.last)+1) } } @@ -2907,18 +2887,19 @@ func (c *Container) bitmapSetRange(i, j uint64) { var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcount(X) ycnt := popcount(Y) + bitmap := c.bitmap() if x == y { - c.n += int32((j - i) - popcount(c.bitmap[x]&(X&Y))) - c.bitmap[x] |= (X & Y) + c.n += int32((j - i) - popcount(bitmap[x]&(X&Y))) + bitmap[x] |= (X & Y) } else { - c.n += int32(xcnt - popcount(c.bitmap[x]&X)) - c.bitmap[x] |= X + c.n += int32(xcnt - popcount(bitmap[x]&X)) + bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int32(64 - popcount(c.bitmap[i])) - c.bitmap[i] = maxBitmap + c.n += int32(64 - popcount(bitmap[i])) + bitmap[i] = maxBitmap } - c.n += int32(ycnt - popcount(c.bitmap[y]&Y)) - c.bitmap[y] |= Y + c.n += int32(ycnt - popcount(bitmap[y]&Y)) + bitmap[y] |= Y } } @@ -2930,14 +2911,15 @@ func (c *Container) bitmapSetRangeIgnoreN(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + bitmap := c.bitmap() if x == y { - c.bitmap[x] |= (X & Y) + bitmap[x] |= (X & Y) } else { - c.bitmap[x] |= X + bitmap[x] |= X for i := x + 1; i < y; i++ { - c.bitmap[i] = maxBitmap + bitmap[i] = maxBitmap } - c.bitmap[y] |= Y + bitmap[y] |= Y } } @@ -2947,22 +2929,23 @@ func (c *Container) bitmapXorRange(i, j uint64) { y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + bitmap := c.bitmap() if x == y { - cnt := popcount(c.bitmap[x]) - c.bitmap[x] ^= (X & Y) //// flip - c.n += int32(popcount(c.bitmap[x]) - cnt) + cnt := popcount(bitmap[x]) + bitmap[x] ^= (X & Y) //// flip + c.n += int32(popcount(bitmap[x]) - cnt) } else { - cnt := popcount(c.bitmap[x]) - c.bitmap[x] ^= X - c.n += int32(popcount(c.bitmap[x]) - cnt) + cnt := popcount(bitmap[x]) + bitmap[x] ^= X + c.n += int32(popcount(bitmap[x]) - cnt) for i := x + 1; i < y; i++ { - cnt = popcount(c.bitmap[i]) - c.bitmap[i] ^= maxBitmap - c.n += int32(popcount(c.bitmap[i]) - cnt) + cnt = popcount(bitmap[i]) + bitmap[i] ^= maxBitmap + c.n += int32(popcount(bitmap[i]) - cnt) } - cnt = popcount(c.bitmap[y]) - c.bitmap[y] ^= Y - c.n += int32(popcount(c.bitmap[y]) - cnt) + cnt = popcount(bitmap[y]) + bitmap[y] ^= Y + c.n += int32(popcount(bitmap[y]) - cnt) } } @@ -2972,63 +2955,68 @@ func (c *Container) bitmapZeroRange(i, j uint64) { y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + bitmap := c.bitmap() if x == y { - c.n -= int32(popcount(c.bitmap[x] & (X & Y))) - c.bitmap[x] &= ^(X & Y) + c.n -= int32(popcount(bitmap[x] & (X & Y))) + bitmap[x] &= ^(X & Y) } else { - c.n -= int32(popcount(c.bitmap[x] & X)) - c.bitmap[x] &= ^X + c.n -= int32(popcount(bitmap[x] & X)) + bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int32(popcount(c.bitmap[i])) - c.bitmap[i] = 0 + c.n -= int32(popcount(bitmap[i])) + bitmap[i] = 0 } - c.n -= int32(popcount(c.bitmap[y] & Y)) - c.bitmap[y] &= ^Y + c.n -= int32(popcount(bitmap[y] & Y)) + bitmap[y] &= ^Y } } func (c *Container) equals(c2 *Container) bool { - if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n { + if c.mapped != c2.mapped || c.typ != c2.typ || c.n != c2.n { return false } - if c.containerType == containerArray { - if len(c.array) != len(c2.array) { + if c.typ == containerArray { + ca, c2a := c.array(), c2.array() + if len(ca) != len(c2a) { return false } - for i := 0; i < len(c.array); i++ { - if c.array[i] != c2.array[i] { + for i := 0; i < len(ca); i++ { + if ca[i] != c2a[i] { return false } } - } else if c.containerType == containerBitmap { - if len(c.bitmap) != len(c2.bitmap) { + } else if c.typ == containerBitmap { + cb, c2b := c.bitmap(), c2.bitmap() + if len(cb) != len(c2b) { return false } - for i := 0; i < len(c.bitmap); i++ { - if c.bitmap[i] != c2.bitmap[i] { + for i := 0; i < len(cb); i++ { + if cb[i] != c2b[i] { return false } } - } else if c.containerType == containerRun { - if len(c.runs) != len(c2.runs) { + } else if c.typ == containerRun { + cr, c2r := c.runs(), c2.runs() + if len(cr) != len(c2r) { return false } - for i := 0; i < len(c.runs); i++ { - if c.runs[i] != c2.runs[i] { + for i := 0; i < len(cr); i++ { + if cr[i] != c2r[i] { return false } } } else { - panic(fmt.Sprintf("unknown container type: %v", c.containerType)) + panic(fmt.Sprintf("unknown container type: %v", c.typ)) } return true } func unionArrayBitmap(a, b *Container) *Container { output := b.Clone() - for _, v := range a.array { + bitmap := output.bitmap() + for _, v := range a.array() { if !output.bitmapContains(v) { - output.bitmap[v/64] |= (1 << uint64(v%64)) + bitmap[v/64] |= (1 << uint64(v%64)) output.n++ } } @@ -3039,8 +3027,9 @@ func unionArrayBitmap(a, b *Container) *Container { // of a will need to be repaired after the fact. func unionBitmapArrayInPlace(a, b *Container) { a.unmap() - for _, v := range b.array { - a.bitmap[v>>6] |= (uint64(1) << (v % 64)) + bitmap := a.bitmap() + for _, v := range b.array() { + bitmap[v>>6] |= (uint64(1) << (v % 64)) } } @@ -3049,10 +3038,9 @@ func unionBitmapBitmap(a, b *Container) *Container { // see https://go101.org/article/bounds-check-elimination.html var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] - buf = make([]uint64, bitmapN) - ob = buf[:bitmapN] + ab = a.bitmap()[:bitmapN] + bb = b.bitmap()[:bitmapN] + ob = make([]uint64, bitmapN)[:bitmapN] n int32 ) @@ -3062,11 +3050,7 @@ func unionBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := &Container{ - bitmap: ob, - n: n, - containerType: containerBitmap, - } + output := NewContainerBitmap(n, ob) return output } @@ -3079,8 +3063,8 @@ func unionBitmapBitmapInPlace(a, b *Container) { // see https://go101.org/article/bounds-check-elimination.html var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] + ab = a.bitmap()[:bitmapN] + bb = b.bitmap()[:bitmapN] ) // Manually unroll loop to make it a little faster. @@ -3125,17 +3109,18 @@ func difference(a, b *Container) *Container { // differenceArrayArray computes the difference bween two arrays. func differenceArrayArray(a, b *Container) *Container { statsHit("difference/ArrayArray") - output := &Container{containerType: containerArray} - na, nb := len(a.array), len(b.array) + output := NewContainerArray(nil) + aa, ab := a.array(), b.array() + na, nb := len(aa), len(ab) for i, j := 0, 0; i < na; { - va := a.array[i] + va := aa[i] if j >= nb { output.add(va) i++ continue } - vb := b.array[j] + vb := ab[j] if va < vb { output.add(va) i++ @@ -3157,48 +3142,51 @@ func differenceArrayRun(a, b *Container) *Container { return a.Clone() } - output := &Container{array: make([]uint16, 0, a.n), containerType: containerArray} + output := NewContainerArray(make([]uint16, 0, a.n)) // cardinality upper bound: card(A) i := 0 // array index j := 0 // run index + aa, rb := a.array(), b.runs() // handle overlap for i < int(a.n) { // keep all array elements before beginning of runs - if a.array[i] < b.runs[j].start { - output.add(a.array[i]) + if aa[i] < rb[j].start { + output.add(aa[i]) i++ continue } // if array element in run, skip it - if a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last { + if aa[i] >= rb[j].start && aa[i] <= rb[j].last { i++ continue } // if array element larger than current run, check next run - if a.array[i] > b.runs[j].last { + if aa[i] > rb[j].last { j++ - if j == len(b.runs) { + if j == len(rb) { break } } } - if i < len(a.array) { + if i < len(aa) { // keep all array elements after end of runs // It's possible that output was converted from array to bitmap in output.add() // so check container type before proceeding. - if output.containerType == containerArray { - output.array = append(output.array, a.array[i:]...) + if output.typ == containerArray { + array := output.array() + array = append(array, aa[i:]...) + output.setArray(array) // TODO: consider handling container.n mutations in one place // like we do with container.add(). - output.n += int32(len(a.array[i:])) + output.n += int32(len(aa[i:])) } else { - for _, v := range a.array[i:] { + for _, v := range aa[i:] { output.add(v) } } @@ -3214,8 +3202,8 @@ func differenceBitmapRun(a, b *Container) *Container { } output := a.Clone() - for j := 0; j < len(b.runs); j++ { - output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + for _, run := range b.runs() { + output.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) } return output } @@ -3227,20 +3215,21 @@ func differenceRunArray(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.Clone() } - output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: containerRun} + ra, ab := a.runs(), b.array() + runs := make([]interval16, 0, len(ra)) bidx := 0 - vb := b.array[bidx] + vb := ab[bidx] RUNLOOP: - for _, run := range a.runs { + for _, run := range ra { start := run.start for vb < run.start { bidx++ - if bidx >= len(b.array) { + if bidx >= len(ab) { break } - vb = b.array[bidx] + vb = ab[bidx] } for vb >= run.start && vb <= run.last { if vb == start { @@ -3249,30 +3238,29 @@ RUNLOOP: } start++ bidx++ - if bidx >= len(b.array) { + if bidx >= len(ab) { break } - vb = b.array[bidx] + vb = ab[bidx] continue } - output.runs = append(output.runs, interval16{start: start, last: vb - 1}) - output.n += int32(vb - start) + runs = append(runs, interval16{start: start, last: vb - 1}) if vb == 65535 { // overflow break RUNLOOP } start = vb + 1 bidx++ - if bidx >= len(b.array) { + if bidx >= len(ab) { break } - vb = b.array[bidx] + vb = ab[bidx] } if start <= run.last { - output.runs = append(output.runs, interval16{start: start, last: run.last}) - output.n += int32(run.last - start + 1) + runs = append(runs, interval16{start: start, last: run.last}) } } + output := NewContainerRun(runs) output.optimize() return output } @@ -3280,19 +3268,21 @@ RUNLOOP: // differenceRunBitmap computes the difference of an run from a bitmap. func differenceRunBitmap(a, b *Container) *Container { statsHit("difference/RunBitmap") + ra := a.runs() // If a is full, difference is the flip of b. - if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { + if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { return flipBitmap(b) } - output := &Container{containerType: containerRun} - output.n = a.n - if len(a.runs) == 0 { - return output + output := NewContainerRun(nil) + runs := output.runs() + if len(ra) == 0 { + return NewContainerRun(nil) } - for j := 0; j < len(a.runs); j++ { - run := a.runs[j] + output.n = a.n + for _, inputRun := range ra { + run := inputRun add := true - for bit := a.runs[j].start; bit <= a.runs[j].last; bit++ { + for bit := inputRun.start; bit <= inputRun.last; bit++ { if b.bitmapContains(bit) { output.n-- if run.start == bit { @@ -3306,10 +3296,10 @@ func differenceRunBitmap(a, b *Container) *Container { } else { run.last = bit - 1 if run.last >= run.start { - output.runs = append(output.runs, run) + runs = append(runs, run) } run.start = bit + 1 - run.last = a.runs[j].last + run.last = inputRun.last } if run.start > run.last { break @@ -3322,14 +3312,15 @@ func differenceRunBitmap(a, b *Container) *Container { } if run.start <= run.last { if add { - output.runs = append(output.runs, run) + runs = append(runs, run) } } } - if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { + output.setRuns(runs) + if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { output.runToArray() - } else if len(output.runs) > runMaxSize { + } else if len(runs) > runMaxSize { output.runToBitmap() } return output @@ -3342,16 +3333,17 @@ func differenceRunRun(a, b *Container) *Container { return a.Clone() } + ra, rb := a.runs(), b.runs() apos := 0 // current a-run index bpos := 0 // current b-run index - astart := a.runs[apos].start - alast := a.runs[apos].last - bstart := b.runs[bpos].start - blast := b.runs[bpos].last - alen := len(a.runs) - blen := len(b.runs) + astart := ra[apos].start + alast := ra[apos].last + bstart := rb[bpos].start + blast := rb[bpos].last + alen := len(ra) + blen := len(rb) - output := &Container{runs: make([]interval16, 0, alen+blen), containerType: containerRun} // TODO allocate max then truncate? or something else + runs := make([]interval16, 0, alen+blen) // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -3359,71 +3351,70 @@ func differenceRunRun(a, b *Container) *Container { switch { case alast < bstart: // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run - output.runs = append(output.runs, interval16{start: astart, last: alast}) + runs = append(runs, interval16{start: astart, last: alast}) apos++ if apos < alen { - astart = a.runs[apos].start - alast = a.runs[apos].last + astart = ra[apos].start + alast = ra[apos].last } case blast < astart: // current B-run entirely precedes current A-run: advance to next B-run bpos++ if bpos < blen { - bstart = b.runs[bpos].start - blast = b.runs[bpos].last + bstart = rb[bpos].start + blast = rb[bpos].last } default: // overlap if astart < bstart { - output.runs = append(output.runs, interval16{start: astart, last: bstart - 1}) + runs = append(runs, interval16{start: astart, last: bstart - 1}) } if alast > blast { astart = blast + 1 } else { apos++ if apos < alen { - astart = a.runs[apos].start - alast = a.runs[apos].last + astart = ra[apos].start + alast = ra[apos].last } } } } if apos < alen { - output.runs = append(output.runs, interval16{start: astart, last: alast}) + runs = append(runs, interval16{start: astart, last: alast}) apos++ if apos < alen { - output.runs = append(output.runs, a.runs[apos:]...) + runs = append(runs, ra[apos:]...) } } - - output.n = output.count() - return output + return NewContainerRun(runs) } func differenceArrayBitmap(a, b *Container) *Container { statsHit("difference/ArrayBitmap") - output := &Container{containerType: containerArray} - for _, va := range a.array { + output := make([]uint16, 0, a.n) + bitmap := b.bitmap() + for _, va := range a.array() { bmidx := va / 64 bidx := va % 64 mask := uint64(1) << bidx - b := b.bitmap[bmidx] + b := bitmap[bmidx] if mask&^b > 0 { - output.array = append(output.array, va) + output = append(output, va) } } - output.n = int32(len(output.array)) - return output + return NewContainerArray(output) } func differenceBitmapArray(a, b *Container) *Container { statsHit("difference/BitmapArray") output := a.Clone() + bitmap := output.bitmap() - for _, v := range b.array { + for _, v := range b.array() { if output.bitmapContains(v) { - output.bitmap[v/64] &^= (uint64(1) << uint(v%64)) + bitmap[v/64] &^= (uint64(1) << uint(v%64)) output.n-- } } @@ -3439,10 +3430,9 @@ func differenceBitmapBitmap(a, b *Container) *Container { // see https://go101.org/article/bounds-check-elimination.html var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] - buf = make([]uint64, bitmapN) - ob = buf[:bitmapN] + ab = a.bitmap()[:bitmapN] + bb = b.bitmap()[:bitmapN] + ob = make([]uint64, bitmapN)[:bitmapN] n int32 ) @@ -3452,11 +3442,7 @@ func differenceBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := &Container{ - bitmap: ob, - n: n, - containerType: containerBitmap, - } + output := NewContainerBitmap(n, ob) if output.n < ArrayMaxSize { output.bitmapToArray() } @@ -3493,20 +3479,21 @@ func xor(a, b *Container) *Container { func xorArrayArray(a, b *Container) *Container { statsHit("xor/ArrayArray") - output := &Container{containerType: containerArray} - na, nb := len(a.array), len(b.array) + output := NewContainerArray(nil) + aa, ab := a.array(), b.array() + na, nb := len(aa), len(ab) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { - output.add(a.array[i]) + output.add(aa[i]) i++ continue } else if i >= na && j < nb { - output.add(b.array[j]) + output.add(ab[j]) j++ continue } - va, vb := a.array[i], b.array[j] + va, vb := aa[i], ab[j] if va < vb { output.add(va) i++ @@ -3524,7 +3511,7 @@ func xorArrayArray(a, b *Container) *Container { func xorArrayBitmap(a, b *Container) *Container { statsHit("xor/ArrayBitmap") output := b.Clone() - for _, v := range a.array { + for _, v := range a.array() { if b.bitmapContains(v) { output.remove(v) } else { @@ -3534,7 +3521,7 @@ func xorArrayBitmap(a, b *Container) *Container { // It's possible that output was converted from bitmap to array in output.remove() // so we only do this conversion if output is still a bitmap container. - if output.containerType == containerBitmap && output.count() < ArrayMaxSize { + if output.typ == containerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3547,10 +3534,9 @@ func xorBitmapBitmap(a, b *Container) *Container { // see https://go101.org/article/bounds-check-elimination.html var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] - buf = make([]uint64, bitmapN) - ob = buf[:bitmapN] + ab = a.bitmap()[:bitmapN] + bb = b.bitmap()[:bitmapN] + ob = make([]uint64, bitmapN)[:bitmapN] n int32 ) @@ -3560,11 +3546,7 @@ func xorBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := &Container{ - bitmap: ob, - n: n, - containerType: containerBitmap, - } + output := NewContainerBitmap(n, ob) if output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3587,41 +3569,36 @@ func shift(c *Container) (*Container, bool) { func shiftArray(a *Container) (*Container, bool) { statsHit("shift/Array") carry := false - output := &Container{containerType: containerArray} - output.array = make([]uint16, len(a.array)) - output.array = output.array[:0] - output.n = a.n - for _, v := range a.array { + aa := a.array() + output := make([]uint16, 0, len(aa)) + for _, v := range aa { if v+1 == 0 { // overflow carry = true - output.n -= 1 } else { - output.array = append(output.array, v+1) + output = append(output, v+1) } } - return output, carry + return NewContainerArray(output), carry } // shiftBitmap is a bitmap-specific implementation of shift(). func shiftBitmap(a *Container) (*Container, bool) { statsHit("shift/Bitmap") carry := false - output := &Container{containerType: containerBitmap} - output.bitmap = make([]uint64, len(a.bitmap)) - output.bitmap = output.bitmap[:0] - output.n = a.n + output := NewContainerBitmap(a.n, nil) + ba, bo := a.bitmap(), output.bitmap() lastCarry := false - for _, v := range a.bitmap { + for i, v := range ba { carry = (v & (1 << 63)) != 0 v = v << 1 if lastCarry { v |= 1 } - output.bitmap = append(output.bitmap, v) + bo[i] = v lastCarry = carry } if carry { - output.n -= 1 + output.n-- } return output, carry } @@ -3630,27 +3607,25 @@ func shiftBitmap(a *Container) (*Container, bool) { func shiftRun(a *Container) (*Container, bool) { statsHit("shift/Run") carry := false - output := &Container{containerType: containerRun} - output.runs = make([]interval16, len(a.runs)) - output.runs = output.runs[:0] - for _, v := range a.runs { + ra := a.runs() + ro := make([]interval16, 0, len(ra)) + + for _, v := range ra { if v.start+1 == 0 { // final run was 1 bit on container edge carry = true - output.n -= 1 break } else if v.last+1 == 0 { // final run ends on container edge - v.start += 1 + v.start++ carry = true - output.n -= 1 } else { - v.start += 1 - v.last += 1 + v.start++ + v.last++ carry = false } - output.runs = append(output.runs, v) + ro = append(ro, v) } - return output, carry + return NewContainerRun(ro), carry } // opType represents a type of operation. @@ -3905,17 +3880,18 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) { // xorArrayRun computes the exclusive or of an array and a run container. func xorArrayRun(a, b *Container) *Container { statsHit("xor/ArrayRun") - output := &Container{containerType: containerRun} - na, nb := len(a.array), len(b.runs) + output := NewContainerRun(nil) + aa, rb := a.array(), b.runs() + na, nb := len(aa), len(rb) var vb interval16 var va uint16 lastI, lastJ := -1, -1 for i, j := 0, 0; i < na || j < nb; { if i < na && i != lastI { - va = a.array[i] + va = aa[i] } if j < nb && j != lastJ { - vb = b.runs[j] + vb = rb[j] } lastI = i lastJ = j @@ -3961,7 +3937,7 @@ func xorArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > runMaxSize { + } else if len(output.runs()) > runMaxSize { output.runToBitmap() } return output @@ -4064,14 +4040,15 @@ type xorstm struct { // xorRunRun computes the exclusive or of two run containers. func xorRunRun(a, b *Container) *Container { statsHit("xor/RunRun") - na, nb := len(a.runs), len(b.runs) + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) if na == 0 { return b.Clone() } if nb == 0 { return a.Clone() } - output := &Container{containerType: containerRun} + output := NewContainerRun(nil) lastI, lastJ := -1, -1 @@ -4079,12 +4056,12 @@ func xorRunRun(a, b *Container) *Container { for i, j := 0, 0; i < na || j < nb; { if i < na && lastI != i { - state.va = a.runs[i] + state.va = ra[i] state.vaValid = true } if j < nb && lastJ != j { - state.vb = b.runs[j] + state.vb = rb[j] state.vbValid = true } lastI, lastJ = i, j @@ -4102,9 +4079,10 @@ func xorRunRun(a, b *Container) *Container { } - if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { + l := len(output.runs()) + if output.n < ArrayMaxSize && int32(l) > output.n/2 { output.runToArray() - } else if len(output.runs) > runMaxSize { + } else if l > runMaxSize { output.runToBitmap() } return output @@ -4114,15 +4092,11 @@ func xorRunRun(a, b *Container) *Container { func xorBitmapRun(a, b *Container) *Container { statsHit("xor/BitmapRun") output := a.Clone() - for j := 0; j < len(b.runs); j++ { - output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + + for _, run := range b.runs() { + output.bitmapXorRange(uint64(run.start), uint64(run.last)+1) } - if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { - output.runToArray() - } else if len(output.runs) > runMaxSize { - output.runToBitmap() - } return output } @@ -4294,17 +4268,13 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.containerType { + switch c.typ { case containerArray: - c.runs = nil - c.bitmap = nil - c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]) case containerBitmap: - c.array = nil - c.runs = nil - c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] + c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]) default: - return fmt.Errorf("unsupported container type %d", c.containerType) + return fmt.Errorf("unsupported container type %d", c.typ) } } return nil @@ -4315,26 +4285,21 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { for i := 0; i < int(keyN); i++ { citer.Next() _, c := citer.Value() - switch c.containerType { + switch c.typ { case containerRun: - c.array = nil - c.bitmap = nil runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize]) - c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount] + c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount]) + runs := c.runs() - for o := range c.runs { // must convert from start:length to start:end :( - c.runs[o].last = c.runs[o].start + c.runs[o].last + for o := range runs { // must convert from start:length to start:end :( + runs[o].last = runs[o].start + runs[o].last } pos += int((runCount * interval16Size) + runCountHeaderSize) case containerArray: - c.runs = nil - c.bitmap = nil - c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n] + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n]) pos += int(c.n * 2) case containerBitmap: - c.array = nil - c.runs = nil - c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN] + c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN]) pos += bitmapN * 8 } } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index f99d922e8..8c8bbd9ac 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -248,22 +248,18 @@ type testOp struct { exp string } -func doContainer(containerType byte, data interface{}) *Container { - c := &Container{ - containerType: containerType, - } - - switch containerType { +func doContainer(typ byte, data interface{}) *Container { + switch typ { case containerArray: - c.array = data.([]uint16) + return NewContainerArray(data.([]uint16)) case containerBitmap: - c.bitmap = data.([]uint64) + c := NewContainerBitmap(0, data.([]uint64)) + c.n = c.count() + return c case containerRun: - c.runs = data.([]interval16) + return NewContainerRun(data.([]interval16)) } - c.n = c.count() - - return c + return nil } func setupContainerTests() map[byte]map[string]*Container { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 16f5e4e7b..dc2f770db 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -33,11 +33,11 @@ func (iv interval16) String() string { } func (c *Container) String() string { - return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.containerType) + return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%s", c.info().Type, c.n, len(c.array()), len(c.runs()), len(c.bitmap()), containerTypeNames[c.typ]) } func TestRunAppendInterval(t *testing.T) { - a := Container{containerType: containerRun} + a := NewContainerRun(nil) tests := []struct { base []interval16 app interval16 @@ -66,7 +66,7 @@ func TestRunAppendInterval(t *testing.T) { } for i, test := range tests { - a.runs = test.base + a.setRuns(test.base) if n := a.runAppendInterval(test.app); n != test.exp { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, n) } @@ -86,7 +86,7 @@ func TestInterval16RunLen(t *testing.T) { } func TestContainerRunAdd(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: containerRun} + c := NewContainerRun(nil) tests := []struct { op uint16 exp []interval16 @@ -105,10 +105,10 @@ func TestContainerRunAdd(t *testing.T) { c.mapped = true ret := c.add(test.op) if !ret { - t.Fatalf("result of adding new bit should be true: %v", c.runs) + t.Fatalf("result of adding new bit should be true: %v", c.runs()) } - if !reflect.DeepEqual(c.runs, test.exp) { - t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs, test.op) + if !reflect.DeepEqual(c.runs(), test.exp) { + t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs(), test.op) } if c.mapped { t.Fatalf("container should not be mapped after adding bit %v", test.op) @@ -117,22 +117,22 @@ func TestContainerRunAdd(t *testing.T) { } func TestContainerRunAdd2(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: containerRun} + c := NewContainerRun(nil) ret := c.add(0) if !ret { - t.Fatalf("result of adding new bit should be true: %v", c.runs) + t.Fatalf("result of adding new bit should be true: %v", c.runs()) } - if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) { - t.Fatalf("should have 1 run of length 1, but have %v", c.runs) + if !reflect.DeepEqual(c.runs(), []interval16{{start: 0, last: 0}}) { + t.Fatalf("should have 1 run of length 1, but have %v", c.runs()) } ret = c.add(0) if ret { - t.Fatalf("result of adding existing bit should be false: %v", c.runs) + t.Fatalf("result of adding existing bit should be false: %v", c.runs()) } } func TestRunCountRange(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: containerRun} + c := NewContainerRun(nil) cnt := c.runCountRange(2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) @@ -185,12 +185,12 @@ func TestRunCountRange(t *testing.T) { // verify that the disparate ops resulted in three separate runs cnt = c.countRuns() if cnt != 3 { - t.Fatalf("should get 3 total runs, but got: %v [%v]", cnt, c.runs) + t.Fatalf("should get 3 total runs, but got: %v [%v]", cnt, c.runs()) } } func TestRunContains(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: containerRun} + c := NewContainerRun(nil) if c.runContains(5) { t.Fatalf("empty run container should not contain 5") } @@ -212,7 +212,7 @@ func TestRunContains(t *testing.T) { } func TestBitmapCountRange(t *testing.T) { - c := Container{containerType: containerBitmap} + c := NewContainerBitmap(0, nil) tests := []struct { start int32 end int32 @@ -229,7 +229,7 @@ func TestBitmapCountRange(t *testing.T) { } for i, test := range tests { - c.bitmap = test.bitmap + c.setBitmap(test.bitmap) if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp { t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret) } @@ -237,14 +237,8 @@ func TestBitmapCountRange(t *testing.T) { } func TestIntersectionCountArrayBitmap3(t *testing.T) { - a, b := &Container{}, &Container{} - a.containerType = containerBitmap - a.bitmap = getFullBitmap() - a.n = maxContainerVal + 1 + a, b := NewContainerBitmap(maxContainerVal+1, getFullBitmap()), NewContainerBitmap(maxContainerVal+1, getFullBitmap()) - b.containerType = containerBitmap - b.bitmap = getFullBitmap() - b.n = maxContainerVal + 1 res := intersectBitmapBitmap(a, b) if res.n != res.count() || res.n != maxContainerVal+1 { t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) @@ -264,7 +258,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) { } func TestIntersectionCountArrayBitmap2(t *testing.T) { - a, b := &Container{}, &Container{} + a, b := NewContainerArray(nil), NewContainerBitmap(0, nil) tests := []struct { array []uint16 bitmap []uint64 @@ -298,10 +292,8 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } for i, test := range tests { - a.array = test.array - a.containerType = containerArray - b.bitmap = test.bitmap - b.containerType = containerBitmap + a.setArray(test.array) + b.setBitmap(test.bitmap) ret := intersectionCountArrayBitmap(a, b) if ret != test.exp { t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp) @@ -310,7 +302,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} + c := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) tests := []struct { op uint16 exp []interval16 @@ -331,8 +323,8 @@ func TestRunRemove(t *testing.T) { for i, test := range tests { c.mapped = true ret := c.remove(test.op) - if ret != test.expRet || !reflect.DeepEqual(c.runs, test.exp) { - t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs) + if ret != test.expRet || !reflect.DeepEqual(c.runs(), test.exp) { + t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs()) } if ret && c.mapped { t.Fatalf("test #%v container was not unmapped although bit %v was removed", i, test.op) @@ -344,50 +336,52 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} + c := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) max := c.max() if max != 16 { - t.Fatalf("max for %v should be 16", c.runs) + t.Fatalf("max for %v should be 16", c.runs()) } - c = Container{runs: []interval16{}} + c = NewContainerRun(nil) max = c.max() if max != 0 { - t.Fatalf("max for %v should be 0", c.runs) + t.Fatalf("max for %v should be 0", c.runs()) } } func TestIntersectionCountArrayRun(t *testing.T) { - a := &Container{containerType: containerArray, array: []uint16{1, 5, 10, 11, 12}} - b := &Container{containerType: containerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + a := NewContainerArray([]uint16{1, 5, 10, 11, 12}) + b := NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}) ret := intersectionCountArrayRun(a, b) if ret != 3 { - t.Fatalf("count of %v with %v should be 3, but got %v", a.array, b.runs, ret) + t.Fatalf("count of %v with %v should be 3, but got %v", a.array(), b.runs(), ret) } } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &Container{containerType: containerBitmap, bitmap: []uint64{0x8000000000000000}} - b := &Container{containerType: containerRun, runs: []interval16{{start: 63, last: 64}}} + ob := make([]uint64, bitmapN) + ob[0] = 1 << 63 + a := NewContainerBitmap(1, ob) + b := NewContainerRun([]interval16{{start: 63, last: 64}}) ret := intersectionCountBitmapRun(a, b) if ret != 1 { - t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) + t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap(), b.runs(), ret) } - a = &Container{containerType: containerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &Container{containerType: containerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + a = NewContainerBitmap(29, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) + b = NewContainerRun([]interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}) ret = intersectionCountBitmapRun(a, b) if ret != 14 { - t.Fatalf("count of %v with %v should be 14, but got %v", a.bitmap, b.runs, ret) + t.Fatalf("count of %v with %v should be 14, but got %v", a.bitmap(), b.runs(), ret) } } func TestIntersectionCountRunRun(t *testing.T) { - a := &Container{} - b := &Container{} + a := NewContainerRun(nil) + b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -425,10 +419,10 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.containerType = containerRun - b.containerType = containerRun - a.runs = test.aruns - b.runs = test.bruns + a.typ = containerRun + b.typ = containerRun + a.setRuns(test.aruns) + b.setRuns(test.bruns) ret := intersectionCountRunRun(a, b) if ret != test.exp { t.Fatalf("test #%v failed intersecting %v with %v should be %v, but got %v", i, test.aruns, test.bruns, test.exp, ret) @@ -437,8 +431,8 @@ func TestIntersectionCountRunRun(t *testing.T) { } func TestIntersectArrayRun(t *testing.T) { - a := &Container{} - b := &Container{} + a := NewContainerArray(nil) + b := NewContainerRun(nil) tests := []struct { array []uint16 runs []interval16 @@ -467,20 +461,18 @@ func TestIntersectArrayRun(t *testing.T) { } for i, test := range tests { - a.containerType = containerArray - b.containerType = containerRun - a.array = test.array - b.runs = test.runs + a.setArray(test.array) + b.setRuns(test.runs) ret := intersectArrayRun(a, b) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } } } func TestIntersectRunRun(t *testing.T) { - a := &Container{} - b := &Container{} + a := NewContainerRun(nil) + b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -525,24 +517,22 @@ func TestIntersectRunRun(t *testing.T) { }, } for i, test := range tests { - a.containerType = containerRun - b.containerType = containerRun - a.runs = test.aruns - b.runs = test.bruns + a.setRuns(test.aruns) + b.setRuns(test.bruns) ret := intersectRunRun(a, b) if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) } - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } } } func TestIntersectBitmapRunBitmap(t *testing.T) { - a := &Container{bitmap: make([]uint64, bitmapN)} - b := &Container{} + a := NewContainerBitmap(0, nil) + b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -581,19 +571,19 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap, test.bitmap) - b.runs = test.runs + copy(a.bitmap(), test.bitmap) + b.setRuns(test.runs) b.n = 4097 // ;) exp := make([]uint64, bitmapN) copy(exp, test.exp) - a.containerType = containerBitmap - b.containerType = containerRun + a.typ = containerBitmap + b.typ = containerRun ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() } - if !reflect.DeepEqual(ret.bitmap, exp) { - t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap) + if !reflect.DeepEqual(ret.bitmap(), exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap()) } if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) @@ -603,8 +593,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { } func TestIntersectBitmapRunArray(t *testing.T) { - a := &Container{bitmap: make([]uint64, bitmapN)} - b := &Container{} + a := NewContainerBitmap(0, nil) + b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -643,13 +633,11 @@ func TestIntersectBitmapRunArray(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap, test.bitmap) - b.runs = test.runs - a.containerType = containerBitmap - b.containerType = containerRun + copy(a.bitmap(), test.bitmap) + b.setRuns(test.runs) ret := intersectBitmapRun(a, b) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) @@ -661,22 +649,13 @@ func TestIntersectBitmapRunArray(t *testing.T) { func TestUnionMixed(t *testing.T) { // array container - a := &Container{} - a.array = []uint16{1, 4, 5, 7, 10, 11, 12} - a.containerType = containerArray - a.n = 7 + a := NewContainerArray([]uint16{1, 4, 5, 7, 10, 11, 12}) // bitmap container - b := &Container{bitmap: make([]uint64, bitmapN)} - b.bitmap[0] = uint64(0x3) - b.n = 2 - b.containerType = containerBitmap + b := NewContainerBitmap(2, []uint64{0x3}) // run container - r := &Container{} - r.runs = []interval16{{start: 5, last: 10}} - r.containerType = containerRun - r.n = 6 + r := NewContainerRun([]interval16{{start: 5, last: 10}}) t.Run("various container Unions", func(t *testing.T) { tests := []struct { @@ -702,116 +681,95 @@ func TestUnionMixed(t *testing.T) { } else if res.isRun() { res.runToArray() } - if !reflect.DeepEqual(res.array, tt.exp) { - t.Fatalf("test %s expected %v, but got %v", tt.name, tt.exp, res.array) + if !reflect.DeepEqual(res.array(), tt.exp) { + t.Fatalf("test %s expected %v, but got %v", tt.name, tt.exp, res.array()) } } }) } func TestIntersectMixed(t *testing.T) { - a := &Container{} - b := &Container{} - c := &Container{} + a := NewContainerRun([]interval16{{start: 5, last: 10}}) + b := NewContainerArray([]uint16{1, 4, 5, 7, 10, 11, 12}) + c := NewContainerBitmap(2, []uint64{0x60}) - a.runs = []interval16{{start: 5, last: 10}} - a.n = 6 - a.containerType = containerRun - b.array = []uint16{1, 4, 5, 7, 10, 11, 12} - b.n = 7 - b.containerType = containerArray res := intersect(a, b) - if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { - t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5, 7, 10}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array()) } res = intersect(b, a) - if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { - t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5, 7, 10}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array()) } - res = intersect(a, a) - if !reflect.DeepEqual(res.runs, []interval16{{start: 5, last: 10}}) { - t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs) + if !reflect.DeepEqual(res.runs(), []interval16{{start: 5, last: 10}}) { + t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs()) } - c.bitmap = []uint64{0x60} - c.n = 2 - c.containerType = containerBitmap res = intersect(c, a) - if !reflect.DeepEqual(res.array, []uint16{5, 6}) { - t.Fatalf("test #4 expected %v, but got %v", []uint16{6}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5, 6}) { + t.Fatalf("test #4 expected %v, but got %v", []uint16{6}, res.array()) } res = intersect(a, c) - if !reflect.DeepEqual(res.array, []uint16{5, 6}) { - t.Fatalf("test #5 expected %v, but got %v", []uint16{6}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5, 6}) { + t.Fatalf("test #5 expected %v, but got %v", []uint16{6}, res.array()) } res = intersect(b, c) - if !reflect.DeepEqual(res.array, []uint16{5}) { - t.Fatalf("test #6 expected %v, but got %v", []uint16{5}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{5}, res.array()) } res = intersect(c, b) - if !reflect.DeepEqual(res.array, []uint16{5}) { - t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5}) { + t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array()) } } func TestDifferenceMixed(t *testing.T) { - a := &Container{} - b := &Container{} - c := &Container{} - d := &Container{} + a := NewContainerRun([]interval16{{start: 5, last: 10}}) - a.runs = []interval16{{start: 5, last: 10}} - a.n = a.runCountRange(0, 100) - a.containerType = containerRun + b := NewContainerArray([]uint16{0, 2, 4, 6, 8, 10, 12}) - b.array = []uint16{0, 2, 4, 6, 8, 10, 12} - b.n = int32(len(b.array)) - b.containerType = containerArray + c := NewContainerBitmap(0, MakeBitmap([]uint64{0x64})) + c.n = c.countRange(0, 100) - d.array = []uint16{1, 3, 5, 7, 9, 11, 12} - d.n = int32(len(d.array)) - d.containerType = containerArray + d := NewContainerArray([]uint16{1, 3, 5, 7, 9, 11, 12}) res := difference(a, b) - if !reflect.DeepEqual(res.array, []uint16{5, 7, 9}) { + if !reflect.DeepEqual(res.array(), []uint16{5, 7, 9}) { t.Fatalf("test #1 expected %v, but got %#v", []uint16{5, 7, 9}, res) } res = difference(b, a) - if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 12}) { - t.Fatalf("test #2 expected %v, but got %v", []uint16{0, 2, 4, 12}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{0, 2, 4, 12}) { + t.Fatalf("test #2 expected %v, but got %v", []uint16{0, 2, 4, 12}, res.array()) } res = difference(a, a) - if !reflect.DeepEqual(res.runs, []interval16{}) { - t.Fatalf("test #3 expected empty but got %v", res.runs) + if !reflect.DeepEqual(res.runs(), []interval16{}) { + t.Fatalf("test #3 expected empty but got %v", res.runs()) } - c.bitmap = MakeBitmap([]uint64{0x64}) - c.n = c.countRange(0, 100) - c.containerType = containerBitmap res = difference(c, a) - if !reflect.DeepEqual(res.bitmap, MakeBitmap([]uint64{0x4})) { - t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) + if !reflect.DeepEqual(res.bitmap(), MakeBitmap([]uint64{0x4})) { + t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap()) } res = difference(a, c) - if !reflect.DeepEqual(res.runs, []interval16{{start: 7, last: 10}}) { - t.Fatalf("test #5 expected %v, but got %v", []interval16{{start: 7, last: 10}}, res.runs) + if !reflect.DeepEqual(res.runs(), []interval16{{start: 7, last: 10}}) { + t.Fatalf("test #5 expected %v, but got %v", []interval16{{start: 7, last: 10}}, res.runs()) } res = difference(b, c) - if !reflect.DeepEqual(res.array, []uint16{0, 4, 8, 10, 12}) { - t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 4, 8, 10, 12}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{0, 4, 8, 10, 12}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 4, 8, 10, 12}, res.array()) } res = difference(c, b) - if !reflect.DeepEqual(res.array, []uint16{5}) { - t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{5}) { + t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array()) } res = difference(b, b) @@ -825,20 +783,20 @@ func TestDifferenceMixed(t *testing.T) { } res = difference(d, b) - if !reflect.DeepEqual(res.array, []uint16{1, 3, 5, 7, 9, 11}) { - t.Fatalf("test #10 expected %v, but got %d", []uint16{1, 3, 5, 7, 9, 11}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{1, 3, 5, 7, 9, 11}) { + t.Fatalf("test #10 expected %v, but got %d", []uint16{1, 3, 5, 7, 9, 11}, res.array()) } res = difference(b, d) - if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 6, 8, 10}) { - t.Fatalf("test #11 expected %v, but got %d", []uint16{0, 2, 4, 6, 8, 10}, res.array) + if !reflect.DeepEqual(res.array(), []uint16{0, 2, 4, 6, 8, 10}) { + t.Fatalf("test #11 expected %v, but got %d", []uint16{0, 2, 4, 6, 8, 10}, res.array()) } } func TestUnionRunRun(t *testing.T) { - a := &Container{} - b := &Container{} + a := NewContainerRun(nil) + b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -886,20 +844,18 @@ func TestUnionRunRun(t *testing.T) { }, } for i, test := range tests { - a.runs = test.aruns - b.runs = test.bruns - a.containerType = containerRun - b.containerType = containerRun + a.setRuns(test.aruns) + b.setRuns(test.bruns) ret := unionRunRun(a, b) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } } } func TestUnionArrayRun(t *testing.T) { - a := &Container{} - b := &Container{} + a := NewContainerArray(nil) + b := NewContainerRun(nil) tests := []struct { array []uint16 runs []interval16 @@ -928,19 +884,17 @@ func TestUnionArrayRun(t *testing.T) { } for i, test := range tests { - a.array = test.array - b.runs = test.runs - a.containerType = containerArray - b.containerType = containerRun + a.setArray(test.array) + b.setRuns(test.runs) ret := unionArrayRun(a, b) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } } } func TestBitmapSetRange(t *testing.T) { - c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -965,11 +919,11 @@ func TestBitmapSetRange(t *testing.T) { } for i, test := range tests { - copy(c.bitmap, test.bitmap) + copy(c.bitmap(), test.bitmap) c.n = c.countRange(0, 65535) c.bitmapSetRange(test.start, test.last+1) - if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } if test.expN != c.n { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) @@ -978,7 +932,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &Container{containerType: containerArray} + a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint64 @@ -997,17 +951,18 @@ func TestArrayToBitmap(t *testing.T) { exp := make([]uint64, bitmapN) copy(exp, test.exp) - a.array = test.array + a.setArray(test.array) a.n = int32(len(test.array)) a.arrayToBitmap() - if !reflect.DeepEqual(a.bitmap, exp) { - t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) + if !reflect.DeepEqual(a.bitmap(), exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } + a.bitmapToArray() } } func TestBitmapToArray(t *testing.T) { - a := &Container{containerType: containerBitmap} + a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint16 @@ -1022,23 +977,25 @@ func TestBitmapToArray(t *testing.T) { }, } for i, test := range tests { - a.bitmap = make([]uint64, bitmapN) + a.setBitmap(make([]uint64, bitmapN)) + bitmap := a.bitmap() n := int32(0) for i, v := range test.bitmap { - a.bitmap[i] = v + bitmap[i] = v n += int32(popcount(v)) } a.n = n a.bitmapToArray() - if !reflect.DeepEqual(a.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) + if !reflect.DeepEqual(a.array(), test.exp) { + t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, a.array()) } + a.arrayToBitmap() } } func TestRunToBitmap(t *testing.T) { - a := &Container{containerType: containerRun} + a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint64 @@ -1073,11 +1030,12 @@ func TestRunToBitmap(t *testing.T) { n += int(popcount(v)) } - a.runs = test.runs + a.typ = containerRun + a.setRuns(test.runs) a.n = int32(n) a.runToBitmap() - if !reflect.DeepEqual(a.bitmap, exp) { - t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) + if !reflect.DeepEqual(a.bitmap(), exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } } } @@ -1092,7 +1050,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &Container{containerType: containerBitmap} + a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []interval16 @@ -1150,27 +1108,28 @@ func TestBitmapToRun(t *testing.T) { tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF for i, test := range tests { - a.bitmap = make([]uint64, bitmapN) + a.setBitmap(make([]uint64, bitmapN)) + bitmap := a.bitmap() n := 0 for i, v := range test.bitmap { - a.bitmap[i] = v + bitmap[i] = v n += int(popcount(v)) } a.n = int32(n) - x := a.bitmap + x := bitmap a.bitmapToRun() - if !reflect.DeepEqual(a.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) + if !reflect.DeepEqual(a.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) } a.runToBitmap() - if !reflect.DeepEqual(a.bitmap, x) { - t.Fatalf("test #%v expected %v, but got %v", i, a.bitmap, x) + if !reflect.DeepEqual(a.bitmap(), x) { + t.Fatalf("test #%v expected %v, but got %v", i, a.bitmap(), x) } } } func TestArrayToRun(t *testing.T) { - a := &Container{containerType: containerArray} + a := NewContainerArray(nil) tests := []struct { array []uint16 exp []interval16 @@ -1194,17 +1153,18 @@ func TestArrayToRun(t *testing.T) { } for i, test := range tests { - a.array = test.array + a.typ = containerArray + a.setArray(test.array) a.n = int32(len(test.array)) a.arrayToRun() - if !reflect.DeepEqual(a.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) + if !reflect.DeepEqual(a.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) } } } func TestRunToArray(t *testing.T) { - a := &Container{containerType: containerRun} + a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint16 @@ -1228,17 +1188,18 @@ func TestRunToArray(t *testing.T) { } for i, test := range tests { - a.runs = test.runs + a.typ = containerRun + a.setRuns(test.runs) a.n = int32(len(test.exp)) a.runToArray() - if !reflect.DeepEqual(a.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) + if !reflect.DeepEqual(a.array(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array()) } } } func TestBitmapZeroRange(t *testing.T) { - c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -1261,27 +1222,28 @@ func TestBitmapZeroRange(t *testing.T) { expN: 13, }, } + bitmap := c.bitmap() for i, test := range tests { - copy(c.bitmap, test.bitmap) + copy(bitmap, test.bitmap) c.n = c.countRange(0, 65535) c.bitmapZeroRange(test.start, test.last+1) - if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, bitmap[:len(test.bitmap)]) } if test.expN != c.n { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) } for i := range test.bitmap { - c.bitmap[i] = 0 + bitmap[i] = 0 } } } func TestUnionBitmapRun(t *testing.T) { - a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: containerRun} + a := NewContainerBitmap(0, nil) + b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1296,28 +1258,29 @@ func TestUnionBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap, test.bitmap) + copy(a.bitmap(), test.bitmap) a.n = a.bitmapCountRange(0, 65535) - b.runs = test.runs + b.setRuns(test.runs) b.n = b.runCountRange(0, 65535) ret := unionBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() } - if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test #%v expected %x, but got %x", i, test.exp, ret.bitmap[:len(test.exp)]) + bitmap := ret.bitmap() + if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test #%v expected %x, but got %x", i, test.exp, bitmap[:len(test.exp)]) } if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) } for i := range test.bitmap { - a.bitmap[i] = 0 + a.bitmap()[i] = 0 } } } func TestBitmapCountRuns(t *testing.T) { - c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp int32 @@ -1341,7 +1304,7 @@ func TestBitmapCountRuns(t *testing.T) { } for i, test := range tests { - copy(c.bitmap, test.bitmap) + copy(c.bitmap(), test.bitmap) ret := c.bitmapCountRuns() if ret != test.exp { @@ -1349,13 +1312,13 @@ func TestBitmapCountRuns(t *testing.T) { } for j := range test.bitmap { - c.bitmap[j] = 0 + c.bitmap()[j] = 0 } } test := tests[3] for j, v := range test.bitmap { - c.bitmap[1024-len(test.bitmap)+j] = v + c.bitmap()[1024-len(test.bitmap)+j] = v } ret := c.bitmapCountRuns() @@ -1365,7 +1328,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &Container{containerType: containerArray} + c := NewContainerArray(nil) tests := []struct { array []uint16 exp int32 @@ -1397,7 +1360,7 @@ func TestArrayCountRuns(t *testing.T) { } for i, test := range tests { - c.array = test.array + c.setArray(test.array) ret := c.arrayCountRuns() if ret != test.exp { t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) @@ -1406,8 +1369,8 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &Container{containerType: containerArray} - b := &Container{containerType: containerRun} + a := NewContainerArray(nil) + b := NewContainerRun(nil) tests := []struct { array []uint16 runs []interval16 @@ -1420,20 +1383,20 @@ func TestDifferenceArrayRun(t *testing.T) { }, } for i, test := range tests { - a.array = test.array - a.n = int32(len(a.array)) - b.runs = test.runs + a.setArray(test.array) + a.n = int32(len(a.array())) + b.setRuns(test.runs) b.n = b.runCountRange(0, 100) ret := differenceArrayRun(a, b) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } } } func TestDifferenceRunArray(t *testing.T) { - a := &Container{containerType: containerRun} - b := &Container{containerType: containerArray} + a := NewContainerRun(nil) + b := NewContainerArray(nil) tests := []struct { runs []interval16 array []uint16 @@ -1486,13 +1449,13 @@ func TestDifferenceRunArray(t *testing.T) { }, } for i, test := range tests { - a.runs = test.runs + a.setRuns(test.runs) a.n = a.runCountRange(0, 100) - b.array = test.array - b.n = int32(len(b.array)) + b.setArray(test.array) + b.n = int32(len(b.array())) ret := differenceRunArray(a, b) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } } } @@ -1505,12 +1468,12 @@ func MakeLastBitSet() []uint64 { obj := NewFileBitmap(65535) c := obj.container(0) c.arrayToBitmap() - return c.bitmap + return c.bitmap() } func TestDifferenceRunBitmap(t *testing.T) { - a := &Container{containerType: containerRun} - b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + a := NewContainerRun(nil) + b := NewContainerBitmap(0, nil) tests := []struct { runs []interval16 bitmap []uint64 @@ -1558,20 +1521,20 @@ func TestDifferenceRunBitmap(t *testing.T) { }, } for i, test := range tests { - a.runs = test.runs + a.setRuns(test.runs) a.n = a.runCountRange(0, 65536) - copy(b.bitmap, test.bitmap) + copy(b.bitmap(), test.bitmap) b.n = b.bitmapCountRange(0, 65536) ret := differenceRunBitmap(a, b) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } } } func TestDifferenceBitmapRun(t *testing.T) { - a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: containerRun} + a := NewContainerBitmap(0, nil) + b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1639,20 +1602,20 @@ func TestDifferenceBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap, test.bitmap) + copy(a.bitmap(), test.bitmap) a.n = a.bitmapCountRange(0, 65536) - b.runs = test.runs + b.setRuns(test.runs) b.n = b.runCountRange(0, 65536) ret := differenceBitmapRun(a, b) - if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) + if !reflect.DeepEqual(ret.bitmap()[:len(test.exp)], test.exp) { + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap()[:len(test.exp)]) } } } func TestDifferenceBitmapArray(t *testing.T) { - b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} - a := &Container{containerType: containerArray} + b := NewContainerBitmap(0, nil) + a := NewContainerArray(nil) tests := []struct { bitmap []uint64 array []uint16 @@ -1690,19 +1653,19 @@ func TestDifferenceBitmapArray(t *testing.T) { }, } for i, test := range tests { - b.bitmap[0] = test.bitmap[0] + b.bitmap()[0] = test.bitmap[0] b.n = b.count() - a.array = test.array + a.setArray(test.array) ret := differenceBitmapArray(b, a) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %X, but got %X", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret.array()) } } } func TestDifferenceBitmapBitmap(t *testing.T) { - a := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} - b := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + a := NewContainerBitmap(0, nil) + b := NewContainerBitmap(0, nil) tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1720,19 +1683,19 @@ func TestDifferenceBitmapBitmap(t *testing.T) { }, } for i, test := range tests { - a.bitmap[0] = test.abitmap[0] - b.bitmap[0] = test.bbitmap[0] + a.bitmap()[0] = test.abitmap[0] + b.bitmap()[0] = test.bbitmap[0] ret := differenceBitmapBitmap(a, b) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array) + if !reflect.DeepEqual(ret.array(), test.exp) { + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array()) } } } func TestDifferenceRunRun(t *testing.T) { - a := &Container{containerType: containerRun} - b := &Container{containerType: containerRun} + a := NewContainerRun(nil) + b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -1750,13 +1713,13 @@ func TestDifferenceRunRun(t *testing.T) { }, } for i, test := range tests { - a.runs = test.aruns + a.setRuns(test.aruns) a.n = a.runCountRange(0, 100) - b.runs = test.bruns + b.setRuns(test.bruns) b.n = b.runCountRange(0, 100) ret := differenceRunRun(a, b) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } if ret.n != test.expn { t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.n) @@ -1765,7 +1728,7 @@ func TestDifferenceRunRun(t *testing.T) { } func TestWriteReadArray(t *testing.T) { - ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: containerArray} + ca := NewContainerArray([]uint16{1, 10, 100, 1000}) ba := NewFileBitmap() ba.Containers.Put(0, ca) ba2 := NewFileBitmap() @@ -1778,16 +1741,16 @@ func TestWriteReadArray(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(ba2.Containers.Get(0).array, ca.array) { - t.Fatalf("array test expected %x, but got %x", ca.array, ba2.Containers.Get(0).array) + if !reflect.DeepEqual(ba2.Containers.Get(0).array(), ca.array()) { + t.Fatalf("array test expected %x, but got %x", ca.array(), ba2.Containers.Get(0).array()) } } func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: containerBitmap} + cb := NewContainerBitmap(129*32, nil) for i := 0; i < 129; i++ { - cb.bitmap[i] = 0x5555555555555555 + cb.bitmap()[i] = 0x5555555555555555 } bb := NewFileBitmap() bb.Containers.Put(0, cb) @@ -1801,31 +1764,34 @@ func TestWriteReadBitmap(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap) + if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap(), cb.bitmap()) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap(), bb2.Containers.Get(0).bitmap()) } } func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: containerBitmap} + cb := NewContainerBitmap(65536, nil) for i := 0; i < bitmapN; i++ { - cb.bitmap[i] = 0xffffffffffffffff + cb.bitmap()[i] = 0xffffffffffffffff } bb := NewFileBitmap() bb.Containers.Put(0, cb) bb2 := NewFileBitmap() var buf bytes.Buffer - _, err := bb.WriteTo(&buf) + _, err := bb.writeToUnoptimized(&buf) if err != nil { t.Fatalf("error writing: %v", err) } + if !cb.isBitmap() { + t.Fatalf("how can i test a bitmap if it's not a bitmap") + } err = bb2.UnmarshalBinary(buf.Bytes()) if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap) + if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap(), cb.bitmap()) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap(), bb2.Containers.Get(0).bitmap()) } if bb2.Containers.Get(0).n != cb.n { @@ -1837,7 +1803,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: containerRun} + cr := NewContainerRun([]interval16{{start: 3, last: 13}, {start: 100, last: 109}}) br := NewFileBitmap() br.Containers.Put(0, cr) br2 := NewFileBitmap() @@ -1850,8 +1816,8 @@ func TestWriteReadRun(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(br2.Containers.Get(0).runs, cr.runs) { - t.Fatalf("run test expected %x, but got %x", cr.runs, br2.Containers.Get(0).runs) + if !reflect.DeepEqual(br2.Containers.Get(0).runs(), cr.runs()) { + t.Fatalf("run test expected %x, but got %x", cr.runs(), br2.Containers.Get(0).runs()) } } @@ -1862,21 +1828,21 @@ func TestXorArrayRun(t *testing.T) { exp *Container }{ { - a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: containerArray, n: 12}, + a: NewContainerArray([]uint16{1, 5, 10, 11, 12}), + b: NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}), + exp: NewContainerArray([]uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}), }, { - a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: containerArray, n: 12}, + a: NewContainerArray([]uint16{1, 5, 10, 11, 12, 13, 14}), + b: NewContainerRun([]interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}), + exp: NewContainerArray([]uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}), }, { - a: &Container{array: []uint16{65535}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: containerRun}, - exp: &Container{array: []uint16{65534}, containerType: containerArray, n: 1}, + a: NewContainerArray([]uint16{65535}), + b: NewContainerRun([]interval16{{start: 65534, last: 65535}}), + exp: NewContainerArray([]uint16{65534}), }, { - a: &Container{array: []uint16{65535}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: containerRun}, - exp: &Container{array: []uint16{}, containerType: containerArray, n: 0}, + a: NewContainerArray([]uint16{65535}), + b: NewContainerRun([]interval16{{start: 65535, last: 65535}}), + exp: NewContainerArray([]uint16{}), }, } @@ -1885,11 +1851,11 @@ func TestXorArrayRun(t *testing.T) { test.b.n = test.b.count() ret := xor(test.a, test.b) if !reflect.DeepEqual(ret, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret) + t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret) } ret = xor(test.b, test.a) if !reflect.DeepEqual(ret, test.exp) { - t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret) + t.Fatalf("test #%v.1 expected %#v, but got %#v", i, test.exp, ret) } } @@ -1897,23 +1863,21 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := &Container{containerType: containerRun} - b := &Container{containerType: containerRun} - a.runs = []interval16{{start: 4, last: 10}} - b.runs = []interval16{{start: 5, last: 10}} + a := NewContainerRun([]interval16{{start: 4, last: 10}}) + b := NewContainerRun([]interval16{{start: 5, last: 10}}) ret := xorRunRun(a, b) - if !reflect.DeepEqual(ret.array, []uint16{4}) { - t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array) + if !reflect.DeepEqual(ret.array(), []uint16{4}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array()) } ret = xorRunRun(b, a) - if !reflect.DeepEqual(ret.array, []uint16{4}) { - t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array) + if !reflect.DeepEqual(ret.array(), []uint16{4}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array()) } } func TestXorRunRun(t *testing.T) { - a := &Container{containerType: containerRun} - b := &Container{containerType: containerRun} + a := NewContainerRun(nil) + b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -1996,21 +1960,21 @@ func TestXorRunRun(t *testing.T) { }, } for i, test := range tests { - a.runs = test.aruns - b.runs = test.bruns + a.setRuns(test.aruns) + b.setRuns(test.bruns) ret := xorRunRun(a, b) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } ret = xorRunRun(b, a) - if !reflect.DeepEqual(ret.runs, test.exp) { - t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.runs) + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.runs()) } } } func TestBitmapXorRange(t *testing.T) { - c := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -2063,11 +2027,11 @@ func TestBitmapXorRange(t *testing.T) { } for i, test := range tests { - copy(c.bitmap, test.bitmap) + copy(c.bitmap(), test.bitmap) c.n = c.countRange(0, 65535) c.bitmapXorRange(test.start, test.last+1) - if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } if test.expN != c.n { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) @@ -2076,8 +2040,6 @@ func TestBitmapXorRange(t *testing.T) { } func TestXorBitmapRun(t *testing.T) { - a := &Container{containerType: containerBitmap} - b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -2090,16 +2052,17 @@ func TestXorBitmapRun(t *testing.T) { }, } for i, test := range tests { - a.bitmap = test.bitmap - b.runs = test.runs + a := NewContainerBitmap(0, test.bitmap) + e := NewContainerBitmap(0, test.exp) + b := NewContainerRun(test.runs) //xorBitmapRun ret := xor(a, b) - if !reflect.DeepEqual(ret.bitmap, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap) + if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { + t.Fatalf("test #%v expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } ret = xor(b, a) - if !reflect.DeepEqual(ret.bitmap, test.exp) { - t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.bitmap) + if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { + t.Fatalf("test #%v.1 expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } } @@ -2529,10 +2492,7 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &Container{containerType: containerArray}, &Container{ - containerType: containerBitmap, - bitmap: make([]uint64, bitmapN), - } + a, b := NewContainerArray(nil), NewContainerBitmap(0, nil) tests := []struct { array []uint16 bitmap []uint64 @@ -2576,11 +2536,9 @@ func TestIntersectArrayBitmap(t *testing.T) { } for i, test := range tests { - a.array = test.array - a.containerType = containerArray - copy(b.bitmap, test.bitmap) - b.containerType = containerBitmap - ret := intersectArrayBitmap(a, b).array + a.setArray(test.array) + copy(b.bitmap(), test.bitmap) + ret := intersectArrayBitmap(a, b).array() if len(ret) == 0 && len(test.exp) == 0 { continue } @@ -2717,13 +2675,6 @@ func TestContainerCombinations(t *testing.T) { containerTypes := []byte{containerArray, containerBitmap, containerRun} - // map used for a more descriptive print - cm := map[byte]string{ - containerArray: "array", - containerBitmap: "bitmap", - containerRun: "run", - } - testOps := []testOp{ // intersect {intersect, "empty", "empty", "empty"}, @@ -3295,7 +3246,7 @@ func TestContainerCombinations(t *testing.T) { for _, testOp := range testOps { for _, x := range containerTypes { for _, y := range containerTypes { - desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) + desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), containerTypeNames[x], testOp.x, containerTypeNames[y], testOp.y) ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y]) exp := testOp.exp @@ -3313,8 +3264,8 @@ func TestContainerCombinations(t *testing.T) { } // Because xorRunRun resulting in an empty container returns an array container with a // nil slice array, then we need to check len() on array first (look for 0). - if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { - t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) + if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) { + t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) } } else if ct == containerBitmap { if clone.isArray() { @@ -3325,8 +3276,8 @@ func TestContainerCombinations(t *testing.T) { if clone.n != cts[ct][exp].n { t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) } - if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { - t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) + if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) { + t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) } } else if ct == containerRun { if clone.isArray() { @@ -3337,8 +3288,8 @@ func TestContainerCombinations(t *testing.T) { if clone.n != cts[ct][exp].n { t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) } - if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) { - t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs) + if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) { + t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) } } } @@ -3408,16 +3359,7 @@ func BenchmarkBitmapRepair(b *testing.B) { } func newTestBitmapContainer() *Container { - var ( - buf = make([]uint64, bitmapN) - ob = buf[:bitmapN] - container = &Container{ - bitmap: ob, - n: 0, - containerType: containerBitmap, - } - ) - return container + return NewContainerBitmap(0, nil) } /* @@ -3443,9 +3385,7 @@ func TestEquals(t *testing.T) { } */ func TestShiftArray(t *testing.T) { - a := &Container{ - containerType: containerArray, - } + a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint16 @@ -3469,22 +3409,21 @@ func TestShiftArray(t *testing.T) { } for i, test := range tests { - a.array = test.array - a.n = int32(len(a.array)) + a.setArray(test.array) + a.n = int32(len(a.array())) ret1, _ := shift(a) // test generic shift function ret2, _ := shiftArray(a) // test array-specific shift function - if !reflect.DeepEqual(ret1.array, test.exp) { - t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array) - } else if !reflect.DeepEqual(ret2.array, test.exp) { - t.Fatalf("test #%v shiftArray() expected %v, but got %v", i, test.exp, ret2.array) + if !reflect.DeepEqual(ret1.array(), test.exp) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array()) + } else if !reflect.DeepEqual(ret2.array(), test.exp) { + t.Fatalf("test #%v shiftArray() expected %v, but got %v", i, test.exp, ret2.array()) } } } func TestShiftBitmap(t *testing.T) { - a := &Container{ - containerType: containerBitmap, - } + // note, bitmaps are provided for us by the ensuing tests + a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint64 @@ -3504,21 +3443,20 @@ func TestShiftBitmap(t *testing.T) { } for i, test := range tests { - a.bitmap = test.bitmap + a.setBitmap(test.bitmap) a.n = 1 ret1, _ := shift(a) // test generic shift function ret2, _ := shiftBitmap(a) // test bitmap-specific shift function - if !reflect.DeepEqual(ret1.bitmap, test.exp) { - t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.bitmap) - } else if !reflect.DeepEqual(ret2.bitmap, test.exp) { - t.Fatalf("test #%v shiftBitmap() expected %v, but got %v", i, test.exp, ret2.bitmap) + e := NewContainerBitmap(1, test.exp) + if !reflect.DeepEqual(ret1.bitmap(), e.bitmap()) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, e.bitmap(), ret1.bitmap()) + } else if !reflect.DeepEqual(ret2.bitmap(), test.exp) { + t.Fatalf("test #%v shiftBitmap() expected %v, but got %v", i, e.bitmap(), ret2.bitmap()) } } } func TestShiftRun(t *testing.T) { - a := &Container{ - containerType: containerRun, - } + a := NewContainerRun(nil) tests := []struct { runs []interval16 @@ -3551,14 +3489,14 @@ func TestShiftRun(t *testing.T) { } for i, test := range tests { - a.runs = test.runs + a.setRuns(test.runs) a.n = test.n ret1, c1 := shift(a) // test generic shift function ret2, c2 := shiftRun(a) // test run-specific shift function - if !reflect.DeepEqual(ret1.runs, test.exp) && c1 == test.carry && ret1.n == test.en { - t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs, ret1.n) - } else if !reflect.DeepEqual(ret2.runs, test.exp) && c2 == test.carry && ret2.n == test.en { - t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs, ret2.n) + if !reflect.DeepEqual(ret1.runs(), test.exp) && c1 == test.carry && ret1.n == test.en { + t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs(), ret1.n) + } else if !reflect.DeepEqual(ret2.runs(), test.exp) && c2 == test.carry && ret2.n == test.en { + t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs(), ret2.n) } } } diff --git a/roaring/roaring_nop_paranoia.go b/roaring/roaring_nop_paranoia.go new file mode 100644 index 000000000..ce9af8d54 --- /dev/null +++ b/roaring/roaring_nop_paranoia.go @@ -0,0 +1,5 @@ +// +build !roaringparanoia + +package roaring + +const roaringParanoia = false diff --git a/roaring/roaring_paranoia.go b/roaring/roaring_paranoia.go new file mode 100644 index 000000000..910e1bd40 --- /dev/null +++ b/roaring/roaring_paranoia.go @@ -0,0 +1,5 @@ +// +build roaringparanoia + +package roaring + +const roaringParanoia = true diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index d39103483..9d0b3c8f8 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -435,14 +435,14 @@ func TestBitmap_UnionInPlace1(t *testing.T) { result.UnionInPlace(bm0, bm1) if n := result.Count(); n != 2682675 { - t.Fatalf("unexpected n: %d", n) + t.Fatalf("unexpected n: got %d, expected 2682675", n) } bm := testBM() result = roaring.NewBitmap() result.UnionInPlace(bm, bm0) if n := result.Count(); n != 75009 { - t.Fatalf("unexpected n: %d", n) + t.Fatalf("unexpected n: got %d, expected 75009", n) } result = roaring.NewBitmap()