diff --git a/ctl/check_test.go b/ctl/check_test.go index af223c18d..33a19feb3 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -17,7 +17,6 @@ package ctl import ( "bytes" "encoding/hex" - "golang.org/x/net/context" "io" "io/ioutil" "math/rand" @@ -25,6 +24,8 @@ import ( "path/filepath" "strings" "testing" + + "context" ) func TestCheckCommand_RunCacheFile(t *testing.T) { @@ -84,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) { var buf bytes.Buffer io.Copy(&buf, r) - if err.Error() != "invalid roaring file" { + if !strings.HasPrefix(err.Error(), "invalid roaring file") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) } // Todo: need correct roaring file for happy path diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 000000000..69912cca5 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,21 @@ ++++ +title = "Architecture" ++++ + +## Architecture + +### Roaring bitmap storage format + +Bitmaps are persisted to disk using a file format very similar to the [Roaring Bitmap format spec](https://github.com/RoaringBitmap/RoaringFormatSpec). Pilosa's format uses 64-bit IDs, so it is not binary-compatible with the spec. Some parts of the format are simpler, and an additional section is included. Specific differences include: + +* The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3. +* The cookie includes file format version in bytes 2-3 (currently equal to zero). +* The offset header section is always included. +* RLE runs are serialized as [start, last], not [start, length]. +* After the container storage section is an operation log, of unspecified length. + +![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.svg) + +All values are little-endian. The first two bytes of the cookie is 12346 when the file contains no RLE containers, or 12347 when it does. In the no-RLE case, the runFlagBitset is absent. Otherwise the format is identical in both cases. Container types are determined by their cardinality - a container with 4096 or more values is a bitmap, a container with fewer is an array or RLE container. A high bit in runFlagBitset indicates an RLE container. + +Storing the runFlagBitset in a separate section, indicated by the cookie value, keeps this format backward compatible with older storage versions that do not support RLE containers. diff --git a/fragment.go b/fragment.go index c4fd56764..5458ee34a 100644 --- a/fragment.go +++ b/fragment.go @@ -274,8 +274,7 @@ func (f *Fragment) openCache() error { // Read in all rows by ID. // This will cause them to be added to the cache. for _, id := range pb.IDs { - //n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) - n := f.row(id, true, true).Count() + n := f.row(id, false, false).Count() f.cache.BulkAdd(id, n) } f.cache.Invalidate() diff --git a/fragment_test.go b/fragment_test.go index 5c7ae66cb..702eec70a 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -938,3 +938,30 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { t.Fatalf("unexpected pair(1): %v", pairs[2]) } } + +func TestFragment_Snapshot_Run(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set bits on the fragment. + for i := uint64(1); i < 3; i++ { + if _, err := f.SetBit(1000, i); err != nil { + t.Fatal(err) + } + } + + // Snapshot bitmap and verify data. + if err := f.Snapshot(); err != nil { + t.Fatal(err) + } else if n := f.Row(1000).Count(); n != 2 { + t.Fatalf("unexpected count: %d", n) + } + + // Close and reopen the fragment & verify the data. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if n := f.Row(1000).Count(); n != 2 { + t.Fatalf("unexpected count (reopen): %d", n) + } +} + diff --git a/roaring/internal_test.go b/roaring/internal_test.go index 59fbccf1c..107851489 100644 --- a/roaring/internal_test.go +++ b/roaring/internal_test.go @@ -23,29 +23,29 @@ import ( func TestBitmapIterator(t *testing.T) { for i, tt := range []struct { bitmap []uint64 - values []uint32 + values []uint16 }{ // Empty { bitmap: []uint64{6}, // 0110 - values: []uint32{1, 2}, + values: []uint16{1, 2}, }, // Single uint64 bitmap { bitmap: []uint64{6}, // 0110 - values: []uint32{1, 2}, + values: []uint16{1, 2}, }, // Multi uint64 bitmap { bitmap: []uint64{1 << 63, 1, 0, 1, 3 << 62}, - values: []uint32{63, 64, 192, 318, 319}, + values: []uint16{63, 64, 192, 318, 319}, }, } { itr := newBitmapIterator(tt.bitmap) - var a []uint32 + var a []uint16 for v, eof := itr.next(); !eof; v, eof = itr.next() { a = append(a, v) } diff --git a/roaring/roaring.go b/roaring/roaring.go index 03c4445f9..67799631f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -26,23 +26,45 @@ import ( ) const ( - // cookie is the first four bytes in a roaring bitmap file. - cookie = uint32(12346) + // magicNumber is an identifier, in bytes 0-1 of the file. + magicNumberNoRuns = uint32(12346) + magicNumber = uint32(12347) - // headerSize is the size of the cookie and key count at the beginning of a file. - headerSize = 4 + 4 + // storageVersion indicates the storage version, in bytes 2-3. + storageVersion = uint32(0) + + // cookie is the first four bytes in a roaring bitmap file, + // formed by joining magicNumber and storageVersion + cookieNoRuns = magicNumberNoRuns + storageVersion<<16 + cookie = magicNumber + storageVersion<<16 + + // headerBaseSize is the size in bytes of the cookie and key count at the + // beginning of a file. Headers in files with runs also include + // runFlagBitset, of length (numContainers+7)/8. + headerBaseSize = 4 + 4 + + // runCountHeaderSize is the size in bytes of the run count stored + // at the beginning of every serialized run container. + runCountHeaderSize = 2 + + // interval32Size is the size of a single run in a container.runs. + interval16Size = 4 // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 // manual allocation size tuned to our average client data - manualAlloc = 524288 + manualAlloc = 524288 + ContainerArray = byte(1) + ContainerBitmap = byte(2) + ContainerRun = byte(3) + maxContainerVal = 0xffff ) // Bitmap represents a roaring bitmap. type Bitmap struct { keys []uint64 // keys for containers - containers []*container // array and bitmap containers + containers []*container // array, bitmap and RLE containers // Number of operations written to the writer. opN int @@ -109,10 +131,9 @@ func (b *Bitmap) add(v uint64) bool { // If index is negative then there's not an exact match // and a container needs to be added. if i < 0 { - b.insertAt(hb, newContainer(), -i-1) + b.insertAt(hb, newContainer(), int(-i-1)) i = -i - 1 } - return b.containers[i].add(lowbits(v)) } @@ -176,19 +197,20 @@ func (b *Bitmap) Count() (n uint64) { // CountRange returns the number of bits set between [start, end). func (b *Bitmap) CountRange(start, end uint64) (n uint64) { + i := search64(b.keys, highbits(start)) j := search64(b.keys, highbits(end)) // If range is entirely in one container then just count that range. - if i > 0 && i == j { - return uint64(b.containers[i].countRange(lowbits(start), lowbits(end))) + if i >= 0 && i == j { + return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end)))) } // Count first partial container. if i < 0 { i = -i } else { - n += uint64(b.containers[i].countRange(lowbits(start), (bitmapN*64)+1)) + n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1)) } // Count last container. @@ -198,7 +220,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { j = len(b.containers) } } else { - n += uint64(b.containers[j].countRange(0, lowbits(end))) + n += uint64(b.containers[j].countRange(0, int(lowbits(end)))) } // Count containers in between. @@ -293,39 +315,19 @@ func (b *Bitmap) container(key uint64) *container { return b.containers[i] } -func insertU64(original []uint64, position int, value uint64) []uint64 { - l := len(original) - target := original - if cap(original) == l { - target = make([]uint64, l+1, l+manualAlloc) - copy(target, original[:position]) - } else { - target = append(target, 0) - } - copy(target[position+1:], original[position:]) - target[position] = value - return target -} - -func insertContainer(original []*container, position int, value *container) []*container { - l := len(original) - target := original - if cap(original) == l { - target = make([]*container, l+1, l+manualAlloc) - copy(target, original[:position]) - } else { - target = append(target, nil) - } - copy(target[position+1:], original[position:]) - target[position] = value - return target -} func (b *Bitmap) insertAt(key uint64, c *container, i int) { - b.keys = insertU64(b.keys, i, key) - b.containers = insertContainer(b.containers, i, c) + b.keys = append(b.keys, 0) + copy(b.keys[i+1:], b.keys[i:]) + b.keys[i] = key + + b.containers = append(b.containers, nil) + copy(b.containers[i+1:], b.containers[i:]) + b.containers[i] = c } -// IntersectionCount returns the number of intersections between b and other. +// IntersectionCount returns the number of set bits that would result in an +// intersection between b and other. It is more efficient than actually +// intersecting the two and counting the result. func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { var n uint64 for i, j := 0, 0; i < len(b.containers) && j < len(other.containers); { @@ -335,7 +337,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { } else if ki > kj { j++ } else { - n += intersectionCount(b.containers[i], other.containers[j]) + n += uint64(intersectionCount(b.containers[i], other.containers[j])) i, j = i+1, j+1 } } @@ -437,7 +439,6 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { } } - return output } @@ -504,18 +505,33 @@ func (b *Bitmap) countEmptyContainers() int { return result } +// Optimize converts array and bitmap containers to run containers as necessary. +func (b *Bitmap) Optimize() { + for _, c := range b.containers { + c.Optimize() + } +} + // WriteTo writes b to w. func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { + b.Optimize() // Remove empty containers before persisting. //b.removeEmptyContainers() + containerCount := len(b.keys) - b.countEmptyContainers() + thisCookie := cookieNoRuns + headerSize := headerBaseSize // Build header before writing individual container blocks. - buf := make([]byte, headerSize+(containerCount*(4+8+4))) - binary.LittleEndian.PutUint32(buf[0:], cookie) + // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) + buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) + // Cookie header section. + binary.LittleEndian.PutUint32(buf[0:], thisCookie) binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) + empty := 0 - // Encode keys and cardinality. + // Descriptive header section: encode keys and cardinality. + // Key and cardinality are stored interleaved here, 12 bytes per container. for i, key := range b.keys { c := b.containers[i] @@ -525,23 +541,25 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) if c.n > 0 { binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key)) - binary.LittleEndian.PutUint32(buf[headerSize+(i-empty)*12+8:], uint32(c.n-1)) + binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8:], uint16(c.container_type)) + binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8+2:], uint16(c.n-1)) } else { empty++ } } - // Write the offset for each container block. + // Offset header section: write the offset for each container block. + // 4 bytes per container. offset := uint32(len(buf)) empty = 0 for i, c := range b.containers { if c.n > 0 { binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset)) + offset += uint32(c.size()) } else { empty++ } - offset += uint32(c.size()) } // Write header. @@ -551,7 +569,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { return n, err } - // Write each container block. + // Container storage section: write each container block. for _, c := range b.containers { if c.n > 0 { nn, err := c.WriteTo(w) @@ -567,34 +585,46 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // UnmarshalBinary decodes b from a binary-encoded byte slice. func (b *Bitmap) UnmarshalBinary(data []byte) error { - if len(data) < headerSize { + if len(data) < headerBaseSize { return errors.New("data too small") } - // Verify the first 4 bytes are the correct cookie. - if v := binary.LittleEndian.Uint32(data[0:4]); v != cookie { - return errors.New("invalid roaring file") + // Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion. + fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) + fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) + if fileMagic == magicNumberNoRuns { + // noop + // } else if fileMagic == magicNumber { + // containsRuns = true + } else { + return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } - // Read key count. + if fileVersion != storageVersion { + return fmt.Errorf("wrong roaring version, file is v%d, server requires v%d", fileVersion, storageVersion) + } + + // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[4:8]) b.keys = make([]uint64, keyN) b.containers = make([]*container, keyN) - // Read container key headers. - for i, buf := 0, data[8:]; i < int(keyN); i, buf = i+1, buf[12:] { + headerSize := headerBaseSize + + // Descriptive header section: Read container keys and cardinalities. + for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) b.containers[i] = &container{ - n: int(binary.LittleEndian.Uint32(buf[8:12])) + 1, - mapped: true, + container_type: byte(binary.LittleEndian.Uint16(buf[8:10])), + n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, + mapped: true, } } + opsOffset := headerSize + int(keyN)*12 // Read container offsets and attach data. - opsOffset := 8 + int(keyN)*12 for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { offset := binary.LittleEndian.Uint32(buf[0:4]) - // Verify the offset is within the bounds of the input data. if int(offset) >= len(data) { return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) @@ -602,22 +632,18 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Map byte slice directly to the container data. c := b.containers[i] - if c.n <= ArrayMaxSize { - c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n] - // TODO: instead of commenting this out, we need to make it a configuration option - //for _, v := range c.array { - // assert(lowbits(uint64(v)) == v, "array value out of range: %d", v) - //} - opsOffset = int(offset) + len(c.array)*4 - } else { + switch c.container_type { + case ContainerRun: + runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) + c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount] + opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size + case ContainerArray: + c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] + opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32) + case ContainerBitmap: c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] - opsOffset = int(offset) + len(c.bitmap)*8 + opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64) } - - // Verify container count on load. - // TODO: instead of commenting this out, we need to make it a configuration option - //count := c.count() - //assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n) } // Read ops log until the end of the file. @@ -742,14 +768,14 @@ type BitmapInfo struct { // Iterator represents an iterator over a Bitmap. type Iterator struct { - bitmap *Bitmap - i, j int + bitmap *Bitmap + i, j, k int // i: container; j: array index, bit index, or run index; k: offset within the run } // eof returns true if the iterator is at the end of the bitmap. -func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) } +func (itr *Iterator) eof() bool { return int(itr.i) >= len(itr.bitmap.containers) } -// Seek moves to the first value equal to or greater than v. +// Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { // Move to the correct container. itr.i = search64(itr.bitmap.keys, highbits(seek)) @@ -760,15 +786,19 @@ func (itr *Iterator) Seek(seek uint64) { return } - // Move to the correct value index inside the array container. + // Move to the correct value index inside the container. lb := lowbits(seek) - if c := itr.bitmap.containers[itr.i]; c.isArray() { + if int(itr.i) >= len(itr.bitmap.containers) { + panic(fmt.Sprintf("data Corruption %d %d %d", itr.i, len(itr.bitmap.containers), seek)) + } + c := itr.bitmap.containers[itr.i] + if c.isArray() { // Find index in the container. itr.j = search32(c.array, lb) if itr.j < 0 { itr.j = -itr.j - 1 } - if itr.j < len(c.array) { + if int(itr.j) < len(c.array) { itr.j-- return } @@ -778,6 +808,24 @@ func (itr *Iterator) Seek(seek uint64) { return } + if c.isRun() { + if seek == 0 { + itr.i, itr.j, itr.k = 0, 0, -1 + } + + j, contains := binSearchRuns(lb, c.runs) + if contains { + itr.j = j + itr.k = int(lb) - int(c.runs[j].start) - 1 + } else { + // Set iterator to next value in the Bitmap. + itr.j = j + itr.k = -1 + } + + return + } + // If it's a bitmap container then move to index before the value and call next(). itr.j = int(lb) - 1 } @@ -791,21 +839,51 @@ func (itr *Iterator) Next() (v uint64, eof bool) { return 0, true } - // Move to the next item in the container if it's an array container. c := itr.bitmap.containers[itr.i] if c.isArray() { - if itr.j >= c.n-1 { + if itr.j >= int(c.n-1) { + // Reached end of array, move to the next container. itr.i, itr.j = itr.i+1, -1 continue } itr.j++ return itr.peek(), false } + + if c.isRun() { + // Because itr.j for an array container defaults to -1 + // but defaults to 0 for a run container, we need to + // standardize on treating -1 as our default value for itr.j. + // Note that this is easier than changing the default to 0 + // because the array logic uses the negative number space + // to represent offsets to an array position that isn't filled + // (-1 being the first empty space in an array, or 0). + if itr.j == -1 { + itr.j++ + } + r := c.runs[itr.j] + runLength := int(r.last - r.start) + + if itr.k >= runLength { + // Reached end of run, move to the next run. + itr.j, itr.k = itr.j+1, -1 + } + + if itr.j >= len(c.runs) { + // Reached end of runs, move to the next container. + itr.i, itr.j = itr.i+1, -1 + continue + } + + itr.k++ + return itr.peek(), false + } + // Move to the next possible index in the bitmap container. itr.j++ // Find first non-zero bit in current bitmap, if possible. - hb := int(itr.j / 64) + hb := int(itr.j >> 6) if hb >= len(c.bitmap) { itr.i, itr.j = itr.i+1, -1 @@ -820,7 +898,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Otherwise iterate through remaining bitmaps to find next bit. for hb++; hb < len(c.bitmap); hb++ { if c.bitmap[hb] != 0 { - itr.j = int(hb*64) + trailingZeroN(c.bitmap[hb]) + itr.j = int(hb<<6) + trailingZeroN(c.bitmap[hb]) return itr.peek(), false } } @@ -837,82 +915,62 @@ func (itr *Iterator) peek() uint64 { if c.isArray() { return uint64(key)<<16 | uint64(c.array[itr.j]) } + if c.isRun() { + return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint16(itr.k)) + } return uint64(key)<<16 | uint64(itr.j) } -// BufIterator wraps an iterator to provide the ability to unread values. -type BufIterator struct { - buf struct { - v uint64 - eof bool - full bool - } - itr *Iterator -} - -// NewBufIterator returns a buffered iterator that wraps itr. -func NewBufIterator(itr *Iterator) *BufIterator { - return &BufIterator{itr: itr} -} - -// Seek moves to the first pair equal to or greater than pseek/bseek. -func (itr *BufIterator) Seek(v uint64) { - itr.buf.full = false - itr.itr.Seek(v) -} - -// Next returns the next pair in the bitmap. -// If a value has been buffered then it is returned and the buffer is cleared. -func (itr *BufIterator) Next() (v uint64, eof bool) { - if itr.buf.full { - itr.buf.full = false - return itr.buf.v, itr.buf.eof - } - - // Read value onto buffer in case of unread. - itr.buf.v, itr.buf.eof = itr.itr.Next() - return itr.buf.v, itr.buf.eof -} - -// Peek reads the next value but leaves it on the buffer. -func (itr *BufIterator) Peek() (v uint64, eof bool) { - v, eof = itr.Next() - itr.Unread() - return -} - -// Unread pushes previous pair on to the buffer. -// Panics if the buffer is already full. -func (itr *BufIterator) Unread() { - if itr.buf.full { - panic("roaring.BufIterator: buffer full") - } - itr.buf.full = true -} - // The maximum size of array containers. const ArrayMaxSize = 4096 +// The maximum size of run length encoded containers. +const RunMaxSize = 2048 + // container represents a container for uint32 integers. // -// These are used for storing the low bits. Containers are separated into two +// These are used for storing the low bits. Containers are separated into three // types depending on cardinality. For containers with less than 4,096 values, -// an array container is used. For containers with more than 4,096 values, -// the values are encoded into bitmaps. +// an array or RLE container is used, depending on the contents. For containers +// with more than 4,096 values, the values are encoded into bitmaps. type container struct { - n int // number of integers in container - array []uint32 // used for array containers - bitmap []uint64 // used for bitmap containers - mapped bool // mapped directly to a byte slice when true + container_type byte // number of integers in container + n int // number of integers in container + array []uint16 // used for array containers + bitmap []uint64 // used for bitmap containers + runs []interval16 // used for RLE containers + mapped bool // mapped directly to a byte slice when true +} + +type interval16 struct { + start uint16 + last uint16 +} + +// runlen returns the count of integers in the interval. +func (iv interval16) runlen() int { + return 1 + int(iv.last-iv.start) } // newContainer returns a new instance of container. func newContainer() *container { - return &container{} + return &container{container_type: ContainerArray} } // isArray returns true if the container is an array container. -func (c *container) isArray() bool { return c.bitmap == nil } +func (c *container) isArray() bool { + return c.container_type == ContainerArray +} + +// isBitmap returns true if the container is a bitmap container. +func (c *container) isBitmap() bool { + return c.container_type == ContainerBitmap +} + +// isRun returns true if the container is a run-length-encoded container. +func (c *container) isRun() bool { + return c.container_type == ContainerRun +} // unmap creates copies of the containers data in the heap. // @@ -924,7 +982,7 @@ func (c *container) unmap() { } if c.array != nil { - tmp := make([]uint32, len(c.array)) + tmp := make([]uint16, len(c.array)) copy(tmp, c.array) c.array = tmp } @@ -933,26 +991,33 @@ func (c *container) unmap() { copy(tmp, c.bitmap) c.bitmap = tmp } + if c.runs != nil { + tmp := make([]interval16, len(c.runs)) + copy(tmp, c.runs) + c.runs = tmp + } c.mapped = false } // count counts all bits in the container. func (c *container) count() (n int) { - return c.countRange(0, (bitmapN*64)+1) + return c.countRange(0, maxContainerVal+1) } // countRange counts the number of bits set between [start, end). -func (c *container) countRange(start, end uint32) (n int) { +func (c *container) countRange(start, end int) (n int) { if c.isArray() { return c.arrayCountRange(start, end) + } else if c.isRun() { + return c.runCountRange(start, end) } return c.bitmapCountRange(start, end) } -func (c *container) arrayCountRange(start, end uint32) (n int) { - i := sort.Search(len(c.array), func(i int) bool { return c.array[i] >= start }) +func (c *container) arrayCountRange(start, end int) (n int) { + i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start }) for ; i < len(c.array); i++ { - v := c.array[i] + v := int(c.array[i]) if v >= end { break } @@ -961,19 +1026,18 @@ func (c *container) arrayCountRange(start, end uint32) (n int) { return n } -func (c *container) bitmapCountRange(start, end uint32) int { +func (c *container) bitmapCountRange(start, end int) int { var n uint64 i, j := start/64, end/64 - // Special case when start and end fall in the same word. if i == j { - offi, offj := start%64, 64-end%64 + offi, offj := uint(start%64), uint(64-end%64) n += popcount((c.bitmap[i] >> offi) << (offj + offi)) return int(n) } // Count partial starting word. - if off := start % 64; off != 0 { + if off := uint(start) % 64; off != 0 { n += popcount(c.bitmap[i] >> off) i++ } @@ -985,27 +1049,64 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Count partial ending word. if int(j) < len(c.bitmap) { - off := 64 - (end % 64) + off := 64 - (uint(end) % 64) n += popcount(c.bitmap[j] << off) } return int(n) } -// add adds a value to the container. -func (c *container) add(v uint32) bool { - if c.isArray() { - return c.arrayAdd(v) +func (c *container) runCountRange(start, end int) (n int) { + for _, iv := range c.runs { + // iv is before range + if int(iv.last) < start { + continue + } + // iv is after range + if end < int(iv.start) { + break + } + // iv is superset of range + if int(iv.start) < start && int(iv.last) > end { + return int(end - start) + } + // iv is subset of range + if int(iv.start) >= start && int(iv.last) < end { + n += iv.runlen() + } + // iv overlaps beginning of range + if int(iv.start) < start && int(iv.last) < end { + n += int(iv.last) - start + 1 + } + // iv overlaps end of range + if int(iv.start) > start && int(iv.last) >= end { + n += end - int(iv.start) + } } - return c.bitmapAdd(v) + return n } -func (c *container) arrayAdd(v uint32) bool { +// add adds a value to the container. +func (c *container) add(v uint16) (added bool) { + + if c.isArray() { + added = c.arrayAdd(v) + } else if c.isRun() { + added = c.runAdd(v) + } else { + added = c.bitmapAdd(v) + } + if added { + c.n++ + } + return added +} + +func (c *container) arrayAdd(v uint16) bool { // Optimize appending to the end of an array container. if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v { c.unmap() c.array = append(c.array, v) - c.n++ return true } @@ -1017,7 +1118,7 @@ func (c *container) arrayAdd(v uint32) bool { // Convert to a bitmap container if too many values are in an array container. if c.n >= ArrayMaxSize { - c.convertToBitmap() + c.arrayToBitmap() return c.bitmapAdd(v) } @@ -1027,89 +1128,256 @@ func (c *container) arrayAdd(v uint32) bool { c.array = append(c.array, 0) copy(c.array[i+1:], c.array[i:]) c.array[i] = v - c.n++ return true + } -func (c *container) bitmapAdd(v uint32) bool { +func (c *container) bitmapAdd(v uint16) bool { if c.bitmapContains(v) { return false } c.unmap() c.bitmap[v/64] |= (1 << uint64(v%64)) - c.n++ + return true +} + +func (c *container) runAdd(v uint16) bool { + if len(c.runs) == 0 { + c.unmap() + c.runs = []interval16{{start: v, last: v}} + return true + } + i := 0 + var iv interval16 + for i, iv = range c.runs { + if iv.last >= v { + break + } + } + if v >= iv.start && iv.last >= v { + return false + } + c.unmap() + if iv.last < v { + if iv.last == v-1 { + c.runs[i].last += 1 + } else { + c.runs = append(c.runs, interval16{start: v, last: v}) + } + } else if v+1 == iv.start { + // combining two intervals + if i > 0 && c.runs[i-1].last == v-1 { + c.runs[i-1].last = iv.last + c.runs = append(c.runs[:i], c.runs[i+1:]...) + return true + } + // just before an interval + c.runs[i].start -= 1 + } else if i > 0 && v-1 == c.runs[i-1].last { + // just after an interval + c.runs[i-1].last += 1 + } else { + // alone + newIv := interval16{start: v, last: v} + c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...) + } return true } // contains returns true if v is in the container. -func (c *container) contains(v uint32) bool { +func (c *container) contains(v uint16) bool { if c.isArray() { return c.arrayContains(v) + } else if c.isRun() { + return c.runContains(v) + } else { + return c.bitmapContains(v) } - return c.bitmapContains(v) } -func (c *container) arrayContains(v uint32) bool { +func (c *container) bitmapCountRuns() (r int) { + for i := 0; i < 1023; i++ { + v, v1 := c.bitmap[i], c.bitmap[i+1] + r = r + int(popcnt((v<<1)&^v)+((v>>63)&^v1)) + } + vl := c.bitmap[len(c.bitmap)-1] + r = r + int(popcnt((vl<<1)&^vl)+vl>>63) + return r +} + +func (c *container) arrayCountRuns() (r int) { + prev := -2 + for _, v := range c.array { + if prev+1 != int(v) { + r += 1 + } + prev = int(v) + } + return r +} + +func (c *container) countRuns() (r int) { + if c.isArray() { + return c.arrayCountRuns() + } else if c.isBitmap() { + return c.bitmapCountRuns() + } else if c.isRun() { + return len(c.runs) + } + + // sure hope this never happens + return 0 +} + +// Optimize converts the container to the type which will take up the least +// amount of space. +func (c *container) Optimize() { + if c.n == 0 { + return + } + runs := c.countRuns() + + var newType byte + if runs <= RunMaxSize && runs <= c.n/2 { + newType = ContainerRun + } else if c.n < ArrayMaxSize { + newType = ContainerArray + } else { + newType = ContainerBitmap + } + + // Then convert accordingly. + if c.isArray() { + if newType == ContainerBitmap { + c.arrayToBitmap() + } else if newType == ContainerRun { + c.arrayToRun() + } + } else if c.isBitmap() { + if newType == ContainerArray { + c.bitmapToArray() + } else if newType == ContainerRun { + c.bitmapToRun() + } + } else if c.isRun() { + if newType == ContainerBitmap { + c.runToBitmap() + } else if newType == ContainerArray { + c.runToArray() + } + } +} + +func (c *container) arrayContains(v uint16) bool { return search32(c.array, v) >= 0 } -func (c *container) bitmapContains(v uint32) bool { +func (c *container) bitmapContains(v uint16) bool { return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0 } -// remove adds a value to the container. -func (c *container) remove(v uint32) bool { - if c.isArray() { - return c.arrayRemove(v) +// binSearchRuns returns the index of the run containing v, and true, when v is contained; +// or the index of the next run starting after v, and false, when v is not contained. +func binSearchRuns(v uint16, a []interval16) (int, bool) { + i := sort.Search(len(a), + func(i int) bool { return a[i].last >= v }) + if i < len(a) { + return i, (v >= a[i].start) && (v <= a[i].last) } - return c.bitmapRemove(v) + + return i, false } -func (c *container) arrayRemove(v uint32) bool { +// runContains determines if v is in the container assuming c is a run +// container. +func (c *container) runContains(v uint16) bool { + _, found := binSearchRuns(v, c.runs) + return found +} + +// remove removes a value from the container. +func (c *container) remove(v uint16) (removed bool) { + if c.isArray() { + removed = c.arrayRemove(v) + } else if c.isRun() { + removed = c.runRemove(v) + } else { + removed = c.bitmapRemove(v) + } + if removed { + c.n-- + } + return removed +} + +func (c *container) arrayRemove(v uint16) bool { i := search32(c.array, v) if i < 0 { return false } c.unmap() - c.n-- c.array = append(c.array[:i], c.array[i+1:]...) return true } -func (c *container) bitmapRemove(v uint32) bool { +func (c *container) bitmapRemove(v uint16) bool { if !c.bitmapContains(v) { return false } c.unmap() // Lower count and remove element. - c.n-- - c.bitmap[v/64] &^= (uint64(1) << (v % 64)) + // c.n-- // TODO removed this - test it + c.bitmap[v/64] &^= (uint64(1) << uint(v%64)) // Convert to array if we go below the threshold. if c.n == ArrayMaxSize { - c.convertToArray() + c.bitmapToArray() + } + return true +} + +// runRemove removes v from a run container, and returns true if v was removed. +func (c *container) runRemove(v uint16) bool { + i, contains := binSearchRuns(v, c.runs) + if !contains { + return false + } + c.unmap() + if v == c.runs[i].last && v == c.runs[i].start { + c.runs = append(c.runs[:i], c.runs[i+1:]...) + } else if v == c.runs[i].last { + c.runs[i].last -= 1 + } else if v == c.runs[i].start { + c.runs[i].start += 1 + } else if v > c.runs[i].start { + last := c.runs[i].last + c.runs[i].last = v - 1 + c.runs = append(c.runs[:i+1], append([]interval16{{start: v + 1, last: last}}, c.runs[i+1:]...)...) } return true } // max returns the maximum value in the container. -func (c *container) max() uint32 { +func (c *container) max() uint16 { if c.isArray() { return c.arrayMax() + } else if c.isRun() { + return c.runMax() + } else { + return c.bitmapMax() } - return c.bitmapMax() } -func (c *container) arrayMax() uint32 { +func (c *container) arrayMax() uint16 { if len(c.array) == 0 { - return 0 //probably hiding some ugly bug but it prevents a crash + return 0 // probably hiding some ugly bug but it prevents a crash } return c.array[len(c.array)-1] } -func (c *container) bitmapMax() uint32 { +func (c *container) bitmapMax() uint16 { // Search bitmap in reverse order. for i := len(c.bitmap) - 1; i >= 0; i-- { // If value is zero then skip. @@ -1119,22 +1387,38 @@ func (c *container) bitmapMax() uint32 { } // Find the highest set bit. - for j := uint32(63); j >= 0; j-- { + for j := uint16(63); j >= 0; j-- { if v&(1< 1 { + // if current-previous > 1, one run ends and another begins + c.runs = append(c.runs, interval16{start, c.array[i]}) + start = v + } + } + // append final run + c.runs = append(c.runs, interval16{start, c.array[c.n-1]}) + c.array = nil + c.mapped = false +} + +// runToArray converts from RLE format to array format. +func (c *container) runToArray() { + c.container_type = ContainerArray + c.array = make([]uint16, 0, c.n) + + // return early if empty + if c.n == 0 { + c.runs = nil + c.mapped = false + return + } + + for _, r := range c.runs { + for v := int(r.start); v <= int(r.last); v++ { + c.array = append(c.array, uint16(v)) + } + } + c.runs = nil + c.mapped = false +} + // clone returns a copy of c. func (c *container) clone() *container { - other := &container{n: c.n} + other := &container{n: c.n, container_type: c.container_type} if c.array != nil { - other.array = make([]uint32, len(c.array)) + other.array = make([]uint16, len(c.array)) copy(other.array, c.array) } @@ -1166,6 +1586,11 @@ func (c *container) clone() *container { copy(other.bitmap, c.bitmap) } + if c.runs != nil { + other.runs = make([]interval16, len(c.runs)) + copy(other.runs, c.runs) + } + return other } @@ -1173,8 +1598,11 @@ func (c *container) clone() *container { func (c *container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { return c.arrayWriteTo(w) + } else if c.isRun() { + return c.runWriteTo(w) + } else { + return c.bitmapWriteTo(w) } - return c.bitmapWriteTo(w) } func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { @@ -1184,25 +1612,43 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { // Verify all elements are valid. // TODO: instead of commenting this out, we need to make it a configuration option - //for _, v := range c.array { - // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) - //} + for _, v := range c.array { + assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) + } - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n]) + // Write sizeof(uint32) * cardinality bytes. + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) return int64(nn), err } func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) { + // Write sizeof(uint64) * bitmapN bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)]) return int64(nn), err } +func (c *container) runWriteTo(w io.Writer) (n int64, err error) { + if len(c.runs) == 0 { + return 0, nil + } + // Write sizeof(interval16) * runCount bytes. + err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs))) + if err != nil { + return 0, err + } + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval16Size*len(c.runs)]) + return int64(runCountHeaderSize + nn), err +} + // size returns the encoded size of the container, in bytes. func (c *container) size() int { if c.isArray() { - return len(c.array) * 4 + return len(c.array) * 2 // sizeof(uint16) + } else if c.isRun() { + return len(c.runs)*interval16Size + runCountHeaderSize + } else { + return len(c.bitmap) * 8 // sizeof(uint64) } - return len(c.bitmap) * 8 } // info returns the current stats about the container. @@ -1211,15 +1657,20 @@ func (c *container) info() ContainerInfo { if c.isArray() { info.Type = "array" - info.Alloc = len(c.array) * 4 + info.Alloc = len(c.array) * 2 // sizeof(uint16) + } else if c.isRun() { + info.Type = "run" + info.Alloc = len(c.runs)*interval16Size + runCountHeaderSize } else { info.Type = "bitmap" - info.Alloc = len(c.bitmap) * 8 + info.Alloc = len(c.bitmap) * 8 // sizeof(uint64) } if c.mapped { if c.isArray() { info.Pointer = unsafe.Pointer(&c.array[0]) + } else if c.isRun() { + info.Pointer = unsafe.Pointer(&c.runs[0]) } else { info.Pointer = unsafe.Pointer(&c.bitmap[0]) } @@ -1232,14 +1683,24 @@ func (c *container) info() ContainerInfo { func (c *container) check() error { var a ErrorList - if c.n <= ArrayMaxSize { - if len(c.array) != c.n { + if c.isArray() { + if len(c.array) != int(c.n) { a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n)) } - } else { - if n := c.bitmapCountRange(0, uint32(len(c.bitmap)*64)); n != c.n { + } else if c.isRun() { + n := c.runCountRange(0, maxContainerVal+1) + if n != c.n { + a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.n)) + } + } else if c.isBitmap() { + if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.n { a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.n)) } + } else { + a.Append(fmt.Errorf("empty container")) + if c.n != 0 { + a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.n)) + } } if a == nil { @@ -1251,29 +1712,41 @@ func (c *container) check() error { // ContainerInfo represents a point-in-time snapshot of container stats. type ContainerInfo struct { Key uint64 // container key - Type string // container type (array or bitmap) + Type string // container type (array, bitmap, or run) N int // number of bits Alloc int // memory used Pointer unsafe.Pointer // offset within the mmap } -func intersectionCount(a, b *container) uint64 { +func intersectionCount(a, b *container) int { if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) + } else if b.isRun() { + return intersectionCountArrayRun(a, b) } else { return intersectionCountArrayBitmap(a, b) } + } else if a.isRun() { + if b.isArray() { + return intersectionCountArrayRun(b, a) + } else if b.isRun() { + return intersectionCountRunRun(a, b) + } else { + return intersectionCountBitmapRun(b, a) + } } else { if b.isArray() { return intersectionCountArrayBitmap(b, a) + } else if b.isRun() { + return intersectionCountBitmapRun(a, b) } else { return intersectionCountBitmapBitmap(a, b) } } } -func intersectionCountArrayArray(a, b *container) (n uint64) { +func intersectionCountArrayArray(a, b *container) (n int) { na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1289,6 +1762,60 @@ func intersectionCountArrayArray(a, b *container) (n uint64) { return n } +func intersectionCountArrayRun(a, b *container) (n int) { + na, nb := len(a.array), len(b.runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := a.array[i], b.runs[j] + if va < vb.start { + i++ + } else if va >= vb.start && va <= vb.last { + i++ + n++ + } else if va > vb.last { + j++ + } + } + return n +} + +func intersectionCountRunRun(a, b *container) (n int) { + na, nb := len(a.runs), len(b.runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := a.runs[i], b.runs[j] + if va.last < vb.start { + // |--va--| |--vb--| + i++ + } else if va.start > vb.last { + // |--vb--| |--va--| + j++ + } else if va.last > vb.last && va.start >= vb.start { + // |--vb-|-|-va--| + n += 1 + int(vb.last-va.start) + j++ + } else if va.last > vb.last && va.start < vb.start { + // |--va|--vb--|--| + n += 1 + int(vb.last-vb.start) + j++ + } else if va.last <= vb.last && va.start >= vb.start { + // |--vb|--va--|--| + n += 1 + int(va.last-va.start) + i++ + } else if va.last <= vb.last && va.start < vb.start { + // |--va-|-|-vb--| + n += 1 + int(va.last-vb.start) + i++ + } + } + return +} + +func intersectionCountBitmapRun(a, b *container) (n int) { + for _, iv := range b.runs { + n += a.bitmapCountRange(int(iv.start), int(iv.last)+1) + } + return n +} + func intersectionCountArrayBitmapOld(a, b *container) (n uint64) { // Copy array header so we can shrink it. array := a.array @@ -1297,7 +1824,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) { } // Iterate over bitmap and find matching bits. - for i, bn := uint32(0), uint32(len(b.bitmap)); i < bn; i++ { + for i, bn := uint16(0), uint16(len(b.bitmap)); i < bn; i++ { v := b.bitmap[i] // Ignore if bytes are empty or array is done. @@ -1306,7 +1833,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) { } // Check each bit. - for j := uint32(0); j < 64; j++ { + for j := uint16(0); j < 64; j++ { if v&(1<= uint32(len(b.bitmap)) { + i := val >> 6 + if i >= uint16(len(b.bitmap)) { break } off := val % 64 - n += (b.bitmap[i] & (1 << off)) >> off + n += int((b.bitmap[i] & (1 << off)) >> off) } return n } -func intersectionCountBitmapBitmap(a, b *container) (n uint64) { - return popcntAndSlice(a.bitmap, b.bitmap) +func intersectionCountBitmapBitmap(a, b *container) (n int) { + return int(popcntAndSlice(a.bitmap, b.bitmap)) } func intersect(a, b *container) *container { if a.isArray() { if b.isArray() { return intersectArrayArray(a, b) + } else if b.isRun() { + return intersectArrayRun(a, b) } else { return intersectArrayBitmap(a, b) } + } else if a.isRun() { + if b.isArray() { + return intersectArrayRun(b, a) + } else if b.isRun() { + return intersectRunRun(a, b) + } else { + return intersectBitmapRun(b, a) + } } else { if b.isArray() { return intersectArrayBitmap(b, a) + } else if b.isRun() { + return intersectBitmapRun(a, b) } else { return intersectBitmapBitmap(a, b) } @@ -1364,7 +1903,7 @@ func intersect(a, b *container) *container { } func intersectArrayArray(a, b *container) *container { - output := &container{} + output := &container{container_type: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1381,50 +1920,169 @@ func intersectArrayArray(a, b *container) *container { return output } +// intersectArrayRun computes the intersect of an array container and a run +// container. The return is always an array container (since it's guaranteed to +// be low-cardinality) +func intersectArrayRun(a, b *container) *container { + output := &container{container_type: ContainerArray} + na, nb := len(a.array), len(b.runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := a.array[i], b.runs[j] + if va < vb.start { + i++ + } else if va > vb.last { + j++ + } else { + output.array = append(output.array, va) + i++ + } + } + output.n = len(output.array) + return output +} + +// intersectRunRun computes the intersect of two run containers. +func intersectRunRun(a, b *container) *container { + output := &container{container_type: ContainerRun} + na, nb := len(a.runs), len(b.runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := a.runs[i], b.runs[j] + if va.last < vb.start { + // |--va--| |--vb--| + i++ + } else if vb.last < va.start { + // |--vb--| |--va--| + j++ + } else if va.last > vb.last && va.start >= vb.start { + // |--vb-|-|-va--| + output.n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) + j++ + } else if va.last > vb.last && va.start < vb.start { + // |--va|--vb--|--| + output.n += output.runAppendInterval(vb) + j++ + } else if va.last <= vb.last && va.start >= vb.start { + // |--vb|--va--|--| + output.n += output.runAppendInterval(va) + i++ + } else if va.last <= vb.last && va.start < vb.start { + // |--va-|-|-vb--| + output.n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) + i++ + } + } + if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +// intersectBitmapRun returns an array container if the run container's +// cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container. +func intersectBitmapRun(a, b *container) *container { + var output *container + if b.n < ArrayMaxSize { + // output is array container + output = &container{container_type: ContainerArray} + for _, iv := range b.runs { + for i := iv.start; i <= iv.last; i++ { + if a.bitmapContains(i) { + output.array = append(output.array, i) + } + } + } + output.n = len(output.array) + } else { + // right now this iterates through the runs and sets integers in the + // bitmap that are in the runs. alternately, we could zero out ranges in + // the bitmap which are between runs. + output = &container{ + bitmap: make([]uint64, bitmapN), + container_type: ContainerBitmap, + } + for j := 0; j < len(b.runs); j++ { + vb := b.runs[j] + i := vb.start >> 6 // index into a + vastart := i << 6 + valast := vastart + 63 + for valast >= vb.start && vastart <= vb.last && i < bitmapN { + if vastart >= vb.start && valast <= vb.last { // a within b + output.bitmap[i] = a.bitmap[i] + output.n += int(popcnt(a.bitmap[i])) + } else if vb.start >= vastart && vb.last <= valast { // b within a + var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) + bits := a.bitmap[i] & mask + output.bitmap[i] |= bits + output.n += int(popcnt(bits)) + } else if vastart < vb.start { // a overlaps front of b + offset := 64 - (1 + valast - vb.start) + bits := (a.bitmap[i] >> offset) << offset + output.bitmap[i] |= bits + output.n += int(popcnt(bits)) + } else if vb.start < vastart { // b overlaps front of a + offset := 64 - (1 + vb.last - vastart) + bits := (a.bitmap[i] << offset) >> offset + output.bitmap[i] |= bits + output.n += int(popcnt(bits)) + } + // update loop vars + i++ + vastart = i << 6 + valast = vastart + 63 + } + } + if output.n < ArrayMaxSize { + output.bitmapToArray() + } + } + return output +} + func intersectArrayBitmap(a, b *container) *container { - output := &container{} - itr := newBufIterator(newBitmapIterator(b.bitmap)) - for i := 0; i < len(a.array); { - va := a.array[i] - vb, eof := itr.next() - if eof { + output := &container{container_type: ContainerArray} + itra := newArrayIterator(a.array) + itrb := newBitmapIterator(b.bitmap) + va, eof1 := itra.next() + vb, eof2 := itrb.next() + for { + if eof1 || eof2 { break } if va < vb { - i++ - itr.unread() + va, eof1 = itra.next() } else if va > vb { - // nop + vb, eof2 = itrb.next() } else { output.add(va) - i++ + va, eof1 = itra.next() + vb, eof2 = itrb.next() } } return output } func intersectBitmapBitmap(a, b *container) *container { - output := &container{} - itr0 := newBufIterator(newBitmapIterator(a.bitmap)) - itr1 := newBufIterator(newBitmapIterator(b.bitmap)) + output := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + itr0 := newBitmapIterator(a.bitmap) + itr1 := newBitmapIterator(b.bitmap) + va, eof1 := itr0.next() + vb, eof2 := itr1.next() for { - va, eof := itr0.next() - if eof { - break - } - - vb, eof := itr1.next() - if eof { + if eof1 || eof2 { break } if va < vb { - itr1.unread() + va, eof1 = itr0.next() } else if va > vb { - itr0.unread() + vb, eof2 = itr1.next() } else { output.add(va) + va, eof1 = itr0.next() + vb, eof2 = itr1.next() } } return output @@ -1434,12 +2092,24 @@ func union(a, b *container) *container { if a.isArray() { if b.isArray() { return unionArrayArray(a, b) + } else if b.isRun() { + return unionArrayRun(a, b) } else { return unionArrayBitmap(a, b) } + } else if a.isRun() { + if b.isArray() { + return unionArrayRun(b, a) + } else if b.isRun() { + return unionRunRun(a, b) + } else { + return unionBitmapRun(b, a) + } } else { if b.isArray() { return unionArrayBitmap(b, a) + } else if b.isRun() { + return unionBitmapRun(a, b) } else { return unionBitmapBitmap(a, b) } @@ -1447,7 +2117,7 @@ func union(a, b *container) *container { } func unionArrayArray(a, b *container) *container { - output := &container{} + output := &container{container_type: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; ; { if i >= na && j >= nb { @@ -1477,9 +2147,184 @@ func unionArrayArray(a, b *container) *container { return output } +// unionArrayRun optimistically assumes that the result will be a run container, +// and converts to a bitmap or array container afterwards if necessary. +func unionArrayRun(a, b *container) *container { + if b.n == maxContainerVal { + return b.clone() + } + output := &container{container_type: ContainerRun} + na, nb := len(a.array), len(b.runs) + var vb interval16 + var va uint16 + for i, j := 0, 0; i < na || j < nb; { + if i < na { + va = a.array[i] + } + if j < nb { + vb = b.runs[j] + } + if i < na && (j >= nb || va < vb.start) { + output.n += output.runAppendInterval(interval16{start: va, last: va}) + i++ + } else { + output.n += output.runAppendInterval(vb) + j++ + } + } + if output.n < ArrayMaxSize { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +// runAppendInterval adds the given interval to the run container. It assumes +// that the interval comes at the end of the list of runs, and does not check +// that this is the case. It will not behave correctly if the start of the given +// interval is earlier than the start of the last interval in the list of runs. +// Its return value is the amount by which the cardinality of the container was +// increased. +func (c *container) runAppendInterval(v interval16) int { + if len(c.runs) == 0 { + c.runs = append(c.runs, v) + return int(v.last-v.start) + 1 + } else { + last := c.runs[len(c.runs)-1] + if last.last == maxContainerVal { //protect against overflow + return 0 + } + if last.last+1 >= v.start && v.last > last.last { + c.runs[len(c.runs)-1].last = v.last + return int(v.last - last.last) + } else if last.last+1 < v.start { + c.runs = append(c.runs, v) + return int(v.last-v.start) + 1 + } + } + return 0 +} + +func unionRunRun(a, b *container) *container { + if a.n == maxContainerVal { + return a.clone() + } + if b.n == maxContainerVal { + return b.clone() + } + na, nb := len(a.runs), len(b.runs) + output := &container{ + runs: make([]interval16, 0, na+nb), + container_type: ContainerRun, + } + var va, vb interval16 + for i, j := 0, 0; i < na || j < nb; { + if i < na { + va = a.runs[i] + } + if j < nb { + vb = b.runs[j] + } + if i < na && (j >= nb || va.start < vb.start) { + output.n += output.runAppendInterval(va) + i++ + } else { + output.n += output.runAppendInterval(vb) + j++ + } + } + if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +func unionBitmapRun(a, b *container) *container { + if b.n == maxContainerVal { + return b.clone() + } + output := a.clone() + for j := 0; j < len(b.runs); j++ { + output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + } + return output +} + +const maxBitmap = 0xFFFFFFFFFFFFFFFF + +// sets all bits in [i, j) (c must be a bitmap container) +func (c *container) bitmapSetRange(i, j uint64) { + x := i >> 6 + y := (j - 1) >> 6 + var X uint64 = maxBitmap << (i % 64) + var Y uint64 = maxBitmap >> (64 - (j % 64)) + xcnt := popcnt(X) + ycnt := popcnt(Y) + if x == y { + c.n += int((j - i) - popcnt(c.bitmap[x]&(X&Y))) + c.bitmap[x] |= (X & Y) + } else { + c.n += int(xcnt - popcnt(c.bitmap[x]&X)) + c.bitmap[x] |= X + for i := x + 1; i < y; i++ { + c.n += int(64 - popcnt(c.bitmap[i])) + c.bitmap[i] = maxBitmap + } + c.n += int(ycnt - popcnt(c.bitmap[y]&Y)) + c.bitmap[y] |= Y + } +} + +// xor's all bits in [i, j) with all true (c must be a bitmap container). +func (c *container) bitmapXorRange(i, j uint64) { + x := i >> 6 + y := (j - 1) >> 6 + var X uint64 = maxBitmap << (i % 64) + var Y uint64 = maxBitmap >> (64 - (j % 64)) + if x == y { + cnt := popcnt(c.bitmap[x]) + c.bitmap[x] ^= (X & Y) //// flip + c.n += int(popcnt(c.bitmap[x]) - cnt) + } else { + cnt := popcnt(c.bitmap[x]) + c.bitmap[x] ^= X + c.n += int(popcnt(c.bitmap[x]) - cnt) + for i := x + 1; i < y; i++ { + cnt = popcnt(c.bitmap[i]) + c.bitmap[i] ^= maxBitmap + c.n += int(popcnt(c.bitmap[i]) - cnt) + } + cnt = popcnt(c.bitmap[y]) + c.bitmap[y] ^= Y + c.n += int(popcnt(c.bitmap[y]) - cnt) + } +} + +// zeroes all bits in [i, j) (c must be a bitmap container) +func (c *container) bitmapZeroRange(i, j uint64) { + x := i >> 6 + y := (j - 1) >> 6 + var X uint64 = maxBitmap << (i % 64) + var Y uint64 = maxBitmap >> (64 - (j % 64)) + if x == y { + c.n -= int(popcnt(c.bitmap[x] & (X & Y))) + c.bitmap[x] &= ^(X & Y) + } else { + c.n -= int(popcnt(c.bitmap[x] & X)) + c.bitmap[x] &= ^X + for i := x + 1; i < y; i++ { + c.n -= int(popcnt(c.bitmap[i])) + c.bitmap[i] = 0 + } + c.n -= int(popcnt(c.bitmap[y] & Y)) + c.bitmap[y] &= ^Y + } +} + func unionArrayBitmap(a, b *container) *container { - output := &container{} - itr := newBufIterator(newBitmapIterator(b.bitmap)) + output := &container{container_type: ContainerArray} + itr := newBufBitmapIterator(newBitmapIterator(b.bitmap)) for i := 0; ; { vb, eof := itr.next() if i >= len(a.array) && eof { @@ -1510,7 +2355,8 @@ func unionArrayBitmap(a, b *container) *container { func unionBitmapBitmap(a, b *container) *container { output := &container{ - bitmap: make([]uint64, bitmapN), + bitmap: make([]uint64, bitmapN), + container_type: ContainerBitmap, } for i := 0; i < bitmapN; i++ { @@ -1526,20 +2372,33 @@ func difference(a, b *container) *container { if a.isArray() { if b.isArray() { return differenceArrayArray(a, b) + } else if b.isRun() { + return differenceArrayRun(a, b) } else { return differenceArrayBitmap(a, b) } + } else if a.isRun() { + if b.isArray() { + return differenceRunArray(a, b) + } else if b.isRun() { + return differenceRunRun(a, b) + } else { + return differenceRunBitmap(a, b) + } } else { if b.isArray() { return differenceBitmapArray(a, b) + } else if b.isRun() { + return differenceBitmapRun(a, b) } else { return differenceBitmapBitmap(a, b) } } } +// differenceArrayArray computes the difference bween two arrays. func differenceArrayArray(a, b *container) *container { - output := &container{} + output := &container{container_type: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na; { va := a.array[i] @@ -1562,9 +2421,207 @@ func differenceArrayArray(a, b *container) *container { return output } +// differenceArrayRun computes the difference of an array from a run. +func differenceArrayRun(a, b *container) *container { + // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { + + if a.n == 0 || b.n == 0 { + return a.clone() + } + + output := &container{array: make([]uint16, 0, a.n), container_type: ContainerArray} + // cardinality upper bound: card(A) + + i := 0 // array index + j := 0 // run index + + // handle overlap + for i < int(a.n) { + + // keep all array elements before beginning of runs + if a.array[i] < b.runs[j].start { + output.add(a.array[i]) + i++ + continue + } + + // if array element in run, skip it + if a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last { + i++ + continue + } + + // if array element larger than current run, check next run + if a.array[i] > b.runs[j].last { + j++ + if j == len(b.runs) { + break + } + } + } + + if i < len(a.array) { + // keep all array elements after end of runs + output.array = append(output.array, a.array[i:]...) + // TODO: consider handling container.n mutations in one place + // like we do with container.add(). + output.n += int(len(a.array[i:])) + } + return output +} + +// differenceBitmapRun computes the difference of an bitmap from a run. +func differenceBitmapRun(a, b *container) *container { + if a.n == 0 || b.n == 0 { + return a.clone() + } + + output := a.clone() + for j := 0; j < len(b.runs); j++ { + output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + } + return output +} + +// differenceRunArray computes the difference of an run from a array. +func differenceRunArray(a, b *container) *container { + if a.n == 0 || b.n == 0 { + return a.clone() + } + itr := newArrayIterator(b.array) + return differenceRunIterator(a, itr) +} + +// differenceRunBitmap computes the difference of an run from a bitmap. +func differenceRunBitmap(a, b *container) *container { + if a.n == 0 || b.n == 0 { + return a.clone() + } + itr := newBufBitmapIterator(newBitmapIterator(b.bitmap)) + return differenceRunIterator(a, itr) +} + +func differenceRunIterator(a *container, itr containerIterator) *container { + + output := &container{runs: make([]interval16, 0, a.n)} + + vb, eof := itr.next() + j := 0 + vr := a.runs[j] + working := !eof + for working { + switch { + case vb < vr.start: //before + case vb > vr.last: //after + if vr.start <= vr.last { + output.n += output.runAppendInterval(vr) + } + j++ + if j < len(a.runs) { + vr = a.runs[j] + } else { + working = false + } + case vb == vr.start: //begining of run + vr.start++ + case vb == a.runs[j].last: //end of run + vr.last-- + if vr.last >= vr.start { + output.n += output.runAppendInterval(vr) + } + j++ + if j < len(a.runs) { + vr = a.runs[j] + } else { + working = false + } + case vb > vr.start: //inside run + output.n += output.runAppendInterval(interval16{start: vr.start, last: vb - 1}) + vr.start = vb + 1 + + } + vb, eof = itr.next() + if eof { + working = false + } + } + if vr.start <= vr.last { + output.n += output.runAppendInterval(vr) + } + if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +// differenceRunRun computes the difference of two runs. +func differenceRunRun(a, b *container) *container { + if a.n == 0 || b.n == 0 { + return a.clone() + } + + apos := 0 // current a-run index + bpos := 0 // current b-run index + astart := a.runs[apos].start + alast := a.runs[apos].last + bstart := b.runs[bpos].start + blast := b.runs[bpos].last + alen := len(a.runs) + blen := len(b.runs) + + output := &container{runs: make([]interval16, 0, alen+blen)} // TODO allocate max then truncate? or something else + // cardinality upper bound: sum of number of runs + // each B-run could split an A-run in two, up to len(b.runs) times + + for apos < alen && bpos < blen { + switch { + case alast < bstart: + // current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run + output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)}) + apos++ + if apos < alen { + astart = a.runs[apos].start + alast = a.runs[apos].last + } + case blast < astart: + // current B-run entirely preceeds current A-run: advance to next B-run + bpos++ + if bpos < blen { + bstart = b.runs[bpos].start + blast = b.runs[bpos].last + } + default: + // overlap + if astart < bstart { + output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(bstart - 1)}) + } + if alast > blast { + astart = blast + 1 + } else { + apos++ + if apos < alen { + astart = a.runs[apos].start + alast = a.runs[apos].last + } + } + } + } + if apos < alen { + output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)}) + apos++ + if apos < alen { + output.runs = append(output.runs, a.runs[apos:]...) + } + } + + return output +} + func differenceArrayBitmap(a, b *container) *container { - output := &container{} - itr := newBufIterator(newBitmapIterator(b.bitmap)) + output := &container{container_type: ContainerArray} + itr := newBufBitmapIterator(newBitmapIterator(b.bitmap)) for i := 0; i < len(a.array); { va := a.array[i] vb, eof := itr.next() @@ -1588,50 +2645,58 @@ func differenceArrayBitmap(a, b *container) *container { } func differenceBitmapArray(a, b *container) *container { - output := &container{} - itr := newBufIterator(newBitmapIterator(a.bitmap)) - array := b.array + output := &container{container_type: ContainerArray} + itr := newBufBitmapIterator(newBitmapIterator(a.bitmap)) + i := 0 + va, eof := itr.next() for { - va, eof := itr.next() if eof { break } - if len(array) == 0 { + if i >= len(b.array) { output.add(va) + va, eof = itr.next() continue } - vb := array[0] + vb := b.array[i] if va < vb { output.add(va) + va, eof = itr.next() } else if va > vb { - array = array[1:] - itr.unread() + i++ } else { - array = array[1:] + i++ + va, eof = itr.next() } } return output } func differenceBitmapBitmap(a, b *container) *container { - output := &container{} - itr0 := newBufIterator(newBitmapIterator(a.bitmap)) - itr1 := newBufIterator(newBitmapIterator(b.bitmap)) + output := &container{container_type: ContainerArray} + itr0 := newBufBitmapIterator(newBitmapIterator(a.bitmap)) + itr1 := newBufBitmapIterator(newBitmapIterator(b.bitmap)) + v0, eof0 := itr0.next() + v1, eof1 := itr1.next() for { - v0, eof0 := itr0.next() - v1, eof1 := itr1.next() - if eof0 { break } else if eof1 { output.add(v0) - } else if v0 < v1 { + v0, eof0 = itr0.next() + continue + } + if v0 < v1 { output.add(v0) - itr1.unread() + v0, eof0 = itr0.next() } else if v0 > v1 { - itr0.unread() + v1, eof1 = itr1.next() + } else { + v0, eof0 = itr0.next() + v1, eof1 = itr1.next() + } } return output @@ -1641,12 +2706,24 @@ func xor(a, b *container) *container { if a.isArray() { if b.isArray() { return xorArrayArray(a, b) + } else if b.isRun() { + return xorArrayRun(a, b) } else { return xorArrayBitmap(a, b) } + } else if a.isRun() { + if b.isArray() { + return xorArrayRun(b, a) + } else if b.isRun() { + return xorRunRun(a, b) + } else { + return xorBitmapRun(b, a) + } } else { if b.isArray() { return xorArrayBitmap(b, a) + } else if b.isRun() { + return xorBitmapRun(a, b) } else { return xorBitmapBitmap(a, b) } @@ -1654,7 +2731,7 @@ func xor(a, b *container) *container { } func xorArrayArray(a, b *container) *container { - output := &container{} + output := &container{container_type: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { @@ -1693,7 +2770,7 @@ func xorArrayBitmap(a, b *container) *container { } if output.count() < ArrayMaxSize { - output.convertToArray() + output.bitmapToArray() } return output @@ -1701,9 +2778,9 @@ func xorArrayBitmap(a, b *container) *container { func xorBitmapBitmap(a, b *container) *container { output := &container{ - bitmap: make([]uint64, bitmapN), + bitmap: make([]uint64, bitmapN), + container_type: ContainerBitmap, } - for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] output.bitmap[i] = v @@ -1711,7 +2788,7 @@ func xorBitmapBitmap(a, b *container) *container { } if output.count() < ArrayMaxSize { - output.convertToArray() + output.bitmapToArray() } return output } @@ -1785,10 +2862,10 @@ func (op *op) UnmarshalBinary(data []byte) error { func (*op) size() int { return 1 + 8 + 4 } func highbits(v uint64) uint64 { return uint64(v >> 16) } -func lowbits(v uint64) uint32 { return uint32(v & 0xFFFF) } +func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } // search32 returns the index of v in a. -func search32(a []uint32, value uint32) int { +func search32(a []uint16, value uint16) int { // Optimize for elements and the last element. n := len(a) if n == 0 { @@ -1895,6 +2972,34 @@ func popcount(x uint64) (n uint64) { return x >> 56 } +// Returns eof as true if there are no values left in the iterator. +type containerIterator interface { + next() (uint16, bool) +} + +// arrayIterator represents an iterator over container array values. +type arrayIterator struct { + array []uint16 + i int +} + +func newArrayIterator(array []uint16) *arrayIterator { + return &arrayIterator{ + array: array, + i: -1, + } +} + +// next returns the next value in the array. +func (itr *arrayIterator) next() (v uint16, eof bool) { + + itr.i++ + if itr.i >= len(itr.array) { + return 0, true + } + return itr.array[itr.i], false +} + // bitmapIterator represents an iterator over container bitmap values. type bitmapIterator struct { bitmap []uint64 @@ -1910,25 +3015,25 @@ func newBitmapIterator(bitmap []uint64) *bitmapIterator { // next returns the next value in the bitmap. // Returns eof as true if there are no values left in the iterator. -func (itr *bitmapIterator) next() (v uint32, eof bool) { - if itr.i+1 >= len(itr.bitmap)*64 { +func (itr *bitmapIterator) next() (v uint16, eof bool) { + if itr.i+1 >= int(len(itr.bitmap)*64) { return 0, true } itr.i++ // Find first non-zero bit in current bitmap, if possible. - hb := int(itr.i / 64) + hb := int(itr.i >> 6) lb := itr.bitmap[hb] >> (uint(itr.i) % 64) if lb != 0 { itr.i = int(itr.i) + trailingZeroN(lb) - return uint32(itr.i), false + return uint16(itr.i), false } // Otherwise iterate through remaining bitmaps to find next bit. for hb++; hb < len(itr.bitmap); hb++ { if itr.bitmap[hb] != 0 { - itr.i = int(hb*64) + trailingZeroN(itr.bitmap[hb]) - return uint32(itr.i), false + itr.i = int(hb<<6) + trailingZeroN(itr.bitmap[hb]) + return uint16(itr.i), false } } @@ -1938,7 +3043,7 @@ func (itr *bitmapIterator) next() (v uint32, eof bool) { // bufBitmapIterator wraps an iterator to provide the ability to unread values. type bufBitmapIterator struct { buf struct { - v uint32 + v uint16 eof bool full bool } @@ -1946,13 +3051,13 @@ type bufBitmapIterator struct { } // newBufBitmapIterator returns a buffered iterator that wraps a bitmapIterator. -func newBufIterator(itr *bitmapIterator) *bufBitmapIterator { +func newBufBitmapIterator(itr *bitmapIterator) *bufBitmapIterator { return &bufBitmapIterator{itr: itr} } // next returns the next pair in the bitmap. // If a value has been buffered then it is returned and the buffer is cleared. -func (itr *bufBitmapIterator) next() (v uint32, eof bool) { +func (itr *bufBitmapIterator) next() (v uint16, eof bool) { if itr.buf.full { itr.buf.full = false return itr.buf.v, itr.buf.eof @@ -2012,3 +3117,201 @@ func assert(condition bool, format string, a ...interface{}) { panic(fmt.Sprintf(format, a...)) } } + +// xorArrayRun computes the exclusive or of an array and a run container. +func xorArrayRun(a, b *container) *container { + output := &container{container_type: ContainerRun} + na, nb := len(a.array), len(b.runs) + var vb interval16 + var va uint16 + last_i, last_j := -1, -1 + for i, j := 0, 0; i < na || j < nb; { + if i < na && i != last_i { + va = a.array[i] + } + if j < nb && j != last_j { + vb = b.runs[j] + } + last_i = i + last_j = j + + if i < na && (j >= nb || va < vb.start) { //before + output.n += output.runAppendInterval(interval16{start: va, last: va}) + i++ + } else if j < nb && (i >= na || va > vb.last) { //after + output.n += output.runAppendInterval(vb) + j++ + } else if va > vb.start { + if va < vb.last { + output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) + vb.start = va + 1 + i++ + if vb.start > vb.last { + j++ + } + } else if va > vb.last { + output.n += output.runAppendInterval(vb) + j++ + } else { // va == vb.last + vb.last-- + if vb.start < vb.last { + output.n += output.runAppendInterval(vb) + } + j++ + i++ + } + + } else { + vb.start++ + i++ + } + } + if output.n < ArrayMaxSize { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +// xorCompare computes first exclusive run between two runs. +func xorCompare(x *xorstm) (r1 interval16, has_data bool) { + has_data = false + if !x.va_valid || !x.vb_valid { + if x.vb_valid { + x.vb_valid = false + r1 = x.vb + has_data = true + return + } + if x.va_valid { + x.va_valid = false + r1 = x.va + has_data = true + return + } + return + } + + if x.va.last < x.vb.start { //va before + x.va_valid = false + r1 = x.va + has_data = true + } else if x.vb.last < x.va.start { //vb before + x.vb_valid = false + r1 = x.vb + has_data = true + } else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal + x.va_valid = false + x.vb_valid = false + } else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { //vb inside + x.vb_valid = false + if x.va.start != x.vb.start { + r1 = interval16{start: x.va.start, last: x.vb.start - 1} + has_data = true + } + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } + + } else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside + x.va_valid = false + if x.vb.start != x.va.start { + r1 = interval16{start: x.vb.start, last: x.va.start - 1} + has_data = true + } + + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } + + } else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap + x.va_valid = false + r1 = interval16{start: x.va.start, last: x.vb.start - 1} + has_data = true + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } + } else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap + x.vb_valid = false + r1 = interval16{start: x.vb.start, last: x.va.start - 1} + has_data = true + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } + } + return +} + +//stm is state machine used to "xor" iterate over runs. +type xorstm struct { + va_valid, vb_valid bool + va, vb interval16 +} + +// xorRunRun computes the exclusive or of two run containers. +func xorRunRun(a, b *container) *container { + na, nb := len(a.runs), len(b.runs) + if na == 0 { + return b.clone() + } + if nb == 0 { + return a.clone() + } + output := &container{} + + last_i, last_j := -1, -1 + + state := &xorstm{} + + for i, j := 0, 0; i < na || j < nb; { + if i < na && last_i != i { + state.va = a.runs[i] + state.va_valid = true + } + + if j < nb && last_j != j { + state.vb = b.runs[j] + state.vb_valid = true + } + last_i, last_j = i, j + + r1, ok := xorCompare(state) + if ok { + output.n += output.runAppendInterval(r1) + } + if !state.va_valid { + i++ + } + if !state.vb_valid { + j++ + } + + } + + if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +// xorRunRun computes the exclusive or of a bitmap and a run container. +func xorBitmapRun(a, b *container) *container { + output := a.clone() + for j := 0; j < len(b.runs); j++ { + output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + } + + if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index caf760c74..ea4e5f0ef 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -14,13 +14,197 @@ package roaring -import "testing" +import ( + "bytes" + "fmt" + "reflect" + "testing" +) + +// String produces a human viewable string of the contents. +func (iv interval16) String() string { + return fmt.Sprintf("[%d, %d]", iv.start, iv.last) +} + +func (c *container) String() string { + return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.container_type) +} + +func TestRunAppendInterval(t *testing.T) { + a := container{container_type: ContainerRun} + tests := []struct { + base []interval16 + app interval16 + exp int + }{ + { + base: []interval16{}, + app: interval16{start: 22, last: 25}, + exp: 4, + }, + { + base: []interval16{{start: 20, last: 23}}, + app: interval16{start: 22, last: 25}, + exp: 2, + }, + { + base: []interval16{{start: 20, last: 23}}, + app: interval16{start: 21, last: 22}, + exp: 0, + }, + { + base: []interval16{{start: 20, last: 23}}, + app: interval16{start: 19, last: 25}, + exp: 2, // runAppendInterval explicitly does not support intervals whose start is < c.runs[-1].start + }, + } + + for i, test := range tests { + a.runs = test.base + if n := a.runAppendInterval(test.app); n != test.exp { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, n) + } + } + +} + +func TestInterval16RunLen(t *testing.T) { + iv := interval16{start: 7, last: 9} + if iv.runlen() != 3 { + t.Fatalf("should be 3") + } + iv = interval16{start: 7, last: 7} + if iv.runlen() != 1 { + t.Fatalf("should be 1") + } +} + +func TestContainerRunAdd(t *testing.T) { + c := container{runs: make([]interval16, 0), container_type: ContainerRun} + tests := []struct { + op uint16 + exp []interval16 + }{ + {1, []interval16{{start: 1, last: 1}}}, + {2, []interval16{{start: 1, last: 2}}}, + {4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}}, + {3, []interval16{{start: 1, last: 4}}}, + {10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}}, + {7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}}, + {6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}}, + {0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}}, + {8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}}, + } + for _, test := range tests { + c.mapped = true + ret := c.add(test.op) + if !ret { + t.Fatalf("result of adding new bit should be true: %v", c.runs) + } + if !reflect.DeepEqual(c.runs, test.exp) { + t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs, test.op) + } + if c.mapped { + t.Fatalf("container should not be mapped after adding bit %v", test.op) + } + } +} + +func TestContainerRunAdd2(t *testing.T) { + c := container{runs: make([]interval16, 0), container_type: ContainerRun} + ret := c.add(0) + if !ret { + t.Fatalf("result of adding new bit should be true: %v", c.runs) + } + if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) { + t.Fatalf("should have 1 run of length 1, but have %v", c.runs) + } + ret = c.add(0) + if ret { + t.Fatalf("result of adding existing bit should be false: %v", c.runs) + } +} + +func TestRunCountRange(t *testing.T) { + c := container{runs: make([]interval16, 0), container_type: ContainerRun} + cnt := c.runCountRange(2, 9) + if cnt != 0 { + t.Fatalf("should get 0 from empty container, but got: %v", cnt) + } + c.add(5) + c.add(6) + c.add(7) + + cnt = c.runCountRange(2, 9) + if cnt != 3 { + t.Fatalf("should get 3 from interval within range, but got: %v", cnt) + } + + c.add(8) + c.add(9) + c.add(10) + c.add(11) + + cnt = c.runCountRange(6, 8) + if cnt != 2 { + t.Fatalf("should get 2 from range within interval, but got: %v", cnt) + } + + cnt = c.runCountRange(3, 9) + if cnt != 4 { + t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt) + } + + cnt = c.runCountRange(9, 14) + if cnt != 3 { + t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt) + } + + c.add(17) + c.add(18) + c.add(19) + + cnt = c.runCountRange(1, 22) + if cnt != 10 { + t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt) + } + + c.add(13) + c.add(14) + + cnt = c.runCountRange(6, 18) + if cnt != 9 { + t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt) + } +} + +func TestRunContains(t *testing.T) { + c := container{runs: make([]interval16, 0), container_type: ContainerRun} + if c.runContains(5) { + t.Fatalf("empty run container should not contain 5") + } + c.add(5) + if !c.runContains(5) { + t.Fatalf("run container with 5 should contain 5") + } + + c.add(6) + c.add(7) + + c.add(9) + c.add(10) + c.add(11) + + if !c.runContains(10) { + t.Fatalf("run container with 10 in second run should contain 10") + } +} func TestBitmapCountRange(t *testing.T) { - c := container{} + c := container{container_type: ContainerBitmap} tests := []struct { - start uint32 - end uint32 + start int + end int bitmap []uint64 exp int }{ @@ -32,6 +216,7 @@ func TestBitmapCountRange(t *testing.T) { {start: 66, end: 71, bitmap: []uint64{0xF, 0xFFFFFFFFFFFFFF18}, exp: 2}, {start: 63, end: 64, bitmap: []uint64{0x8000000000000000}, exp: 1}, } + for i, test := range tests { c.bitmap = test.bitmap if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp { @@ -40,35 +225,62 @@ func TestBitmapCountRange(t *testing.T) { } } +func TestIntersectionCountArrayBitmap3(t *testing.T) { + a, b := &container{}, &container{} + a.container_type = ContainerBitmap + a.bitmap = getFullBitmap() + a.n = maxContainerVal + 1 + + b.container_type = ContainerBitmap + b.bitmap = getFullBitmap() + b.n = maxContainerVal + 1 + res := intersectBitmapBitmap(a, b) + if res.n != res.count() || res.n != maxContainerVal+1 { + t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + } + + a.bitmapToRun() + res = intersectBitmapRun(b, a) + if res.n != res.count() || res.n != maxContainerVal+1 { + t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + } + b.bitmapToRun() + res = intersectRunRun(a, b) + n := intersectionCountRunRun(a, b) + if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) { + t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + } +} + func TestIntersectionCountArrayBitmap2(t *testing.T) { a, b := &container{}, &container{} tests := []struct { - array []uint32 + array []uint16 bitmap []uint64 - exp uint64 + exp int }{ { - array: []uint32{0}, + array: []uint16{0}, bitmap: []uint64{1}, exp: 1, }, { - array: []uint32{0, 1}, + array: []uint16{0, 1}, bitmap: []uint64{3}, exp: 2, }, { - array: []uint32{64, 128, 129, 2000}, + array: []uint16{64, 128, 129, 2000}, bitmap: []uint64{932421, 2}, exp: 0, }, { - array: []uint32{0, 65, 130, 195}, + array: []uint16{0, 65, 130, 195}, bitmap: []uint64{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, exp: 4, }, { - array: []uint32{63, 120, 543, 639, 12000}, + array: []uint16{63, 120, 543, 639, 12000}, bitmap: []uint64{0x8000000000000000, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000000000000000}, exp: 2, }, @@ -76,11 +288,2025 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { for i, test := range tests { a.array = test.array + a.container_type = ContainerArray b.bitmap = test.bitmap - ret1 := intersectionCountArrayBitmapOld(a, b) + b.container_type = ContainerBitmap + ret1 := int(intersectionCountArrayBitmapOld(a, b)) ret2 := intersectionCountArrayBitmap(a, b) if ret1 != ret2 || ret2 != test.exp { t.Fatalf("test #%v intersectCountArrayBitmap fail orig: %v new: %v exp: %v", i, ret1, ret2, test.exp) } } } + +func TestRunRemove(t *testing.T) { + c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} + tests := []struct { + op uint16 + exp []interval16 + expRet bool + }{ + {2, []interval16{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true}, + {10, []interval16{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true}, + {12, []interval16{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true}, + {13, []interval16{{start: 3, last: 9}, {start: 15, last: 16}}, true}, + {16, []interval16{{start: 3, last: 9}, {start: 15, last: 15}}, true}, + {6, []interval16{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true}, + {8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true}, + {8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, + {1, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, + {44, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false}, + } + + for i, test := range tests { + c.mapped = true + ret := c.remove(test.op) + if ret != test.expRet || !reflect.DeepEqual(c.runs, test.exp) { + t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs) + } + if ret && c.mapped { + t.Fatalf("test #%v container was not unmapped although bit %v was removed", i, test.op) + } + if !ret && !c.mapped { + t.Fatalf("test #%v container was unmapped although bit %v was not removed", i, test.op) + } + } +} + +func TestRunMax(t *testing.T) { + c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} + max := c.max() + if max != 16 { + t.Fatalf("max for %v should be 16", c.runs) + } + + c = container{runs: []interval16{}} + max = c.max() + if max != 0 { + t.Fatalf("max for %v should be 0", c.runs) + } +} + +func TestIntersectionCountArrayRun(t *testing.T) { + a := &container{array: []uint16{1, 5, 10, 11, 12}} + b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + + ret := intersectionCountArrayRun(a, b) + if ret != 3 { + t.Fatalf("count of %v with %v should be 3, but got %v", a.array, b.runs, ret) + } +} + +func TestIntersectionCountBitmapRun(t *testing.T) { + a := &container{bitmap: []uint64{0x8000000000000000}} + b := &container{runs: []interval16{{start: 63, last: 64}}} + + ret := intersectionCountBitmapRun(a, b) + if ret != 1 { + t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) + } + + a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + b = &container{runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + + ret = intersectionCountBitmapRun(a, b) + if ret != 14 { + t.Fatalf("count of %v with %v should be 14, but got %v", a.bitmap, b.runs, ret) + } +} + +func TestIntersectionCountRunRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + aruns []interval16 + bruns []interval16 + exp int + }{ + { + aruns: []interval16{}, + bruns: []interval16{{start: 3, last: 8}}, exp: 0}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 3, last: 8}}, exp: 6}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 1, last: 11}}, exp: 9}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 0, last: 2}}, exp: 1}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 1, last: 10}}, exp: 9}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 5, last: 12}}, exp: 6}, + { + aruns: []interval16{{start: 2, last: 10}}, + bruns: []interval16{{start: 10, last: 99}}, exp: 1}, + { + aruns: []interval16{{start: 2, last: 10}, {start: 44, last: 99}}, + bruns: []interval16{{start: 12, last: 14}}, exp: 0}, + { + aruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}}, + bruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}}, exp: 11}, + { + aruns: []interval16{{start: 8, last: 12}, {start: 15, last: 19}}, + bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, + } + for i, test := range tests { + a.runs = test.aruns + b.runs = test.bruns + ret := intersectionCountRunRun(a, b) + if ret != test.exp { + t.Fatalf("test #%v failed intersecting %v with %v should be %v, but got %v", i, test.aruns, test.bruns, test.exp, ret) + } + } +} + +func TestIntersectArrayRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + array []uint16 + runs []interval16 + exp []uint16 + }{ + { + array: []uint16{1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{{start: 5, last: 10}}, + exp: []uint16{5, 7, 10}, + }, + { + array: []uint16{}, + runs: []interval16{{start: 5, last: 10}}, + exp: []uint16(nil), + }, + { + array: []uint16{1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{}, + exp: []uint16(nil), + }, + { + array: []uint16{0, 1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 7}}, + exp: []uint16{0, 1, 4, 5, 7}, + }, + } + + for i, test := range tests { + a.array = test.array + b.runs = test.runs + ret := intersectArrayRun(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + } + } +} + +func TestIntersectRunRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + aruns []interval16 + bruns []interval16 + exp []interval16 + expN int + }{ + { + aruns: []interval16{}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16(nil), + expN: 0, + }, + { + aruns: []interval16{{start: 5, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 5, last: 10}}, + expN: 6, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 5, last: 5}, {start: 7, last: 10}}, + expN: 5, + }, + { + aruns: []interval16{{start: 20, last: 30}}, + bruns: []interval16{{start: 5, last: 10}, {start: 19, last: 21}}, + exp: []interval16{{start: 20, last: 21}}, + expN: 2, + }, + { + aruns: []interval16{{start: 5, last: 10}}, + bruns: []interval16{{start: 7, last: 12}}, + exp: []interval16{{start: 7, last: 10}}, + expN: 4, + }, + { + aruns: []interval16{{start: 5, last: 12}}, + bruns: []interval16{{start: 7, last: 10}}, + exp: []interval16{{start: 7, last: 10}}, + expN: 4, + }, + } + for i, test := range tests { + a.runs = test.aruns + b.runs = test.bruns + ret := intersectRunRun(a, b) + if ret.n != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + } + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } + +} + +func TestIntersectBitmapRunBitmap(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN)} + b := &container{} + tests := []struct { + bitmap []uint64 + runs []interval16 + exp []uint64 + expN int + }{ + { + bitmap: []uint64{1}, + runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}}, + exp: []uint64{1}, + expN: 1, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}}, + exp: []uint64{2}, + expN: 1, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}}, + exp: []uint64{0xe000000000001C02}, + expN: 7, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}, {start: 61, last: 77}}, + exp: []uint64{0xE000000000000002, 0x00000000000003FFF}, + expN: 18, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1}, + runs: []interval16{{start: 63, last: 10000}}, + exp: []uint64{0x8000000000000000, 1, 1, 1, 0xA, 1, 1, 0, 1}, + expN: 9, + }, + } + for i, test := range tests { + for i, v := range test.bitmap { + a.bitmap[i] = v + } + b.runs = test.runs + b.n = 4097 // ;) + exp := make([]uint64, bitmapN) + for i, v := range test.exp { + exp[i] = v + } + ret := intersectBitmapRun(a, b) + if ret.isArray() { + ret.arrayToBitmap() + } + if !reflect.DeepEqual(ret.bitmap, exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap) + } + if ret.n != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + } + } + +} + +func TestIntersectBitmapRunArray(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN)} + b := &container{} + tests := []struct { + bitmap []uint64 + runs []interval16 + exp []uint16 + expN int + }{ + { + bitmap: []uint64{1}, + runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}}, + exp: []uint16{0}, + expN: 1, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}}, + exp: []uint16{1}, + expN: 1, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}}, + exp: []uint16{1, 10, 11, 12, 61, 62, 63}, + expN: 7, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 1, last: 1}, {start: 61, last: 68}}, + exp: []uint16{1, 61, 62, 63, 64, 65, 66, 67, 68}, + expN: 9, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1}, + runs: []interval16{{start: 63, last: 10000}}, + exp: []uint16{63, 64, 128, 192, 257, 259, 320, 384, 512}, + expN: 9, + }, + } + for i, test := range tests { + for i, v := range test.bitmap { + a.bitmap[i] = v + } + b.runs = test.runs + ret := intersectBitmapRun(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + } + if ret.n != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + } + } + +} + +func TestUnionMixed(t *testing.T) { + a := &container{} + b := &container{} + c := &container{} + + a.runs = []interval16{{start: 5, last: 10}} + a.container_type = ContainerRun + a.n = 6 + + b.array = []uint16{1, 4, 5, 7, 10, 11, 12} + b.container_type = ContainerArray + b.n = 7 + res := union(a, b) + if !reflect.DeepEqual(res.array, []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array) + } + res = union(b, a) + if !reflect.DeepEqual(res.array, []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) { + t.Fatalf("test #2 expected %v, but got %v", []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array) + } + + res = union(a, a) + if !reflect.DeepEqual(res.runs, []interval16{{start: 5, last: 10}}) { + t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs) + } + c.bitmap = []uint64{0x3} + c.n = 2 + c.container_type = ContainerBitmap + + res = union(c, a) + if !reflect.DeepEqual(res.bitmap, []uint64{2019}) { + t.Fatalf("test #4 expected %v, but got %v", []uint64{2019}, res.bitmap) + } + res = union(a, c) + if !reflect.DeepEqual(res.bitmap, []uint64{2019}) { + t.Fatalf("test #5 expected %v, but got %v", []uint64{2019}, res.bitmap) + } + + res = union(b, c) + if !reflect.DeepEqual(res.array, []uint16{0, 1, 4, 5, 7, 10, 11, 12}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 1, 4, 5, 7, 10, 11, 12}, res.array) + } + res = union(c, b) + if !reflect.DeepEqual(res.array, []uint16{0, 1, 4, 5, 7, 10, 11, 12}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 1, 4, 5, 7, 10, 11, 12}, res.array) + } + +} +func TestIntersectMixed(t *testing.T) { + a := &container{} + b := &container{} + c := &container{} + + a.runs = []interval16{{start: 5, last: 10}} + a.n = 6 + a.container_type = ContainerRun + b.array = []uint16{1, 4, 5, 7, 10, 11, 12} + b.n = 7 + b.container_type = ContainerArray + res := intersect(a, b) + if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) + } + res = intersect(b, a) + if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) + } + + res = intersect(a, a) + if !reflect.DeepEqual(res.runs, []interval16{{start: 5, last: 10}}) { + t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs) + } + c.bitmap = []uint64{0x60} + c.n = 2 + c.container_type = ContainerBitmap + + res = intersect(c, a) + if !reflect.DeepEqual(res.array, []uint16{5, 6}) { + t.Fatalf("test #4 expected %v, but got %v", []uint16{6}, res.array) + } + + res = intersect(a, c) + if !reflect.DeepEqual(res.array, []uint16{5, 6}) { + t.Fatalf("test #5 expected %v, but got %v", []uint16{6}, res.array) + } + + res = intersect(b, c) + if !reflect.DeepEqual(res.array, []uint16{5}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{5}, res.array) + } + res = intersect(c, b) + if !reflect.DeepEqual(res.array, []uint16{5}) { + t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array) + } + +} +func TestDifferenceMixed(t *testing.T) { + a := &container{} + b := &container{} + c := &container{} + d := &container{} + + a.runs = []interval16{{start: 5, last: 10}} + a.n = a.runCountRange(0, 100) + a.container_type = ContainerRun + + b.array = []uint16{0, 2, 4, 6, 8, 10, 12} + b.n = len(b.array) + b.container_type = ContainerArray + + d.array = []uint16{1, 3, 5, 7, 9, 11, 12} + d.n = len(d.array) + d.container_type = ContainerArray + + res := difference(a, b) + + if !reflect.DeepEqual(res.array, []uint16{5, 7, 9}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 9}, res.array) + } + + res = difference(b, a) + if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 12}) { + t.Fatalf("test #2 expected %v, but got %v", []uint16{0, 2, 4, 12}, res.array) + } + + res = difference(a, a) + if !reflect.DeepEqual(res.runs, []interval16{}) { + t.Fatalf("test #3 expected empty but got %v", res.runs) + } + + c.bitmap = []uint64{0x64} + c.n = c.countRange(0, 100) + c.container_type = ContainerBitmap + res = difference(c, a) + if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) { + t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) + } + + res = difference(a, c) + if !reflect.DeepEqual(res.runs, []interval16{{start: 7, last: 10}}) { + t.Fatalf("test #5 expected %v, but got %v", []interval16{{start: 7, last: 10}}, res.runs) + } + + res = difference(b, c) + if !reflect.DeepEqual(res.array, []uint16{0, 4, 8, 10, 12}) { + t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 4, 8, 10, 12}, res.array) + } + + res = difference(c, b) + if !reflect.DeepEqual(res.array, []uint16{5}) { + t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array) + } + + res = difference(b, b) + if res.n != 0 { + t.Fatalf("test #8 expected 0, but got %d", res.n) + } + + res = difference(c, c) + if res.n != 0 { + t.Fatalf("test #9 expected 0, but got %d", res.n) + } + + res = difference(d, b) + if !reflect.DeepEqual(res.array, []uint16{1, 3, 5, 7, 9, 11}) { + t.Fatalf("test #10 expected %v, but got %d", []uint16{1, 3, 5, 7, 9, 11}, res.array) + } + + res = difference(b, d) + if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 6, 8, 10}) { + t.Fatalf("test #11 expected %v, but got %d", []uint16{0, 2, 4, 6, 8, 10}, res.array) + } + +} + +func TestUnionRunRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + aruns []interval16 + bruns []interval16 + exp []interval16 + }{ + { + aruns: []interval16{}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 5, last: 10}}, + }, + { + aruns: []interval16{{start: 5, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 5, last: 12}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 1, last: 3}, {start: 5, last: 12}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + bruns: []interval16{{start: 2, last: 65535}}, + exp: []interval16{{start: 1, last: 65535}}, + }, + { + aruns: []interval16{{start: 2, last: 65535}}, + bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + exp: []interval16{{start: 1, last: 65535}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + bruns: []interval16{{start: 0, last: 65535}}, + exp: []interval16{{start: 0, last: 65535}}, + }, + { + aruns: []interval16{{start: 0, last: 65535}}, + bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}}, + exp: []interval16{{start: 0, last: 65535}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}}, + bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, + exp: []interval16{{start: 1, last: 9}, {start: 12, last: 27}, {start: 33, last: 34}}, + }, + } + for i, test := range tests { + a.runs = test.aruns + b.runs = test.bruns + ret := unionRunRun(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } +} + +func TestUnionArrayRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + array []uint16 + runs []interval16 + exp []uint16 + }{ + { + array: []uint16{1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{{start: 5, last: 10}}, + exp: []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + }, + { + array: []uint16{}, + runs: []interval16{{start: 5, last: 10}}, + exp: []uint16{5, 6, 7, 8, 9, 10}, + }, + { + array: []uint16{1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{}, + exp: []uint16{1, 4, 5, 7, 10, 11, 12}, + }, + { + array: []uint16{0, 1, 4, 5, 7, 10, 11, 12}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 7}}, + exp: []uint16{0, 1, 2, 3, 4, 5, 7, 10, 11, 12}, + }, + } + + for i, test := range tests { + a.array = test.array + b.runs = test.runs + ret := unionArrayRun(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + } + } +} + +func TestBitmapSetRange(t *testing.T) { + c := &container{bitmap: make([]uint64, bitmapN)} + tests := []struct { + bitmap []uint64 + start uint64 + last uint64 + exp []uint64 + expN int + }{ + { + bitmap: []uint64{0x0000000000FFF900}, + start: 9, + last: 10, + exp: []uint64{0x0000000000FFFF00}, + expN: 16, + }, + { + bitmap: []uint64{0xFF0, 0xFF, 0xFF}, + start: 60, + last: 130, + exp: []uint64{0xF000000000000FF0, 0xFFFFFFFFFFFFFFFF, 0xFF}, + expN: 84, + }, + } + + for i, test := range tests { + for i, v := range test.bitmap { + c.bitmap[i] = v + } + c.n = c.countRange(0, 65535) + c.bitmapSetRange(test.start, test.last+1) + if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + } + if test.expN != c.n { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + } + } +} + +func TestArrayToBitmap(t *testing.T) { + a := &container{} + tests := []struct { + array []uint16 + exp []uint64 + }{ + { + array: []uint16{}, + exp: []uint64{}, + }, + { + array: []uint16{0, 1, 2, 3}, + exp: []uint64{0xF}, + }, + } + + for i, test := range tests { + exp := make([]uint64, bitmapN) + for i, v := range test.exp { + exp[i] = v + } + + a.array = test.array + a.n = len(test.array) + a.arrayToBitmap() + if !reflect.DeepEqual(a.bitmap, exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) + } + } +} + +func TestBitmapToArray(t *testing.T) { + a := &container{} + tests := []struct { + bitmap []uint64 + exp []uint16 + }{ + { + bitmap: []uint64{}, + exp: []uint16{}, + }, + { + bitmap: []uint64{0xF}, + exp: []uint16{0, 1, 2, 3}, + }, + } + for i, test := range tests { + a.bitmap = make([]uint64, bitmapN) + n := 0 + for i, v := range test.bitmap { + a.bitmap[i] = v + n += int(popcount(v)) + } + a.n = n + + a.bitmapToArray() + if !reflect.DeepEqual(a.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) + } + } +} + +func TestRunToBitmap(t *testing.T) { + a := &container{} + tests := []struct { + runs []interval16 + exp []uint64 + }{ + { + runs: []interval16{}, + exp: []uint64{}, + }, + { + runs: []interval16{{start: 0, last: 0}}, + exp: []uint64{1}, + }, + { + runs: []interval16{{start: 0, last: 4}}, + exp: []uint64{31}, + }, + { + runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + exp: []uint64{155876}, + }, + { + runs: []interval16{{start: 0, last: 3}, {start: 60, last: 67}}, + exp: []uint64{0xF00000000000000F, 0x000000000000000F}, + }, + } + + for i, test := range tests { + exp := make([]uint64, bitmapN) + n := 0 + for i, v := range test.exp { + exp[i] = v + n += int(popcount(v)) + } + + a.runs = test.runs + a.n = n + a.runToBitmap() + if !reflect.DeepEqual(a.bitmap, exp) { + t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) + } + } +} + +func getFullBitmap() []uint64 { + x := make([]uint64, 1024, 1024) + for i := range x { + x[i] = uint64(0xFFFFFFFFFFFFFFFF) + } + return x + +} + +func TestBitmapToRun(t *testing.T) { + a := &container{} + tests := []struct { + bitmap []uint64 + exp []interval16 + }{ + { + // empty run + bitmap: []uint64{}, + exp: []interval16{}, + }, + { + // single-bit run + bitmap: []uint64{1}, + exp: []interval16{{start: 0, last: 0}}, + }, + { + // single multi-bit run in one word + bitmap: []uint64{31}, + exp: []interval16{{start: 0, last: 4}}, + }, + { + // multiple runs in one word + bitmap: []uint64{155876}, + exp: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + }, + { + // span two words, both mixed + bitmap: []uint64{0xF00000000000000F, 0x000000000000000F}, + exp: []interval16{{start: 0, last: 3}, {start: 60, last: 67}}, + }, + { + // span two words, first = maxBitmap + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xF}, + exp: []interval16{{start: 0, last: 67}}, + }, + { + // span two words, second = maxBitmap + bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF}, + exp: []interval16{{start: 60, last: 127}}, + }, + { + // span three words + bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF, 0xF}, + exp: []interval16{{start: 60, last: 131}}, + }, + { + bitmap: make([]uint64, bitmapN), + exp: []interval16{{start: 65408, last: 65535}}, + }, + { + bitmap: getFullBitmap(), + exp: []interval16{{start: 0, last: 65535}}, + }, + } + tests[8].bitmap[1022] = 0xFFFFFFFFFFFFFFFF + tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF + + for i, test := range tests { + a.bitmap = make([]uint64, bitmapN) + n := 0 + for i, v := range test.bitmap { + a.bitmap[i] = v + n += int(popcount(v)) + } + a.n = n + x := a.bitmap + a.bitmapToRun() + if !reflect.DeepEqual(a.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) + } + a.runToBitmap() + if !reflect.DeepEqual(a.bitmap, x) { + t.Fatalf("test #%v expected %v, but got %v", i, a.bitmap, x) + } + } +} + +func TestArrayToRun(t *testing.T) { + a := &container{} + tests := []struct { + array []uint16 + exp []interval16 + }{ + { + array: []uint16{}, + exp: []interval16{}, + }, + { + array: []uint16{0}, + exp: []interval16{{start: 0, last: 0}}, + }, + { + array: []uint16{0, 1, 2, 3, 4}, + exp: []interval16{{start: 0, last: 4}}, + }, + { + array: []uint16{2, 5, 6, 7, 13, 14, 17}, + exp: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + }, + } + + for i, test := range tests { + a.array = test.array + a.n = int(len(test.array)) + a.arrayToRun() + if !reflect.DeepEqual(a.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) + } + } +} + +func TestRunToArray(t *testing.T) { + a := &container{} + tests := []struct { + runs []interval16 + exp []uint16 + }{ + { + runs: []interval16{}, + exp: []uint16{}, + }, + { + runs: []interval16{{start: 0, last: 0}}, + exp: []uint16{0}, + }, + { + runs: []interval16{{start: 0, last: 4}}, + exp: []uint16{0, 1, 2, 3, 4}, + }, + { + runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + exp: []uint16{2, 5, 6, 7, 13, 14, 17}, + }, + } + + for i, test := range tests { + a.runs = test.runs + a.n = len(test.exp) + a.runToArray() + if !reflect.DeepEqual(a.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) + } + } +} + +func TestBitmapZeroRange(t *testing.T) { + c := &container{bitmap: make([]uint64, bitmapN)} + tests := []struct { + bitmap []uint64 + start uint64 + last uint64 + exp []uint64 + expN int + }{ + { + bitmap: []uint64{0x0000000000FFFF00}, + start: 9, + last: 10, + exp: []uint64{0x0000000000FFF900}, + expN: 14, + }, + { + bitmap: []uint64{0xFF0, 0xFF, 0xFF}, + start: 60, + last: 130, + exp: []uint64{0xFF0, 0, 0xF8}, + expN: 13, + }, + } + + for i, test := range tests { + for i, v := range test.bitmap { + c.bitmap[i] = v + } + c.n = c.countRange(0, 65535) + c.bitmapZeroRange(test.start, test.last+1) + if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + } + if test.expN != c.n { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + } + for i, _ := range test.bitmap { + c.bitmap[i] = 0 + } + } + +} + +func TestUnionBitmapRun(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN)} + b := &container{} + tests := []struct { + bitmap []uint64 + runs []interval16 + exp []uint64 + expN int + }{ + { + bitmap: []uint64{2}, + runs: []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 78}}, + exp: []uint64{0xC00000000000003F, 0x60FF}, + expN: 18, + }, + } + for i, test := range tests { + for i, v := range test.bitmap { + a.bitmap[i] = v + } + a.n = a.bitmapCountRange(0, 65535) + b.runs = test.runs + ret := unionBitmapRun(a, b) + if ret.isArray() { + ret.arrayToBitmap() + } + if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test #%v expected %x, but got %x", i, test.exp, ret.bitmap[:len(test.exp)]) + } + if ret.n != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + } + for i, _ := range test.bitmap { + a.bitmap[i] = 0 + } + } +} + +func TestBitmapCountRuns(t *testing.T) { + c := &container{bitmap: make([]uint64, bitmapN)} + tests := []struct { + bitmap []uint64 + exp int + }{ + { + bitmap: []uint64{0xFF00FF00}, + exp: 2, + }, + { + bitmap: []uint64{0xFF00FF0000000000, 0x1}, + exp: 2, + }, + { + bitmap: []uint64{0xFF00FF0000000000, 0x2, 0x100}, + exp: 4, + }, + { + bitmap: []uint64{0xFF00FF0000000000, 0x1010101FF0101010, 0x100}, + exp: 10, + }, + } + + for i, test := range tests { + for j, v := range test.bitmap { + c.bitmap[j] = v + } + + ret := c.bitmapCountRuns() + if ret != test.exp { + t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) + } + + for j, _ := range test.bitmap { + c.bitmap[j] = 0 + } + } + + test := tests[3] + for j, v := range test.bitmap { + c.bitmap[1024-len(test.bitmap)+j] = v + + } + ret := c.bitmapCountRuns() + if ret != test.exp { + t.Fatalf("test at end expected %v but got %v", test.exp, ret) + } +} + +func TestArrayCountRuns(t *testing.T) { + c := &container{} + tests := []struct { + array []uint16 + exp int + }{ + { + array: []uint16{}, + exp: 0, + }, + { + array: []uint16{0}, + exp: 1, + }, + { + array: []uint16{1}, + exp: 1, + }, + { + array: []uint16{1, 2, 3, 5}, + exp: 2, + }, + { + array: []uint16{0, 1, 3, 9, 2048, 4096, 4097, 65534, 65535}, + exp: 6, + }, + { + array: []uint16{0, 10, 11, 12}, + exp: 2, + }, + } + + for i, test := range tests { + c.array = test.array + ret := c.arrayCountRuns() + if ret != test.exp { + t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) + } + } +} + +func TestDifferenceArrayRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + array []uint16 + runs []interval16 + exp []uint16 + }{ + { + array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + runs: []interval16{{start: 5, last: 10}}, + exp: []uint16{0, 1, 2, 3, 4, 11, 12}, + }, + } + for i, test := range tests { + a.array = test.array + a.n = len(a.array) + b.runs = test.runs + b.n = b.runCountRange(0, 100) + ret := differenceArrayRun(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + } + } +} + +func TestDifferenceRunArray(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + runs []interval16 + array []uint16 + exp []interval16 + }{ + { + runs: []interval16{{start: 0, last: 12}}, + array: []uint16{5, 6, 7, 8, 9, 10}, + exp: []interval16{{start: 0, last: 4}, {start: 11, last: 12}}, + }, + { + runs: []interval16{{start: 0, last: 12}}, + array: []uint16{0, 1, 2, 3}, + exp: []interval16{{start: 4, last: 12}}, + }, + { + runs: []interval16{{start: 0, last: 12}}, + array: []uint16{9, 10, 11, 12, 13}, + exp: []interval16{{start: 0, last: 8}}, + }, + { + runs: []interval16{{start: 1, last: 12}}, + array: []uint16{0, 9, 10, 11, 12, 13}, + exp: []interval16{{start: 1, last: 8}}, + }, + } + for i, test := range tests { + a.runs = test.runs + a.n = a.runCountRange(0, 100) + b.array = test.array + b.n = len(b.array) + ret := differenceRunArray(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } +} + +func TestDifferenceRunBitmap(t *testing.T) { + a := &container{} + b := &container{bitmap: make([]uint64, bitmapN)} + tests := []struct { + runs []interval16 + bitmap []uint64 + exp []interval16 + }{ + { + runs: []interval16{{start: 0, last: 63}}, + bitmap: []uint64{0x0000FFFF000000F0}, + exp: []interval16{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}}, + }, + { + runs: []interval16{{start: 0, last: 63}}, + bitmap: []uint64{0x8000000000000000}, + exp: []interval16{{start: 0, last: 62}}, + }, + { + runs: []interval16{{start: 0, last: 63}}, + bitmap: []uint64{0x0000000000000001}, + exp: []interval16{{start: 1, last: 63}}, + }, + { + runs: []interval16{{start: 0, last: 63}}, + bitmap: []uint64{0x0, 0x0000000000000001}, + exp: []interval16{{start: 0, last: 63}}, + }, + { + runs: []interval16{{start: 0, last: 65}}, + bitmap: []uint64{0x0, 0x0000000000000001}, + exp: []interval16{{start: 0, last: 63}, {start: 65, last: 65}}, + }, + { + runs: []interval16{{start: 0, last: 65}}, + bitmap: []uint64{0x0, 0x8000000000000000}, + exp: []interval16{{start: 0, last: 65}}, + }, + } + for i, test := range tests { + a.runs = test.runs + a.n = a.runCountRange(0, 100) + for i, v := range test.bitmap { + b.bitmap[i] = v + } + b.n = b.bitmapCountRange(0, 100) + ret := differenceRunBitmap(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } +} + +func TestDifferenceBitmapRun(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN)} + b := &container{} + tests := []struct { + bitmap []uint64 + runs []interval16 + exp []uint64 + }{ + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, + exp: []uint64{0xFFFF0000FFFFFF0F}, + }, + } + for i, test := range tests { + for i, v := range test.bitmap { + a.bitmap[i] = v + } + a.n = a.bitmapCountRange(0, 100) + b.runs = test.runs + b.n = b.runCountRange(0, 100) + ret := differenceBitmapRun(a, b) + if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) + } + } +} + +func TestDifferenceBitmapArray(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN)} + b := &container{} + tests := []struct { + bitmap []uint64 + array []uint16 + exp []uint16 + }{ + { + bitmap: []uint64{0xFF0F}, + array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10}, + exp: []uint16{8, 9, 11, 12, 13, 14, 15}, + }, + } + for i, test := range tests { + a.bitmap = test.bitmap + b.array = test.array + ret := differenceBitmapArray(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array) + } + } +} +func TestDifferenceBitmapBitmap(t *testing.T) { + a := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + tests := []struct { + abitmap []uint64 + bbitmap []uint64 + exp []uint16 + }{ + { + abitmap: []uint64{0xFF00FFFFFFFFFFFF}, + bbitmap: []uint64{0xFFFFFFFFFFFFF000}, + exp: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + { + abitmap: []uint64{0xF}, + bbitmap: []uint64{}, + exp: []uint16{0, 1, 2, 3}, + }, + } + for i, test := range tests { + a.bitmap = test.abitmap + b.bitmap = test.bbitmap + + ret := differenceBitmapBitmap(a, b) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array) + } + } +} + +func TestDifferenceRunRun(t *testing.T) { + a := &container{container_type: ContainerRun} + b := &container{container_type: ContainerRun} + tests := []struct { + aruns []interval16 + bruns []interval16 + exp []interval16 + }{ + { + // this tests all six overlap combinations + // A [ ] [ ] [ ] [ ] [ ] [ ] + // B [ ] [ ] [ ] [ ] [ ] [ ] + aruns: []interval16{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}}, + bruns: []interval16{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}}, + exp: []interval16{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}}, + }, + } + for i, test := range tests { + a.runs = test.aruns + a.n = a.runCountRange(0, 100) + b.runs = test.bruns + b.n = b.runCountRange(0, 100) + ret := differenceRunRun(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } +} + +func TestWriteReadArray(t *testing.T) { + ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, container_type: ContainerArray} + ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}} + ba2 := &Bitmap{} + var buf bytes.Buffer + _, err := ba.WriteTo(&buf) + if err != nil { + t.Fatalf("error writing: %v", err) + } + err = ba2.UnmarshalBinary(buf.Bytes()) + if err != nil { + t.Fatalf("error unmarshaling: %v", err) + } + if !reflect.DeepEqual(ba2.containers[0].array, ca.array) { + t.Fatalf("array test expected %x, but got %x", ca.array, ba2.containers[0].array) + } +} + +func TestWriteReadBitmap(t *testing.T) { + // create bitmap containing > 4096 bits + cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, container_type: ContainerBitmap} + for i := 0; i < 129; i++ { + cb.bitmap[i] = 0x5555555555555555 + } + bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}} + bb2 := &Bitmap{} + var buf bytes.Buffer + _, err := bb.WriteTo(&buf) + if err != nil { + t.Fatalf("error writing: %v", err) + } + err = bb2.UnmarshalBinary(buf.Bytes()) + if err != nil { + t.Fatalf("error unmarshaling: %v", err) + } + if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap) + } +} + +func TestWriteReadFullBitmap(t *testing.T) { + // create bitmap containing > 4096 bits + cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, container_type: ContainerBitmap} + for i := 0; i < bitmapN; i++ { + cb.bitmap[i] = 0xffffffffffffffff + } + bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}} + bb2 := &Bitmap{} + var buf bytes.Buffer + _, err := bb.WriteTo(&buf) + if err != nil { + t.Fatalf("error writing: %v", err) + } + err = bb2.UnmarshalBinary(buf.Bytes()) + if err != nil { + t.Fatalf("error unmarshaling: %v", err) + } + if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap) + } + + if bb2.containers[0].n != cb.n { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n) + } + if bb2.containers[0].count() != cb.count() { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n) + } +} + +func TestWriteReadRun(t *testing.T) { + cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, container_type: ContainerRun} + br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}} + br2 := &Bitmap{} + var buf bytes.Buffer + _, err := br.WriteTo(&buf) + if err != nil { + t.Fatalf("error writing: %v", err) + } + err = br2.UnmarshalBinary(buf.Bytes()) + if err != nil { + t.Fatalf("error unmarshaling: %v", err) + } + if !reflect.DeepEqual(br2.containers[0].runs, cr.runs) { + t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs) + } +} + +func TestXorArrayRun(t *testing.T) { + a := &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray} + b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} + exp := []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16} + + //ret := xorArrayRun(a, b) + ret := xor(a, b) + if !reflect.DeepEqual(ret.array, exp) { + t.Fatalf("test #1 expected %v, but got %v", exp, ret.array) + } + + ret = xor(b, a) + if !reflect.DeepEqual(ret.array, exp) { + t.Fatalf("test #2 expected %v, but got %v", exp, ret.array) + } + c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray} + // exp = []int16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16} + expr := []interval16{{start: 1, last: 4}, {start: 6, last: 9}, {start: 11, last: 11}, {start: 14, last: 16}} + ret = xor(b, c) + if !reflect.DeepEqual(ret.runs, expr) { + t.Fatalf("test #3 expected %v, but got %v", exp, ret.runs) + } + ret = xor(c, b) + if !reflect.DeepEqual(ret.runs, expr) { + t.Fatalf("test #4 expected %v, but got %v", exp, ret.array) + } +} + +//special case that didn't fit the xorrunrun table testing below. +func TestXorRunRun1(t *testing.T) { + a := &container{container_type: ContainerRun} + b := &container{container_type: ContainerRun} + a.runs = []interval16{{start: 4, last: 10}} + b.runs = []interval16{{start: 5, last: 10}} + ret := xorRunRun(a, b) + if !reflect.DeepEqual(ret.array, []uint16{4}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array) + } + ret = xorRunRun(b, a) + if !reflect.DeepEqual(ret.array, []uint16{4}) { + t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array) + } +} + +func TestXorRunRun(t *testing.T) { + a := &container{container_type: ContainerRun} + b := &container{container_type: ContainerRun} + tests := []struct { + aruns []interval16 + bruns []interval16 + exp []interval16 + }{ + { + aruns: []interval16{}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 5, last: 10}}, + }, + { + aruns: []interval16{{start: 0, last: 4}}, + bruns: []interval16{{start: 6, last: 10}}, + exp: []interval16{{start: 0, last: 4}, {start: 6, last: 10}}, + }, + { + aruns: []interval16{{start: 0, last: 6}}, + bruns: []interval16{{start: 4, last: 10}}, + exp: []interval16{{start: 0, last: 3}, {start: 7, last: 10}}, + }, + { + aruns: []interval16{{start: 4, last: 10}}, + bruns: []interval16{{start: 0, last: 6}}, + exp: []interval16{{start: 0, last: 3}, {start: 7, last: 10}}, + }, + { + aruns: []interval16{{start: 0, last: 10}}, + bruns: []interval16{{start: 0, last: 6}}, + exp: []interval16{{start: 7, last: 10}}, + }, + { + aruns: []interval16{{start: 0, last: 6}}, + bruns: []interval16{{start: 0, last: 10}}, + exp: []interval16{{start: 7, last: 10}}, + }, + { + aruns: []interval16{{start: 0, last: 6}}, + bruns: []interval16{{start: 0, last: 10}}, + exp: []interval16{{start: 7, last: 10}}, + }, + { + aruns: []interval16{{start: 5, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 11, last: 12}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval16{{start: 5, last: 10}}, + exp: []interval16{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval16{{start: 2, last: 65535}}, + exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval16{{start: 2, last: 65535}}, + bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval16{{start: 0, last: 65535}}, + exp: []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval16{{start: 0, last: 65535}}, + bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + exp: []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}}, + bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, + exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}}, + }, + } + for i, test := range tests { + a.runs = test.aruns + b.runs = test.bruns + ret := xorRunRun(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + ret = xorRunRun(b, a) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.runs) + } + } +} + +func TestBitmapXorRange(t *testing.T) { + c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + tests := []struct { + bitmap []uint64 + start uint64 + last uint64 + exp []uint64 + expN int + }{ + { + bitmap: []uint64{0x0000000000000000}, + start: 0, + last: 2, + exp: []uint64{0x0000000000000007}, + expN: 3, + }, + { + bitmap: []uint64{0xF1}, + start: 4, + last: 8, + exp: []uint64{0x101}, + expN: 2, + }, + { + bitmap: []uint64{0xAA}, + start: 0, + last: 7, + exp: []uint64{0x55}, + expN: 4, + }, + { + bitmap: []uint64{0x0, 0x0000000000000000, 0x0000000000000000}, + start: 63, + last: 128, + exp: []uint64{0x8000000000000000, 0xFFFFFFFFFFFFFFFF, 0x000000000000001}, + expN: 66, + }, + { + bitmap: []uint64{0x0, 0x00000000000000FF, 0x0000000000000000}, + start: 63, + last: 128, + exp: []uint64{0x8000000000000000, 0xFFFFFFFFFFFFFF00, 0x000000000000001}, + expN: 58, + }, + { + bitmap: []uint64{0x0, 0x0, 0x0}, + start: 129, + last: 131, + exp: []uint64{0x0000000000000000, 0x0000000000000000, 0x00000000000000E}, + expN: 3, + }, + } + + for i, test := range tests { + for i, v := range test.bitmap { + c.bitmap[i] = v + } + c.n = c.countRange(0, 65535) + c.bitmapXorRange(test.start, test.last+1) + if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) { + t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)]) + } + if test.expN != c.n { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + } + } +} + +func TestXorBitmapRun(t *testing.T) { + a := &container{container_type: ContainerBitmap} + b := &container{container_type: ContainerRun} + tests := []struct { + bitmap []uint64 + runs []interval16 + exp []uint64 + }{ + { + bitmap: []uint64{0x0, 0x0, 0x0}, + runs: []interval16{{start: 129, last: 131}}, + exp: []uint64{0x0, 0x0, 0x00000000000000E}, + }, + } + for i, test := range tests { + a.bitmap = test.bitmap + b.runs = test.runs + //xorBitmapRun + ret := xor(a, b) + if !reflect.DeepEqual(ret.bitmap, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap) + } + ret = xor(b, a) + if !reflect.DeepEqual(ret.bitmap, test.exp) { + t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.bitmap) + } + } + +} + +func TestIteratorArray(t *testing.T) { + // use values that span two containers + b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + if !b.containers[0].isArray() { + t.Fatalf("wrong container type") + } + + itr := b.Iterator() + if !(itr.i == 0 && itr.j == -1) { + t.Fatalf("iterator did not zero correctly: %v\n", itr) + } + + itr.Seek(1000) + if !(itr.i == 0 && itr.j == 3) { + t.Fatalf("iterator did not seek correctly: %v\n", itr) + } + + itr.Seek(10000) + itr.Next() + val, eof := itr.Next() + if !(itr.i == 1 && itr.j == 0 && val == 90000 && !eof) { + t.Fatalf("iterator did not next correctly across containers: %v\n", itr) + } + + itr.Seek(80000) + if !(itr.i == 1 && itr.j == -1) { + t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) + } + + itr.Seek(100000) + if !(itr.i == 1 && itr.j == 0) { + t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) + } + + val, eof = itr.Next() + if !(val == 100000 && !eof) { + t.Fatalf("iterator did not next correctly: %d, %v\n", val, eof) + } + + val, eof = itr.Next() + if !(val == 0 && eof) { + t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof) + } +} + +func TestIteratorBitmap(t *testing.T) { + // use values that span two containers + // this dataset will update to bitmap after enough Adds, + // but won't update to RLE until Optimize() is called + b := NewBitmap() + for i := uint64(61000); i < 71000; i++ { + b.Add(i) + } + for i := uint64(75000); i < 75100; i++ { + b.Add(i) + } + if !b.containers[0].isBitmap() { + t.Fatalf("wrong container type") + } + + itr := b.Iterator() + if !(itr.i == 0 && itr.j == -1) { + t.Fatalf("iterator did not zero correctly: %v\n", itr) + } + + itr.Seek(65000) + if !(itr.i == 0 && itr.j == 64999) { + t.Fatalf("iterator did not seek correctly: %v\n", itr) + } + + itr.Seek(65535) + itr.Next() + val, eof := itr.Next() + if !(itr.i == 1 && itr.j == 0 && val == 65536 && !eof) { + t.Fatalf("iterator did not next correctly across containers: %v\n", itr) + } + + itr.Seek(74000) + if !(itr.i == 1 && itr.j == 8463) { + t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) + } + + itr.Seek(70999) + if !(itr.i == 1 && itr.j == 5462) { + t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) + } + + val, eof = itr.Next() + if !(val == 70999 && !eof) { + t.Fatalf("iterator did not next correctly: %d, %v\n", val, eof) + } + + itr.Seek(75100) + val, eof = itr.Next() + if !(val == 0 && eof) { + t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof) + } +} + +func TestIteratorRuns(t *testing.T) { + b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b.Optimize() + if !b.containers[0].isRun() { + t.Fatalf("wrong container type") + } + + itr := b.Iterator() + if !(itr.i == 0 && itr.j == 0 && itr.k == -1) { + t.Fatalf("iterator did not zero correctly: %v\n", itr) + } + + itr.Seek(4) + if !(itr.i == 0 && itr.j == 0 && itr.k == 3) { + t.Fatalf("iterator did not seek correctly: %v\n", itr) + } + itr.Next() + itr.Next() + val, eof := itr.Next() + if !(val == 1000 && !eof) { + t.Fatalf("iterator did not next correctly across runs: %v, %v", val, itr) + } + itr.Next() + val, eof = itr.Next() + if !(val == 1002 && !eof) { + t.Fatalf("iterator did not next correctly within a run: %v, %v", val, itr) + } + itr.Next() + itr.Next() + itr.Next() + val, eof = itr.Next() + if !(val == 100000 && !eof) { + t.Fatalf("iterator did not next correctly across containers: %v, %v", val, itr) + } + + itr.Seek(500) + if !(itr.i == 0 && itr.j == 1 && itr.k == -1) { + t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) + } + + itr.Seek(1004) + if !(itr.i == 0 && itr.j == 1 && itr.k == 3) { + t.Fatalf("iterator did not seek correctly in multiple runs: %v\n", itr) + } + + itr.Seek(1005) + if !(itr.i == 0 && itr.j == 1 && itr.k == 4) { + t.Fatalf("iterator did not seek correctly to end of run: %v\n", itr) + } + + itr.Seek(100005) + if !(itr.i == 1 && itr.j == 0 && itr.k == 4) { + t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) + } + + val, eof = itr.Next() + val, eof = itr.Next() + if !(val == 0 && eof) { + t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof) + } +} + +func TestRunBinSearchContains(t *testing.T) { + tests := []struct { + runs []interval16 + index uint16 + exp struct { + index int + found bool + } + }{ + { + runs: []interval16{{start: 0, last: 10}}, + index: uint16(3), + exp: struct { + index int + found bool + }{index: 0, found: true}, + }, + { + runs: []interval16{{start: 0, last: 10}}, + index: uint16(13), + exp: struct { + index int + found bool + }{index: 0, found: false}, + }, + { + runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, + index: uint16(13), + exp: struct { + index int + found bool + }{index: 0, found: false}, + }, + { + runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, + index: uint16(36), + exp: struct { + index int + found bool + }{index: 1, found: false}, + }, + } + for i, test := range tests { + index := test.index + runs := test.runs + idx, found := binSearchRuns(index, runs) + + if test.exp.index != idx && test.exp.found != found { + t.Fatalf("test #%v expected %v , but got %v %v", i, test.exp, idx, found) + } + } +} + +func TestRunBinSearch(t *testing.T) { + tests := []struct { + runs []interval16 + search uint16 + exp bool + expi int + }{ + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 1, + exp: false, + expi: 0, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 2, + exp: true, + expi: 0, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 5, + exp: true, + expi: 0, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 10, + exp: true, + expi: 0, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 20, + exp: false, + expi: 1, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 55, + exp: true, + expi: 1, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 70, + exp: false, + expi: 2, + }, + { + runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, + search: 100, + exp: false, + expi: 3, + }, + } + for i, test := range tests { + idx, contains := binSearchRuns(test.search, test.runs) + if !(test.exp == contains && test.expi == idx) { + t.Fatalf("test #%v expected (%v, %v) but got (%v, %v)", i, test.exp, test.expi, contains, idx) + } + } +} +func TestBitmap_RemoveEmptyContainers(t *testing.T) { + bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1.Remove(2 << 16) + if bm1.countEmptyContainers() != 1 { + t.Fatalf("Should be 1 empty container ") + } + bm1.removeEmptyContainers() + + if bm1.countEmptyContainers() != 0 { + t.Fatalf("Should be no empty containers ") + } +} + +func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { + bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1.Remove(2 << 16) + var buf bytes.Buffer + if _, err := bm1.WriteTo(&buf); err != nil { + t.Fatalf("Failure to write to bitmap buffer. ") + } + bm0 := NewBitmap() + bm0.UnmarshalBinary(buf.Bytes()) + if bm0.countEmptyContainers() != 0 { + t.Fatalf("Should be no empty containers ") + } + if bm0.Count() != bm1.Count() { + t.Fatalf("Counts do not match after a marshal %d %d", bm0.Count(), bm1.Count()) + } +} + +func Test_BufBitmapIterator_Next(t *testing.T) { + b := NewBitmap() + for i := uint64(0); i < 4097; i++ { + b.Add(i) + } + if !b.containers[0].isBitmap() { + t.Fatalf("wrong container type") + } + + bin := []uint16{} + + itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap)) + x := uint16(0) + + for i := 0; i < 10; i++ { + x, _ = itr.next() + bin = append(bin, x) + } + exp := []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + if !reflect.DeepEqual(bin, exp) { + t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin) + } + + // ensure that unread points next back one such that the last value is repeated + itr.unread() + x, _ = itr.next() + bin = append(bin, x) + exp = append(exp, uint16(9)) + if !reflect.DeepEqual(bin, exp) { + t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin) + } +} + +func Test_BufBitmapIterator_UnreadPanic(t *testing.T) { + + defer func() { + if r := recover(); r == nil { + t.Errorf("BufBitmapIterator unread did not panic") + } + }() + + b := NewBitmap() + for i := uint64(0); i < 4097; i++ { + b.Add(i) + } + if !b.containers[0].isBitmap() { + t.Fatalf("wrong container type") + } + + itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap)) + for i := 0; i < 10; i++ { + itr.next() + } + + // ensure that unreading back-to-back panics + itr.unread() + itr.unread() +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 4166ca880..8de92eb4e 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -28,6 +28,102 @@ import ( _ "github.com/pilosa/pilosa/test" ) +func TestBitmapClone(t *testing.T) { + b := roaring.NewBitmap() + for i := uint64(61000); i < 71000; i++ { + b.Add(i) + } + c := b.Clone() + if !reflect.DeepEqual(b, c) { + t.Fatalf("Clone Objects not equal\n") + } + d := func() *roaring.Bitmap { //anybody know how to declare a nil value? + return nil + }() + e := d.Clone() + if e != nil { + t.Fatalf("Clone nil Objects not equal\n") + } +} + +func TestContainerCount(t *testing.T) { + b := roaring.NewBitmap(65535) + + if b.Count() != b.CountRange(0, 65546) { + t.Fatalf("Count != CountRange\n") + } +} + +func TestCheckBitmap(t *testing.T) { + b := roaring.NewBitmap() + x := 0 + for i := uint64(61000); i < 71000; i++ { + x++ + b.Add(i) + } + for i := uint64(75000); i < 75100; i++ { + x++ + b.Add(i) + } + err := b.Check() + if err != nil { + t.Fatalf("%v\n", err) + } +} + +func TestCheckArray(t *testing.T) { + b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + err := b.Check() + if err != nil { + t.Fatalf("%v\n", err) + } +} + +func TestCheckRun(t *testing.T) { + b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b.Optimize() // convert to runs + err := b.Check() + if err != nil { + t.Fatalf("%v\n", err) + } +} +func TestCheckFullRun(t *testing.T) { + b := roaring.NewBitmap() + for i := uint64(0); i < 2097152; i++ { + if i%16384 == 0 { + b.Optimize() // convert to runs + } + b.Add(i) + } + err := b.Check() + if err != nil { + t.Fatalf("Before %v\n", err) + } + b.Optimize() // convert to runs + err = b.Check() + if err != nil { + t.Fatalf("After %v\n", err) + } +} + +// Ensure that we can transition between runs and arrays when materializing the bitmap. +func TestContainerTransitions(t *testing.T) { + // [run, run][array][run] + b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) + b.Optimize() // convert to runs + if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) { + t.Fatalf("unexpected slice: %+v", b.Slice()) + } + + // Test the case where last and first bits of adjoining containers are set. + // [run][array][run] + b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) + b2.Optimize() // convert to runs + if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) { + t.Fatalf("unexpected slice: %+v", b2.Slice()) + } +} + // Ensure an empty bitmap returns false if checking for existence. func TestBitmap_Contains_Empty(t *testing.T) { if roaring.NewBitmap().Contains(1000) { @@ -55,6 +151,7 @@ func TestBitmap_Slice_Empty(t *testing.T) { } // Ensure a bitmap can return a slice of values within a range. +// TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { t.Fatalf("unexpected slice: %+v", a) @@ -95,6 +192,55 @@ func TestBitmap_Max(t *testing.T) { } } +func TestBitmap_BitmapCountRange(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + for i := uint64(628); i < 2683301; i++ { + bm0.Add(i) + } + bm0.Add(2683307) + if n := bm0.CountRange(1, 2683311); n != 2682674 { + t.Fatalf("unexpected n: %d", n) + } + + if n := bm0.CountRange(2683177, 2683310); n != 125 { + t.Fatalf("unexpected n: %d", n) + } + + if n := bm0.CountRange(2683301, 3000000); n != 1 { + t.Fatalf("unexpected n: %d", n) + } + + if n := bm0.CountRange(0, 1); n != 1 { + t.Fatalf("unexpected n: %d", n) + } + + // Test the case where the range is outside of the bitmap space. + if n := bm0.CountRange(10000000, 10000001); n != 0 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_ArrayCountRange(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177, 2683313) + if n := bm0.CountRange(1, 2683313); n != 1 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_RunCountRange(t *testing.T) { + bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) + bm0.Optimize() // convert to runs + if n := bm0.CountRange(15, 1000003); n != 5 { + t.Fatalf("unexpected n: %d", n) + } + + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) + bm1.Optimize() // convert to runs + if n := bm1.CountRange(5, 12); n != 7 { + t.Fatalf("unexpected n: %d", n) + } +} + func TestBitmap_Intersection(t *testing.T) { bm0 := roaring.NewBitmap(0, 2683177) bm1 := roaring.NewBitmap() @@ -109,6 +255,105 @@ func TestBitmap_Intersection(t *testing.T) { } +func TestBitmap_Union1(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + bm1.Add(4000000) + + result := bm0.Union(bm1) + if n := result.Count(); n != 2682675 { + t.Fatalf("unexpected n: %d", n) + } + bm := testBM() + result = bm.Union(bm0) + if n := result.Count(); n != 75009 { + t.Fatalf("unexpected n: %d", n) + } + result = bm.Union(bm) + if n := result.Count(); n != 75007 { + t.Fatalf("unexpected n: %d", n) + } + +} + +func TestBitmap_Intersection_Empty(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() + + result := bm0.Intersect(bm1) + if n := result.Count(); n != 0 { + t.Fatalf("unexpected n: %d", n) + } + +} + +func TestBitmap_IntersectArrayArray(t *testing.T) { + bm0 := roaring.NewBitmap(0, 1, 2683, 5005) + bm1 := roaring.NewBitmap(0, 2683, 2684, 5000) + + result := bm0.Intersect(bm1) + if n := result.Count(); n != 2 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_IntersectBitmapBitmap(t *testing.T) { + bm0 := roaring.NewBitmap() + for i := uint64(0); i < 65536; i += 2 { + bm0.Add(i) + } + + bm1 := roaring.NewBitmap() + for i := uint64(0); i < 65536; i += 3 { + bm1.Add(i) + } + + result := bm0.Intersect(bm1) + if n := result.Count(); n != 10923 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_IntersectRunRun(t *testing.T) { + // Intersect two runs that result in an array. + bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) + bm0.Optimize() // convert to runs + bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11) + bm1.Optimize() // convert to runs + result := bm0.Intersect(bm1) + if n := result.Count(); n != 3 { + t.Fatalf("unexpected n: %d", n) + } + + // Intersect two runs that result in a bitmap. + bm2 := roaring.NewBitmap() + runLen := uint64(25) + spaceLen := uint64(8) + offset := (runLen / 2) + spaceLen + for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) { + for j := uint64(0); j < runLen; j++ { + bm2.Add(offset + i + j) + } + } + bm2.Optimize() // convert to runs + bm3 := roaring.NewBitmap() + runLen = uint64(32) + spaceLen = uint64(1) + for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { + for j := uint64(0); j < runLen; j++ { + bm3.Add(i + j) + } + } + bm3.Optimize() // convert to runs + result = bm2.Intersect(bm3) + if n := result.Count(); n != 47628 { + t.Fatalf("unexpected n: %d", n) + } +} + func TestBitmap_Difference(t *testing.T) { bm0 := roaring.NewBitmap(0, 2683177) bm1 := roaring.NewBitmap() @@ -116,12 +361,40 @@ func TestBitmap_Difference(t *testing.T) { bm1.Add(i) } result := bm0.Difference(bm1) - //expect to have just 0 if n := result.Count(); n != 1 { t.Fatalf("unexpected n: %d", n) } } +func TestBitmap_Difference_Empty(t *testing.T) { + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() + result := bm0.Difference(bm1) + if n := result.Count(); n != 2 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_DifferenceArrayArray(t *testing.T) { + bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20) + bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18) + result := bm0.Difference(bm1) + if n := result.Count(); n != 5 { + t.Fatalf("unexpected n: %d", n) + } +} + +func TestBitmap_DifferenceArrayRun(t *testing.T) { + bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) + + bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) + bm1.Optimize() // convert to runs + result := bm0.Difference(bm1) + if n := result.Count(); n != 6 { + t.Fatalf("unexpected n: %d", n) + } +} + func TestBitmap_Union(t *testing.T) { bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) @@ -131,6 +404,25 @@ func TestBitmap_Union(t *testing.T) { } } +func TestBitmap_Xor(t *testing.T) { + bm0 := testBM() + bm1 := roaring.NewBitmap(0, 1, 2, 3) + result := bm1.Xor(bm0) + if n := result.Count(); n != 75011 { + t.Fatalf("unexpected n: %d", n) + } + + result = bm0.Xor(bm1) + if n := result.Count(); n != 75011 { + t.Fatalf("unexpected n: %d", n) + } + + result = bm0.Xor(bm0) + if n := result.Count(); n != 0 { + t.Fatalf("unexpected n: %d", n) + } +} + func TestBitmap_Xor_ArrayArray(t *testing.T) { bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) @@ -166,13 +458,18 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { result := bm0.Xor(bm1) if n := result.Count(); n != 4999 { - t.Fatalf("unexpected n: %d", n) + t.Fatalf("test #1 unexpected n: %d", n) + } + + result = bm1.Xor(bm0) + if n := result.Count(); n != 4999 { + t.Fatalf("test #2 unexpected n: %d", n) } //equivalence bitmap test result = result.Xor(result) if n := result.Count(); n > 0 { - t.Fatalf("unexpected n: %d", n) + t.Fatalf("test 3 unexpected n: %d", n) } empty := roaring.NewBitmap() @@ -265,7 +562,7 @@ func TestBitmap_Flip_After(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003) bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) if n := bm0.IntersectionCount(bm1); n != 3 { @@ -275,6 +572,49 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { } } +// Ensure bitmap can return the number of intersecting bits in two bitmaps. +func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { + bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1.Optimize() // convert to runs + + if n := bm0.IntersectionCount(bm1); n != 3 { + t.Fatalf("unexpected n: %d", n) + } else if n := bm1.IntersectionCount(bm0); n != 3 { + t.Fatalf("unexpected n (reverse): %d", n) + } +} + +// Ensure bitmap can return the number of intersecting bits in two bitmaps. +func TestBitmap_IntersectionCount_RunRun(t *testing.T) { + bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) + bm0.Optimize() // convert to runs + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1.Optimize() // convert to runs + + if n := bm0.IntersectionCount(bm1); n != 6 { + t.Fatalf("unexpected n: %d", n) + } else if n := bm1.IntersectionCount(bm0); n != 6 { + t.Fatalf("unexpected n (reverse): %d", n) + } +} + +// Ensure bitmap can return the number of intersecting bits in two bitmaps. +func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { + bm0 := roaring.NewBitmap() + for i := uint64(3); i <= 1000006; i += 2 { + bm0.Add(i) + } + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1.Optimize() // convert to runs + + if n := bm0.IntersectionCount(bm1); n != 4 { + t.Fatalf("unexpected n: %d", n) + } else if n := bm1.IntersectionCount(bm0); n != 4 { + t.Fatalf("unexpected n (reverse): %d", n) + } +} + // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) @@ -311,6 +651,21 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { t.Fatalf("unexpected n (reverse): %d", n) } } +func TestBitmap_IntersectionCount_Mixed(t *testing.T) { + bm0 := testBM() + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) + bm3 := roaring.NewBitmap(131072) + + if n := bm0.IntersectionCount(bm0); n != bm0.Count() { + t.Fatalf("unexpected n: %d", n) + } + if n := bm0.IntersectionCount(bm1); n != 1 { + t.Fatalf("unexpected n: %d", n) + } + if n := bm0.IntersectionCount(bm3); n != 1 { + t.Fatalf("unexpected n: %d", n) + } +} func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } @@ -325,10 +680,18 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { m := make(map[uint64]struct{}) // Add values to the bitmap and set. + manual_count := uint64(0) for _, v := range a { - bm.Add(v) + new_bit, _ := bm.Add(v) + if new_bit { + manual_count++ + } m[v] = struct{}{} } + //check count + if manual_count != bm.Count() { + t.Fatalf("expected bitmap Add count to be: %d got: %d", manual_count, bm.Count()) + } // Verify existence. for _, v := range a { @@ -353,7 +716,14 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { // Remove all values in random order. for _, i := range rand.Perm(len(a)) { - bm.Remove(a[i]) + removed, _ := bm.Remove(a[i]) + if removed { + manual_count-- + } + //check count + if manual_count != bm.Count() { + t.Fatalf("expected bitmap Remove count to be: %d got: %d", manual_count, bm.Count()) + } } // Verify all values have been removed. @@ -383,6 +753,8 @@ func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000, true) } +// TODO update for RLE + // Ensure a bitmap can be marshaled and unmarshaled. func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { if testing.Short() { @@ -446,6 +818,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { } // Ensure iterator can iterate over all the values on the bitmap. +// TODO duplicate for all container types func TestIterator(t *testing.T) { itr := roaring.NewBitmap(1, 2, 3).Iterator() itr.Seek(0) @@ -460,6 +833,62 @@ func TestIterator(t *testing.T) { } } +// testBM creates a bitmap with 3 containers: array, bitmap, and run. +func testBM() *roaring.Bitmap { + + bm := roaring.NewBitmap() + //the array + for i := uint64(0); i < 1024; i += 4 { + bm.Add((1 << 16) + i) + } + //the bitmap + for i := uint64(0); i < 16384; i += 2 { + bm.Add((2 << 16) + i) + } + //small run + for i := uint64(0); i < 1024; i += 1 { + bm.Add((3 << 16) + i) + } + //large run + for i := uint64(0); i < 65535; i += 1 { + bm.Add((4 << 16) + i) + } + bm.Optimize() + //count 75007 + return bm +} + +func TestBitmapOffsetRange(t *testing.T) { + bm := testBM() + + bm1 := bm.OffsetRange(0, 0, 327680) + if bm1.Count() != bm.Count() { + t.Fatalf("Not Equal %d %d", bm1.Count(), bm.Count()) + } + bm1 = bm.OffsetRange(0, 0, 131072) + if bm1.Count() != 256 { + t.Fatalf("Not Equal %d %d", bm1.Count(), 256) + } + +} +func TestBitmapContains(t *testing.T) { + bm := testBM() + + //search for run value present + if found := bm.Contains(3 << 16); !found { + t.Fatalf("Test #1 Not Found %d ", 3<<16) + } + + //search for value not present + if found := bm.Contains((3 << 16) + 2048); found { + t.Fatalf("Test #2 Found %d ", (3<<16)+2048) + } +} + +func TestBitmapBufIterator(t *testing.T) { + +} + var benchmarkBitmapIntersectionCountData struct { a, b *roaring.Bitmap } @@ -531,3 +960,11 @@ func diff(a, b []uint64) string { } return "" } + +func TestBitmap_Intersect(t *testing.T) { + bm0 := testBM() + result := bm0.Intersect(bm0) + if bm0.Count() != result.Count() { + t.Fatalf("Counts do not match %d %d", bm0.Count(), result.Count()) + } +} diff --git a/server.go b/server.go index abc03e476..97883fae4 100644 --- a/server.go +++ b/server.go @@ -162,7 +162,12 @@ func (s *Server) Open() error { s.Holder.Broadcaster = s.Broadcaster // Serve HTTP. - go func() { http.Serve(ln, s.Handler) }() + go func() { + err := http.Serve(ln, s.Handler) + if err != nil { + s.Logger().Printf("HTTP handler terminated with error: %s\n", err) + } + }() // Start background monitoring. s.wg.Add(3)