diff --git a/roaring/roaring.go b/roaring/roaring.go new file mode 100644 index 000000000..4315c257f --- /dev/null +++ b/roaring/roaring.go @@ -0,0 +1,723 @@ +// package roaring implements roaring bitmaps with support for incremental changes. +package roaring + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/fnv" + "io" + "unsafe" +) + +const ( + // cookie is the first four bytes in a roaring bitmap file. + cookie = uint32(12346) + + // headerSize is the size of the cookie and key count at the beginning of a file. + headerSize = 4 + 4 + + // bitmapN is the number of values in a container.bitmap. + bitmapN = (1 << 16) / 64 +) + +// Bitmap represents a roaring bitmap. +type Bitmap struct { + keys []uint16 // keys for containers + containers []*container // array and bitmap containers + + // Number of operations written to the writer. + opN int + + // Writer where operations are appended to. + OpWriter io.Writer +} + +// NewBitmap returns a Bitmap with an initial set of values. +func NewBitmap(a ...uint32) *Bitmap { + b := &Bitmap{} + b.Add(a...) + return b +} + +// Add adds values to the bitmap. +func (b *Bitmap) Add(a ...uint32) error { + for _, v := range a { + // Create an add operation. + op := &op{typ: opTypeAdd, value: v} + + // Write operation to op log. + if err := b.writeOp(op); err != nil { + return err + } + + // Apply to the in-memory bitmap. + op.apply(b) + } + + return nil +} + +func (b *Bitmap) add(v uint32) { + hb := highbits(v) + i := search(b.keys, hb) + + // If index is negative then there's not an exact match + // and a container needs to be added. + if i < 0 { + b.insertAt(hb, newContainer(), -i-1) + i = -i - 1 + } + + println("DBG*", highbits(v)) + b.containers[i].add(lowbits(v)) +} + +// Contains returns true if v is in the bitmap. +func (b *Bitmap) Contains(v uint32) bool { + c := b.container(highbits(v)) + if c == nil { + return false + } + return c.contains(lowbits(v)) +} + +// Remove removes values from the bitmap. +func (b *Bitmap) Remove(a ...uint32) error { + for _, v := range a { + // Create an add operation. + op := &op{typ: opTypeRemove, value: v} + + // Write operation to op log. + if err := b.writeOp(op); err != nil { + return err + } + + // Apply operation to the bitmap. + op.apply(b) + } + return nil +} + +func (b *Bitmap) remove(v uint32) { + hb := highbits(v) + i := search(b.keys, hb) + if i < 0 { + return + } + b.containers[i].remove(lowbits(v)) +} + +// Slice returns a slice of all integers in the bitmap. +func (b *Bitmap) Slice() []uint32 { + var a []uint32 + itr := b.iterator() + for v := itr.Seek(0); !itr.EOF(); v = itr.Next() { + a = append(a, v) + } + return a +} + +// SliceRange returns a slice of integers between [start, end). +func (b *Bitmap) SliceRange(start, end uint32) []uint32 { + var a []uint32 + itr := b.iterator() + for v := itr.Seek(start); !itr.EOF() && v < end; v = itr.Next() { + a = append(a, v) + } + return a +} + +// ForEach executes fn for each value in the bitmap. +func (b *Bitmap) ForEach(fn func(uint32)) { + itr := b.iterator() + for v := itr.Seek(0); !itr.EOF(); v = itr.Next() { + fn(v) + } +} + +// ForEachRange executes fn for each value in the bitmap between [start, end). +func (b *Bitmap) ForEachRange(start, end uint32, fn func(uint32)) { + itr := b.iterator() + for v := itr.Seek(start); !itr.EOF() && v < end; v = itr.Next() { + fn(v) + } +} + +// container returns the container with the given key. +func (b *Bitmap) container(key uint16) *container { + i := search(b.keys, key) + if i < 0 { + return nil + } + return b.containers[i] +} + +func (b *Bitmap) insertAt(key uint16, c *container, i int) { + b.keys = append(b.keys, 0) + copy(b.keys[i+1:], b.keys[i:]) + b.keys[i] = key + + b.containers = append(b.containers, nil) + copy(b.containers[i+1:], b.containers[i:]) + b.containers[i] = c +} + +// WriteTo writes b to w. +func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { + // Build header before writing individual container blocks. + buf := make([]byte, headerSize+(len(b.keys)*(2+2+4))) + binary.LittleEndian.PutUint32(buf[0:], cookie) + binary.LittleEndian.PutUint32(buf[4:], uint32(len(b.keys))) + + // Encode keys and cardinality. + for i, key := range b.keys { + binary.LittleEndian.PutUint16(buf[headerSize+i*4:], uint16(key)) + binary.LittleEndian.PutUint16(buf[headerSize+i*4+2:], uint16(b.containers[i].n-1)) + } + + // Write the offset for each container block. + offset := uint32(len(buf)) + for i, c := range b.containers { + binary.LittleEndian.PutUint32(buf[headerSize+(len(b.keys)*4)+(i*4):], uint32(offset)) + offset += uint32(c.size()) + } + + // Write header. + i, err := w.Write(buf) + n += int64(i) + if err != nil { + return n, err + } + + // Write each container block. + for _, c := range b.containers { + nn, err := c.WriteTo(w) + n += nn + if err != nil { + return n, err + } + } + + return n, nil +} + +// UnmarshalBinary decodes b from a binary-encoded byte slice. +func (b *Bitmap) UnmarshalBinary(data []byte) error { + if len(data) < headerSize { + return errors.New("data too small") + } + + // Verify the first 4 bytes are the correct cookie. + if v := binary.LittleEndian.Uint32(data[0:4]); v != cookie { + return errors.New("invalid roaring file") + } + + // Read key count. + keyN := binary.LittleEndian.Uint32(data[4:8]) + b.keys = make([]uint16, keyN) + b.containers = make([]*container, keyN) + + // Read container key headers. + for i, buf := 0, data[8:]; i < int(keyN); i, buf = i+1, buf[4:] { + b.keys[i] = binary.LittleEndian.Uint16(buf[0:2]) + b.containers[i] = &container{n: int(binary.LittleEndian.Uint16(buf[2:4])) + 1} + } + + // Read container offsets and attach data. + opsOffset := 8 + int(keyN)*4 + for i, buf := 0, data[8+int(keyN)*4:]; i < int(keyN); i, buf = i+1, buf[4:] { + offset := binary.LittleEndian.Uint32(buf[0:4]) + + // Verify the offset is within the bounds of the input data. + if int(offset) >= len(data) { + return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) + } + + // Map byte slice directly to the container data. + c := b.containers[i] + if c.n <= arrayMaxSize { + c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] + opsOffset = int(offset) + len(c.array)*2 + } else { + c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] + opsOffset = int(offset) + len(c.bitmap)*8 + } + } + + // Read ops log until the end of the file. + buf := data[opsOffset:] + for { + // Exit when there are no more ops to parse. + if len(buf) == 0 { + break + } + + // Unmarshal the op and apply it. + var op op + if err := op.UnmarshalBinary(buf); err != nil { + // FIXME(benbjohnson): return error with position so file can be trimmed. + return err + } + op.apply(b) + + // Move the buffer forward. + buf = buf[op.size():] + } + + return nil +} + +// writeOp writes op to the OpWriter, if available. +func (b *Bitmap) writeOp(op *op) error { + if b.OpWriter == nil { + return nil + } + + if _, err := op.WriteTo(b.OpWriter); err != nil { + return err + } + + b.opN++ + return nil +} + +// iterator returns an iterator for the bitmap. +func (b *Bitmap) iterator() *iterator { return &iterator{bitmap: b} } + +// iterator represents an iterator over a Bitmap. +type iterator struct { + bitmap *Bitmap + i, j int +} + +// EOF returns true if the iterator is at the end of the bitmap. +func (itr *iterator) EOF() bool { return itr.i >= len(itr.bitmap.containers) } + +// Seek moves to the first value equal to or greater than v. +func (itr *iterator) Seek(seek uint32) uint32 { + // Move to the correct container. + itr.i = search(itr.bitmap.keys, highbits(seek)) + if itr.i < 0 { + itr.i = -itr.i - 1 + } + if itr.EOF() { + return 0 + } + + // Move to the correct value index inside the array container. + lb := lowbits(seek) + if c := itr.bitmap.containers[itr.i]; c.isArray() { + // Find index in the container. + itr.j = search(c.array, lb) + if itr.j < 0 { + itr.j = -itr.j - 1 + } + if itr.j < len(c.array) { + return itr.peek() + } + + // If it's at the end of the container then move to the next one. + itr.i, itr.j = itr.i+1, -1 + return itr.Next() + } + + // If it's a bitmap container then move to index before the value and call next(). + itr.j = int(lb) - 1 + return itr.Next() +} + +// Next returns the next value in the bitmap. +func (itr *iterator) Next() uint32 { + // Iterate over containers until we find the next value or EOF. + for { + if itr.EOF() { + return 0 + } + + // Move to the next item in the container if it's an array container. + c := itr.bitmap.containers[itr.i] + if c.isArray() { + if itr.j >= c.n-1 { + itr.i, itr.j = itr.i+1, -1 + continue + } + itr.j++ + return itr.peek() + } + + // Move to the next possible index in the bitmap container. + itr.j++ + + // Find first non-zero bit in current bitmap, if possible. + hb := int(itr.j / 64) + lb := c.bitmap[hb] >> (uint(itr.j) % 64) + if lb != 0 { + itr.j = int(itr.j) + trailingZeroN(lb) + return itr.peek() + } + + // Otherwise iterate through remaining bitmaps to find next bit. + for hb++; hb < len(c.bitmap); hb++ { + if c.bitmap[hb] != 0 { + itr.j = int(hb*64) + trailingZeroN(c.bitmap[hb]) + return itr.peek() + } + } + + // If no bits found then move to the next container. + itr.i, itr.j = itr.i+1, -1 + } +} + +// peek returns the current value. +func (itr *iterator) peek() uint32 { + key := itr.bitmap.keys[itr.i] + c := itr.bitmap.containers[itr.i] + if c.isArray() { + return uint32(key)<<16 | uint32(c.array[itr.j]) + } + return uint32(key)<<16 | uint32(itr.j) +} + +// The maximum size of array containers. +const arrayMaxSize = 4096 + +// container represents a container for uint16 integers. +// +// These are used for storing the low bits. Containers are separated into two +// types depending on cardinality. For containers with less than 4,096 values, +// an array container is used. For containers with more than 4,096 values, +// the values are encoded into bitmaps. +type container struct { + n int // number of integers in container + array []uint16 // used for array containers + bitmap []uint64 // used for bitmap containers + mapped bool // mapped directly to a byte slice when true +} + +// newContainer returns a new instance of container. +func newContainer() *container { + return &container{} +} + +// isArray returns true if the container is an array container. +func (c *container) isArray() bool { return c.bitmap == nil } + +// unmap creates copies of the containers data in the heap. +// +// This is performed when altering the container since its contents could be +// pointing at a read-only mmap. +func (c *container) unmap() { + if !c.mapped { + return + } + + if c.array != nil { + tmp := make([]uint16, len(c.array)) + copy(tmp, c.array) + c.array = tmp + } + if c.bitmap != nil { + tmp := make([]uint64, len(c.bitmap)) + copy(tmp, c.bitmap) + c.bitmap = tmp + } + c.mapped = false +} + +// add adds a value to the container. +func (c *container) add(v uint16) { + if c.isArray() { + c.arrayAdd(v) + return + } + c.bitmapAdd(v) +} + +func (c *container) arrayAdd(v uint16) { + // Optimize appending to the end of an array container. + if c.n > 0 && c.isArray() && c.array[c.n-1] < v { + c.unmap() + c.array = append(c.array, v) + c.n++ + return + } + + // Find index of the integer in the container. Exit if it already exists. + i := search(c.array, v) + if i >= 0 { + return + } + + // Convert to a bitmap container if too many values are in an array container. + if c.n >= arrayMaxSize { + c.convertToBitmap() + c.bitmapAdd(v) + return + } + + // Otherwise insert into array. + c.unmap() + println("DBG&&&&&", v) + i = -i - 1 + c.array = append(c.array, 0) + copy(c.array[i+1:], c.array[i:]) + c.array[i] = v + c.n++ +} + +func (c *container) bitmapAdd(v uint16) { + if c.bitmapContains(v) { + return + } + c.unmap() + c.bitmap[v/64] |= (1 << uint64(v%64)) + c.n++ +} + +// contains returns true if v is in the container. +func (c *container) contains(v uint16) bool { + if c.isArray() { + return c.arrayContains(v) + } + return c.bitmapContains(v) +} + +func (c *container) arrayContains(v uint16) bool { + return search(c.array, v) >= 0 +} + +func (c *container) bitmapContains(v uint16) bool { + return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0 +} + +// remove adds a value to the container. +func (c *container) remove(v uint16) { + if c.isArray() { + c.arrayRemove(v) + return + } + c.bitmapRemove(v) +} + +func (c *container) arrayRemove(v uint16) { + i := search(c.array, v) + if i < 0 { + return + } + c.unmap() + + c.n-- + c.array = append(c.array[:i], c.array[i+1:]...) +} + +func (c *container) bitmapRemove(v uint16) { + if !c.bitmapContains(v) { + return + } + c.unmap() + + // Lower count and remove element. + c.n-- + c.bitmap[v/64] &^= (uint64(1) << (v % 64)) + + // Convert to array if we go below the threshold. + if c.n == arrayMaxSize { + c.convertToArray() + } +} + +// convertToArray converts the values in the bitmap to array values. +func (c *container) convertToArray() { + c.array = make([]uint16, 0, c.n) + for i, bitmap := range c.bitmap { + for bitmap != 0 { + t := bitmap & -bitmap + c.array = append(c.array, uint16((i*64 + int(popcount(t-1))))) + bitmap ^= t + } + } + c.bitmap = nil + c.mapped = false +} + +// convertToBitmap converts the values in array to bitmap values. +func (c *container) convertToBitmap() { + c.bitmap = make([]uint64, bitmapN) + for _, v := range c.array { + c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64)) + } + c.array = nil + c.mapped = false +} + +// WriteTo writes c to w. +func (c *container) WriteTo(w io.Writer) (n int64, err error) { + if c.isArray() { + return c.arrayWriteTo(w) + } + return c.bitmapWriteTo(w) +} + +func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) + return int64(nn), err +} + +func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) { + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)]) + return int64(nn), err +} + +// size returns the encoded size of the container, in bytes. +func (c *container) size() int { + if c.isArray() { + return len(c.array) * 2 + } + return len(c.bitmap) * 8 +} + +// opType represents a type of operation. +type opType uint8 + +const ( + opTypeAdd = opType(0) + opTypeRemove = opType(1) +) + +// op represents an operation on the bitmap. +type op struct { + typ opType + value uint32 +} + +// apply executes the operation against a bitmap. +func (op *op) apply(b *Bitmap) { + switch op.typ { + case opTypeAdd: + b.add(op.value) + case opTypeRemove: + b.remove(op.value) + default: + panic(fmt.Sprintf("invalid op type: %d", op.typ)) + } +} + +// WriteTo writes op to the w. +func (op *op) WriteTo(w io.Writer) (n int64, err error) { + buf := make([]byte, op.size()) + + // Write type and value. + buf[0] = byte(op.typ) + binary.LittleEndian.PutUint32(buf[1:5], op.value) + + // Add checksum at the end. + h := fnv.New32a() + h.Write(buf[0:5]) + binary.LittleEndian.PutUint32(buf[5:9], h.Sum32()) + fmt.Println("") + fmt.Println("W<<<<<<<<<<<<", op.value) + + // Write to writer. + nn, err := w.Write(buf) + return int64(nn), err +} + +// UnmarshalBinary decodes data into an op. +func (op *op) UnmarshalBinary(data []byte) error { + if len(data) < op.size() { + return fmt.Errorf("op data out of bounds: len=%d", len(data)) + } + + // Verify checksum. + h := fnv.New32a() + h.Write(data[0:5]) + if chk := binary.LittleEndian.Uint32(data[5:9]); chk != h.Sum32() { + return fmt.Errorf("checksum mismatch: exp=%08x, got=%08x", h.Sum32(), chk) + } + + // Read type and value. + op.typ = opType(data[0]) + op.value = binary.LittleEndian.Uint32(data[1:5]) + fmt.Println("R>", op.value) + + return nil +} + +// size returns the encoded size of the op, in bytes. +func (*op) size() int { return 1 + 4 + 4 } + +func highbits(v uint32) uint16 { return uint16(v >> 16) } +func lowbits(v uint32) uint16 { return uint16(v & 0xFFFF) } + +// search returns the index of v in a. +func search(a []uint16, value uint16) int { + // Optimize for elements and the last element. + n := len(a) + if n == 0 { + return -1 + } else if a[n-1] == value { + return n - 1 + } + + // Otherwise perform binary search for exact match. + lo, hi := 0, n-1 + for lo+16 <= hi { + i := int(uint((lo + hi)) >> 1) + v := a[i] + + if v < value { + lo = i + 1 + } else if v > value { + hi = i - 1 + } else { + return i + } + } + + // If an exact match isn't found then return a negative index. + for ; lo <= hi; lo++ { + v := a[lo] + if v == value { + return lo + } else if v > value { + break + } + } + return -(lo + 1) +} + +// trailingZeroN returns the number of trailing zeros in v. +// v must be greater than zero. +func trailingZeroN(v uint64) int { + n := int64(63) + if y := v << 32; y != 0 { + n, v = n-32, y + } + if y := v << 16; y != 0 { + n, v = n-16, y + } + if y := v << 8; y != 0 { + n, v = n-8, y + } + if y := v << 4; y != 0 { + n, v = n-4, y + } + if y := v << 2; y != 0 { + n, v = n-2, y + } + return int(n - int64(uint64(v<<1)>>63)) +} + +// bit population count, taken from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) (n uint64) { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go new file mode 100644 index 000000000..9d4559c19 --- /dev/null +++ b/roaring/roaring_test.go @@ -0,0 +1,215 @@ +package roaring_test + +import ( + "bytes" + "fmt" + "math" + "math/rand" + "reflect" + "sort" + "testing" + "testing/quick" + + "github.com/umbel/pilosa/roaring" +) + +func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } +func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } +func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) } +func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 10000, 10000, 20000) } +func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxUint32) } + +// Ensure a bitmap can perform basic operations on randomly generated values. +func testBitmapQuick(t *testing.T, n int, min, max uint32) { + quick.Check(func(a []uint32) bool { + bm := roaring.NewBitmap() + m := make(map[uint32]struct{}) + + // Add values to the bitmap and set. + for _, v := range a { + bm.Add(v) + m[v] = struct{}{} + } + + // Verify existence. + for _, v := range a { + // Check for individual value. + if !bm.Contains(v) { + t.Fatalf("expected bitmap to contain: %d", v) + } + + // Check for next value (which may or may not exist). + if _, ok := m[v+1]; bm.Contains(v+1) != ok { + t.Fatalf("unexpected return from Contains(%d): %v", v+1, bm.Contains(v+1)) + } + } + + // Verify slices are equal. + if got, exp := bm.Slice(), uint32SetSlice(m); !reflect.DeepEqual(got, exp) { + t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp) + } + + // Remove all values in random order. + for _, i := range rand.Perm(len(a)) { + bm.Remove(a[i]) + } + + // Verify all values have been removed. + if slice := bm.Slice(); len(slice) != 0 { + t.Fatalf("expected no values, got: %+v", slice) + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0] = reflect.ValueOf(GenerateUint32Slice(n, min, max, rand)) + }, + }) +} + +// Ensure an empty bitmap returns false if checking for existence. +func TestBitmap_Contains_Empty(t *testing.T) { + if roaring.NewBitmap().Contains(1000) { + t.Fatal("expected false") + } +} + +// Ensure an empty bitmap does nothing when removing an element. +func TestBitmap_Remove_Empty(t *testing.T) { + roaring.NewBitmap().Remove(1000) +} + +// Ensure a bitmap can return a slice of values. +func TestBitmap_Slice(t *testing.T) { + if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint32{1, 2, 3}) { + t.Fatalf("unexpected slice: %+v", a) + } +} + +// Ensure an empty bitmap returns an empty slice of values. +func TestBitmap_Slice_Empty(t *testing.T) { + if a := roaring.NewBitmap().Slice(); len(a) != 0 { + t.Fatalf("unexpected slice: %+v", a) + } +} + +// Ensure a bitmap can return a slice of values within a range. +func TestBitmap_SliceRange(t *testing.T) { + if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint32{1000001, 1000002}) { + t.Fatalf("unexpected slice: %+v", a) + } +} + +// Ensure a bitmap can loop over a set of values. +func TestBitmap_ForEach(t *testing.T) { + var a []uint32 + roaring.NewBitmap(1, 2, 3).ForEach(func(v uint32) { + a = append(a, v) + }) + if !reflect.DeepEqual(a, []uint32{1, 2, 3}) { + t.Fatalf("unexpected values: %+v", a) + } +} + +// Ensure a bitmap can loop over a set of values in a range. +func TestBitmap_ForEachRange(t *testing.T) { + var a []uint32 + roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint32) { + a = append(a, v) + }) + if !reflect.DeepEqual(a, []uint32{2, 3}) { + t.Fatalf("unexpected values: %+v", a) + } +} + +func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000) } +func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000) } +func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000) } +func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 10000, 20000) } +func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) { + testBitmapMarshalQuick(t, 100, 0, math.MaxUint32) +} + +// Ensure a bitmap can be marshaled and unmarshaled. +func testBitmapMarshalQuick(t *testing.T, n int, min, max uint32) { + quick.Check(func(a0, a1 []uint32) bool { + println("=================================================") + + // Create bitmap with initial values set. + bm := roaring.NewBitmap(a0...) + + // Write snapshot to buffer. + var buf bytes.Buffer + if n, err := bm.WriteTo(&buf); err != nil { + t.Fatal(err) + } else if n != int64(buf.Len()) { + t.Fatalf("size mismatch: %d != %d", n, buf.Len()) + } + + // Set buffer as the writer for the ops log. + bm.OpWriter = &buf + + // Add more values to bitmap. + for _, v := range a1 { + if err := bm.Add(v); err != nil { + t.Fatal(err) + } + + // Create new bitmap from ops log data. + bm2 := roaring.NewBitmap() + if err := bm2.UnmarshalBinary(buf.Bytes()); err != nil { + t.Fatal(err) + } + + // Verify the two bitmaps match. + if x, y := bm.Slice(), bm2.Slice(); !reflect.DeepEqual(x, y) { + t.Fatalf("mismatch: %s\n\nbm1=%+v\n\nbm2=%+v\n\n", diff(x, y), x, y) + } + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0] = reflect.ValueOf(GenerateUint32Slice(n, min, max, rand)) + values[1] = reflect.ValueOf(GenerateUint32Slice(100, min, max, rand)) + }, + }) +} + +// GenerateUint32Slice generates between [0, n) random uint32 numbers between min and max. +func GenerateUint32Slice(n int, min, max uint32, rand *rand.Rand) []uint32 { + a := make([]uint32, rand.Intn(n)) + for i := range a { + a[i] = min + uint32(rand.Intn(int(max-min))) + } + return a +} + +// uint32SetSlice returns the values in a uint32 set. +func uint32SetSlice(m map[uint32]struct{}) []uint32 { + a := make([]uint32, 0, len(m)) + for v := range m { + a = append(a, v) + } + sort.Sort(uint32Slice(a)) + return a +} + +// uint32Slice represents a sortable slice of uint32 numbers. +type uint32Slice []uint32 + +func (p uint32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p uint32Slice) Len() int { return len(p) } +func (p uint32Slice) Less(i, j int) bool { return p[i] < p[j] } + +func diff(a, b []uint32) string { + if len(a) != len(b) { + return fmt.Sprintf("len: %d != %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i]) + } + } + return "" +}