Merge pull request #26 from benbjohnson/roaring

Delta roaring bitmap

roaring/roaring.go

@@ -0,0 +1,723 @@
+// package roaring implements roaring bitmaps with support for incremental changes.
+package roaring
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"io"
+	"unsafe"
+)
+
+const (
+	// cookie is the first four bytes in a roaring bitmap file.
+	cookie = uint32(12346)
+
+	// headerSize is the size of the cookie and key count at the beginning of a file.
+	headerSize = 4 + 4
+
+	// bitmapN is the number of values in a container.bitmap.
+	bitmapN = (1 << 16) / 64
+)
+
+// Bitmap represents a roaring bitmap.
+type Bitmap struct {
+	keys       []uint16     // keys for containers
+	containers []*container // array and bitmap containers
+
+	// Number of operations written to the writer.
+	opN int
+
+	// Writer where operations are appended to.
+	OpWriter io.Writer
+}
+
+// NewBitmap returns a Bitmap with an initial set of values.
+func NewBitmap(a ...uint32) *Bitmap {
+	b := &Bitmap{}
+	b.Add(a...)
+	return b
+}
+
+// Add adds values to the bitmap.
+func (b *Bitmap) Add(a ...uint32) error {
+	for _, v := range a {
+		// Create an add operation.
+		op := &op{typ: opTypeAdd, value: v}
+
+		// Write operation to op log.
+		if err := b.writeOp(op); err != nil {
+			return err
+		}
+
+		// Apply to the in-memory bitmap.
+		op.apply(b)
+	}
+
+	return nil
+}
+
+func (b *Bitmap) add(v uint32) {
+	hb := highbits(v)
+	i := search(b.keys, hb)
+
+	// If index is negative then there's not an exact match
+	// and a container needs to be added.
+	if i < 0 {
+		b.insertAt(hb, newContainer(), -i-1)
+		i = -i - 1
+	}
+
+	println("DBG*", highbits(v))
+	b.containers[i].add(lowbits(v))
+}
+
+// Contains returns true if v is in the bitmap.
+func (b *Bitmap) Contains(v uint32) bool {
+	c := b.container(highbits(v))
+	if c == nil {
+		return false
+	}
+	return c.contains(lowbits(v))
+}
+
+// Remove removes values from the bitmap.
+func (b *Bitmap) Remove(a ...uint32) error {
+	for _, v := range a {
+		// Create an add operation.
+		op := &op{typ: opTypeRemove, value: v}
+
+		// Write operation to op log.
+		if err := b.writeOp(op); err != nil {
+			return err
+		}
+
+		// Apply operation to the bitmap.
+		op.apply(b)
+	}
+	return nil
+}
+
+func (b *Bitmap) remove(v uint32) {
+	hb := highbits(v)
+	i := search(b.keys, hb)
+	if i < 0 {
+		return
+	}
+	b.containers[i].remove(lowbits(v))
+}
+
+// Slice returns a slice of all integers in the bitmap.
+func (b *Bitmap) Slice() []uint32 {
+	var a []uint32
+	itr := b.iterator()
+	for v := itr.Seek(0); !itr.EOF(); v = itr.Next() {
+		a = append(a, v)
+	}
+	return a
+}
+
+// SliceRange returns a slice of integers between [start, end).
+func (b *Bitmap) SliceRange(start, end uint32) []uint32 {
+	var a []uint32
+	itr := b.iterator()
+	for v := itr.Seek(start); !itr.EOF() && v < end; v = itr.Next() {
+		a = append(a, v)
+	}
+	return a
+}
+
+// ForEach executes fn for each value in the bitmap.
+func (b *Bitmap) ForEach(fn func(uint32)) {
+	itr := b.iterator()
+	for v := itr.Seek(0); !itr.EOF(); v = itr.Next() {
+		fn(v)
+	}
+}
+
+// ForEachRange executes fn for each value in the bitmap between [start, end).
+func (b *Bitmap) ForEachRange(start, end uint32, fn func(uint32)) {
+	itr := b.iterator()
+	for v := itr.Seek(start); !itr.EOF() && v < end; v = itr.Next() {
+		fn(v)
+	}
+}
+
+// container returns the container with the given key.
+func (b *Bitmap) container(key uint16) *container {
+	i := search(b.keys, key)
+	if i < 0 {
+		return nil
+	}
+	return b.containers[i]
+}
+
+func (b *Bitmap) insertAt(key uint16, c *container, i int) {
+	b.keys = append(b.keys, 0)
+	copy(b.keys[i+1:], b.keys[i:])
+	b.keys[i] = key
+
+	b.containers = append(b.containers, nil)
+	copy(b.containers[i+1:], b.containers[i:])
+	b.containers[i] = c
+}
+
+// WriteTo writes b to w.
+func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
+	// Build header before writing individual container blocks.
+	buf := make([]byte, headerSize+(len(b.keys)*(2+2+4)))
+	binary.LittleEndian.PutUint32(buf[0:], cookie)
+	binary.LittleEndian.PutUint32(buf[4:], uint32(len(b.keys)))
+
+	// Encode keys and cardinality.
+	for i, key := range b.keys {
+		binary.LittleEndian.PutUint16(buf[headerSize+i*4:], uint16(key))
+		binary.LittleEndian.PutUint16(buf[headerSize+i*4+2:], uint16(b.containers[i].n-1))
+	}
+
+	// Write the offset for each container block.
+	offset := uint32(len(buf))
+	for i, c := range b.containers {
+		binary.LittleEndian.PutUint32(buf[headerSize+(len(b.keys)*4)+(i*4):], uint32(offset))
+		offset += uint32(c.size())
+	}
+
+	// Write header.
+	i, err := w.Write(buf)
+	n += int64(i)
+	if err != nil {
+		return n, err
+	}
+
+	// Write each container block.
+	for _, c := range b.containers {
+		nn, err := c.WriteTo(w)
+		n += nn
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}
+
+// UnmarshalBinary decodes b from a binary-encoded byte slice.
+func (b *Bitmap) UnmarshalBinary(data []byte) error {
+	if len(data) < headerSize {
+		return errors.New("data too small")
+	}
+
+	// Verify the first 4 bytes are the correct cookie.
+	if v := binary.LittleEndian.Uint32(data[0:4]); v != cookie {
+		return errors.New("invalid roaring file")
+	}
+
+	// Read key count.
+	keyN := binary.LittleEndian.Uint32(data[4:8])
+	b.keys = make([]uint16, keyN)
+	b.containers = make([]*container, keyN)
+
+	// Read container key headers.
+	for i, buf := 0, data[8:]; i < int(keyN); i, buf = i+1, buf[4:] {
+		b.keys[i] = binary.LittleEndian.Uint16(buf[0:2])
+		b.containers[i] = &container{n: int(binary.LittleEndian.Uint16(buf[2:4])) + 1}
+	}
+
+	// Read container offsets and attach data.
+	opsOffset := 8 + int(keyN)*4
+	for i, buf := 0, data[8+int(keyN)*4:]; i < int(keyN); i, buf = i+1, buf[4:] {
+		offset := binary.LittleEndian.Uint32(buf[0:4])
+
+		// Verify the offset is within the bounds of the input data.
+		if int(offset) >= len(data) {
+			return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
+		}
+
+		// Map byte slice directly to the container data.
+		c := b.containers[i]
+		if c.n <= arrayMaxSize {
+			c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]
+			opsOffset = int(offset) + len(c.array)*2
+		} else {
+			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
+			opsOffset = int(offset) + len(c.bitmap)*8
+		}
+	}
+
+	// Read ops log until the end of the file.
+	buf := data[opsOffset:]
+	for {
+		// Exit when there are no more ops to parse.
+		if len(buf) == 0 {
+			break
+		}
+
+		// Unmarshal the op and apply it.
+		var op op
+		if err := op.UnmarshalBinary(buf); err != nil {
+			// FIXME(benbjohnson): return error with position so file can be trimmed.
+			return err
+		}
+		op.apply(b)
+
+		// Move the buffer forward.
+		buf = buf[op.size():]
+	}
+
+	return nil
+}
+
+// writeOp writes op to the OpWriter, if available.
+func (b *Bitmap) writeOp(op *op) error {
+	if b.OpWriter == nil {
+		return nil
+	}
+
+	if _, err := op.WriteTo(b.OpWriter); err != nil {
+		return err
+	}
+
+	b.opN++
+	return nil
+}
+
+// iterator returns an iterator for the bitmap.
+func (b *Bitmap) iterator() *iterator { return &iterator{bitmap: b} }
+
+// iterator represents an iterator over a Bitmap.
+type iterator struct {
+	bitmap *Bitmap
+	i, j   int
+}
+
+// EOF returns true if the iterator is at the end of the bitmap.
+func (itr *iterator) EOF() bool { return itr.i >= len(itr.bitmap.containers) }
+
+// Seek moves to the first value equal to or greater than v.
+func (itr *iterator) Seek(seek uint32) uint32 {
+	// Move to the correct container.
+	itr.i = search(itr.bitmap.keys, highbits(seek))
+	if itr.i < 0 {
+		itr.i = -itr.i - 1
+	}
+	if itr.EOF() {
+		return 0
+	}
+
+	// Move to the correct value index inside the array container.
+	lb := lowbits(seek)
+	if c := itr.bitmap.containers[itr.i]; c.isArray() {
+		// Find index in the container.
+		itr.j = search(c.array, lb)
+		if itr.j < 0 {
+			itr.j = -itr.j - 1
+		}
+		if itr.j < len(c.array) {
+			return itr.peek()
+		}
+
+		// If it's at the end of the container then move to the next one.
+		itr.i, itr.j = itr.i+1, -1
+		return itr.Next()
+	}
+
+	// If it's a bitmap container then move to index before the value and call next().
+	itr.j = int(lb) - 1
+	return itr.Next()
+}
+
+// Next returns the next value in the bitmap.
+func (itr *iterator) Next() uint32 {
+	// Iterate over containers until we find the next value or EOF.
+	for {
+		if itr.EOF() {
+			return 0
+		}
+
+		// Move to the next item in the container if it's an array container.
+		c := itr.bitmap.containers[itr.i]
+		if c.isArray() {
+			if itr.j >= c.n-1 {
+				itr.i, itr.j = itr.i+1, -1
+				continue
+			}
+			itr.j++
+			return itr.peek()
+		}
+
+		// Move to the next possible index in the bitmap container.
+		itr.j++
+
+		// Find first non-zero bit in current bitmap, if possible.
+		hb := int(itr.j / 64)
+		lb := c.bitmap[hb] >> (uint(itr.j) % 64)
+		if lb != 0 {
+			itr.j = int(itr.j) + trailingZeroN(lb)
+			return itr.peek()
+		}
+
+		// Otherwise iterate through remaining bitmaps to find next bit.
+		for hb++; hb < len(c.bitmap); hb++ {
+			if c.bitmap[hb] != 0 {
+				itr.j = int(hb*64) + trailingZeroN(c.bitmap[hb])
+				return itr.peek()
+			}
+		}
+
+		// If no bits found then move to the next container.
+		itr.i, itr.j = itr.i+1, -1
+	}
+}
+
+// peek returns the current value.
+func (itr *iterator) peek() uint32 {
+	key := itr.bitmap.keys[itr.i]
+	c := itr.bitmap.containers[itr.i]
+	if c.isArray() {
+		return uint32(key)<<16 | uint32(c.array[itr.j])
+	}
+	return uint32(key)<<16 | uint32(itr.j)
+}
+
+// The maximum size of array containers.
+const arrayMaxSize = 4096
+
+// container represents a container for uint16 integers.
+//
+// These are used for storing the low bits. Containers are separated into two
+// types depending on cardinality. For containers with less than 4,096 values,
+// an array container is used. For containers with more than 4,096 values,
+// the values are encoded into bitmaps.
+type container struct {
+	n      int      // number of integers in container
+	array  []uint16 // used for array containers
+	bitmap []uint64 // used for bitmap containers
+	mapped bool     // mapped directly to a byte slice when true
+}
+
+// newContainer returns a new instance of container.
+func newContainer() *container {
+	return &container{}
+}
+
+// isArray returns true if the container is an array container.
+func (c *container) isArray() bool { return c.bitmap == nil }
+
+// unmap creates copies of the containers data in the heap.
+//
+// This is performed when altering the container since its contents could be
+// pointing at a read-only mmap.
+func (c *container) unmap() {
+	if !c.mapped {
+		return
+	}
+
+	if c.array != nil {
+		tmp := make([]uint16, len(c.array))
+		copy(tmp, c.array)
+		c.array = tmp
+	}
+	if c.bitmap != nil {
+		tmp := make([]uint64, len(c.bitmap))
+		copy(tmp, c.bitmap)
+		c.bitmap = tmp
+	}
+	c.mapped = false
+}
+
+// add adds a value to the container.
+func (c *container) add(v uint16) {
+	if c.isArray() {
+		c.arrayAdd(v)
+		return
+	}
+	c.bitmapAdd(v)
+}
+
+func (c *container) arrayAdd(v uint16) {
+	// Optimize appending to the end of an array container.
+	if c.n > 0 && c.isArray() && c.array[c.n-1] < v {
+		c.unmap()
+		c.array = append(c.array, v)
+		c.n++
+		return
+	}
+
+	// Find index of the integer in the container. Exit if it already exists.
+	i := search(c.array, v)
+	if i >= 0 {
+		return
+	}
+
+	// Convert to a bitmap container if too many values are in an array container.
+	if c.n >= arrayMaxSize {
+		c.convertToBitmap()
+		c.bitmapAdd(v)
+		return
+	}
+
+	// Otherwise insert into array.
+	c.unmap()
+	println("DBG&&&&&", v)
+	i = -i - 1
+	c.array = append(c.array, 0)
+	copy(c.array[i+1:], c.array[i:])
+	c.array[i] = v
+	c.n++
+}
+
+func (c *container) bitmapAdd(v uint16) {
+	if c.bitmapContains(v) {
+		return
+	}
+	c.unmap()
+	c.bitmap[v/64] |= (1 << uint64(v%64))
+	c.n++
+}
+
+// contains returns true if v is in the container.
+func (c *container) contains(v uint16) bool {
+	if c.isArray() {
+		return c.arrayContains(v)
+	}
+	return c.bitmapContains(v)
+}
+
+func (c *container) arrayContains(v uint16) bool {
+	return search(c.array, v) >= 0
+}
+
+func (c *container) bitmapContains(v uint16) bool {
+	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
+}
+
+// remove adds a value to the container.
+func (c *container) remove(v uint16) {
+	if c.isArray() {
+		c.arrayRemove(v)
+		return
+	}
+	c.bitmapRemove(v)
+}
+
+func (c *container) arrayRemove(v uint16) {
+	i := search(c.array, v)
+	if i < 0 {
+		return
+	}
+	c.unmap()
+
+	c.n--
+	c.array = append(c.array[:i], c.array[i+1:]...)
+}
+
+func (c *container) bitmapRemove(v uint16) {
+	if !c.bitmapContains(v) {
+		return
+	}
+	c.unmap()
+
+	// Lower count and remove element.
+	c.n--
+	c.bitmap[v/64] &^= (uint64(1) << (v % 64))
+
+	// Convert to array if we go below the threshold.
+	if c.n == arrayMaxSize {
+		c.convertToArray()
+	}
+}
+
+// convertToArray converts the values in the bitmap to array values.
+func (c *container) convertToArray() {
+	c.array = make([]uint16, 0, c.n)
+	for i, bitmap := range c.bitmap {
+		for bitmap != 0 {
+			t := bitmap & -bitmap
+			c.array = append(c.array, uint16((i*64 + int(popcount(t-1)))))
+			bitmap ^= t
+		}
+	}
+	c.bitmap = nil
+	c.mapped = false
+}
+
+// convertToBitmap converts the values in array to bitmap values.
+func (c *container) convertToBitmap() {
+	c.bitmap = make([]uint64, bitmapN)
+	for _, v := range c.array {
+		c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
+	}
+	c.array = nil
+	c.mapped = false
+}
+
+// WriteTo writes c to w.
+func (c *container) WriteTo(w io.Writer) (n int64, err error) {
+	if c.isArray() {
+		return c.arrayWriteTo(w)
+	}
+	return c.bitmapWriteTo(w)
+}
+
+func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n])
+	return int64(nn), err
+}
+
+func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)])
+	return int64(nn), err
+}
+
+// size returns the encoded size of the container, in bytes.
+func (c *container) size() int {
+	if c.isArray() {
+		return len(c.array) * 2
+	}
+	return len(c.bitmap) * 8
+}
+
+// opType represents a type of operation.
+type opType uint8
+
+const (
+	opTypeAdd    = opType(0)
+	opTypeRemove = opType(1)
+)
+
+// op represents an operation on the bitmap.
+type op struct {
+	typ   opType
+	value uint32
+}
+
+// apply executes the operation against a bitmap.
+func (op *op) apply(b *Bitmap) {
+	switch op.typ {
+	case opTypeAdd:
+		b.add(op.value)
+	case opTypeRemove:
+		b.remove(op.value)
+	default:
+		panic(fmt.Sprintf("invalid op type: %d", op.typ))
+	}
+}
+
+// WriteTo writes op to the w.
+func (op *op) WriteTo(w io.Writer) (n int64, err error) {
+	buf := make([]byte, op.size())
+
+	// Write type and value.
+	buf[0] = byte(op.typ)
+	binary.LittleEndian.PutUint32(buf[1:5], op.value)
+
+	// Add checksum at the end.
+	h := fnv.New32a()
+	h.Write(buf[0:5])
+	binary.LittleEndian.PutUint32(buf[5:9], h.Sum32())
+	fmt.Println("")
+	fmt.Println("W<<<<<<<<<<<<", op.value)
+
+	// Write to writer.
+	nn, err := w.Write(buf)
+	return int64(nn), err
+}
+
+// UnmarshalBinary decodes data into an op.
+func (op *op) UnmarshalBinary(data []byte) error {
+	if len(data) < op.size() {
+		return fmt.Errorf("op data out of bounds: len=%d", len(data))
+	}
+
+	// Verify checksum.
+	h := fnv.New32a()
+	h.Write(data[0:5])
+	if chk := binary.LittleEndian.Uint32(data[5:9]); chk != h.Sum32() {
+		return fmt.Errorf("checksum mismatch: exp=%08x, got=%08x", h.Sum32(), chk)
+	}
+
+	// Read type and value.
+	op.typ = opType(data[0])
+	op.value = binary.LittleEndian.Uint32(data[1:5])
+	fmt.Println("R>", op.value)
+
+	return nil
+}
+
+// size returns the encoded size of the op, in bytes.
+func (*op) size() int { return 1 + 4 + 4 }
+
+func highbits(v uint32) uint16 { return uint16(v >> 16) }
+func lowbits(v uint32) uint16  { return uint16(v & 0xFFFF) }
+
+// search returns the index of v in a.
+func search(a []uint16, value uint16) int {
+	// Optimize for elements and the last element.
+	n := len(a)
+	if n == 0 {
+		return -1
+	} else if a[n-1] == value {
+		return n - 1
+	}
+
+	// Otherwise perform binary search for exact match.
+	lo, hi := 0, n-1
+	for lo+16 <= hi {
+		i := int(uint((lo + hi)) >> 1)
+		v := a[i]
+
+		if v < value {
+			lo = i + 1
+		} else if v > value {
+			hi = i - 1
+		} else {
+			return i
+		}
+	}
+
+	// If an exact match isn't found then return a negative index.
+	for ; lo <= hi; lo++ {
+		v := a[lo]
+		if v == value {
+			return lo
+		} else if v > value {
+			break
+		}
+	}
+	return -(lo + 1)
+}
+
+// trailingZeroN returns the number of trailing zeros in v.
+// v must be greater than zero.
+func trailingZeroN(v uint64) int {
+	n := int64(63)
+	if y := v << 32; y != 0 {
+		n, v = n-32, y
+	}
+	if y := v << 16; y != 0 {
+		n, v = n-16, y
+	}
+	if y := v << 8; y != 0 {
+		n, v = n-8, y
+	}
+	if y := v << 4; y != 0 {
+		n, v = n-4, y
+	}
+	if y := v << 2; y != 0 {
+		n, v = n-2, y
+	}
+	return int(n - int64(uint64(v<<1)>>63))
+}
+
+// bit population count, taken from
+// https://code.google.com/p/go/issues/detail?id=4988#c11
+// credit: https://code.google.com/u/arnehormann/
+func popcount(x uint64) (n uint64) {
+	x -= (x >> 1) & 0x5555555555555555
+	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
+	x += x >> 4
+	x &= 0x0f0f0f0f0f0f0f0f
+	x *= 0x0101010101010101
+	return x >> 56
+}

roaring/roaring_test.go

@@ -0,0 +1,215 @@
+package roaring_test
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"math/rand"
+	"reflect"
+	"sort"
+	"testing"
+	"testing/quick"
+
+	"github.com/umbel/pilosa/roaring"
+)
+
+func TestBitmap_Quick_Array1(t *testing.T)     { testBitmapQuick(t, 1000, 1000, 2000) }
+func TestBitmap_Quick_Array2(t *testing.T)     { testBitmapQuick(t, 10000, 0, 1000) }
+func TestBitmap_Quick_Bitmap1(t *testing.T)    { testBitmapQuick(t, 10000, 0, 10000) }
+func TestBitmap_Quick_Bitmap2(t *testing.T)    { testBitmapQuick(t, 10000, 10000, 20000) }
+func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxUint32) }
+
+// Ensure a bitmap can perform basic operations on randomly generated values.
+func testBitmapQuick(t *testing.T, n int, min, max uint32) {
+	quick.Check(func(a []uint32) bool {
+		bm := roaring.NewBitmap()
+		m := make(map[uint32]struct{})
+
+		// Add values to the bitmap and set.
+		for _, v := range a {
+			bm.Add(v)
+			m[v] = struct{}{}
+		}
+
+		// Verify existence.
+		for _, v := range a {
+			// Check for individual value.
+			if !bm.Contains(v) {
+				t.Fatalf("expected bitmap to contain: %d", v)
+			}
+
+			// Check for next value (which may or may not exist).
+			if _, ok := m[v+1]; bm.Contains(v+1) != ok {
+				t.Fatalf("unexpected return from Contains(%d): %v", v+1, bm.Contains(v+1))
+			}
+		}
+
+		// Verify slices are equal.
+		if got, exp := bm.Slice(), uint32SetSlice(m); !reflect.DeepEqual(got, exp) {
+			t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp)
+		}
+
+		// Remove all values in random order.
+		for _, i := range rand.Perm(len(a)) {
+			bm.Remove(a[i])
+		}
+
+		// Verify all values have been removed.
+		if slice := bm.Slice(); len(slice) != 0 {
+			t.Fatalf("expected no values, got: %+v", slice)
+		}
+
+		return true
+	}, &quick.Config{
+		Values: func(values []reflect.Value, rand *rand.Rand) {
+			values[0] = reflect.ValueOf(GenerateUint32Slice(n, min, max, rand))
+		},
+	})
+}
+
+// Ensure an empty bitmap returns false if checking for existence.
+func TestBitmap_Contains_Empty(t *testing.T) {
+	if roaring.NewBitmap().Contains(1000) {
+		t.Fatal("expected false")
+	}
+}
+
+// Ensure an empty bitmap does nothing when removing an element.
+func TestBitmap_Remove_Empty(t *testing.T) {
+	roaring.NewBitmap().Remove(1000)
+}
+
+// Ensure a bitmap can return a slice of values.
+func TestBitmap_Slice(t *testing.T) {
+	if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint32{1, 2, 3}) {
+		t.Fatalf("unexpected slice: %+v", a)
+	}
+}
+
+// Ensure an empty bitmap returns an empty slice of values.
+func TestBitmap_Slice_Empty(t *testing.T) {
+	if a := roaring.NewBitmap().Slice(); len(a) != 0 {
+		t.Fatalf("unexpected slice: %+v", a)
+	}
+}
+
+// Ensure a bitmap can return a slice of values within a range.
+func TestBitmap_SliceRange(t *testing.T) {
+	if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint32{1000001, 1000002}) {
+		t.Fatalf("unexpected slice: %+v", a)
+	}
+}
+
+// Ensure a bitmap can loop over a set of values.
+func TestBitmap_ForEach(t *testing.T) {
+	var a []uint32
+	roaring.NewBitmap(1, 2, 3).ForEach(func(v uint32) {
+		a = append(a, v)
+	})
+	if !reflect.DeepEqual(a, []uint32{1, 2, 3}) {
+		t.Fatalf("unexpected values: %+v", a)
+	}
+}
+
+// Ensure a bitmap can loop over a set of values in a range.
+func TestBitmap_ForEachRange(t *testing.T) {
+	var a []uint32
+	roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint32) {
+		a = append(a, v)
+	})
+	if !reflect.DeepEqual(a, []uint32{2, 3}) {
+		t.Fatalf("unexpected values: %+v", a)
+	}
+}
+
+func TestBitmap_Marshal_Quick_Array1(t *testing.T)  { testBitmapMarshalQuick(t, 1000, 1000, 2000) }
+func TestBitmap_Marshal_Quick_Array2(t *testing.T)  { testBitmapMarshalQuick(t, 10000, 0, 1000) }
+func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000) }
+func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 10000, 20000) }
+func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) {
+	testBitmapMarshalQuick(t, 100, 0, math.MaxUint32)
+}
+
+// Ensure a bitmap can be marshaled and unmarshaled.
+func testBitmapMarshalQuick(t *testing.T, n int, min, max uint32) {
+	quick.Check(func(a0, a1 []uint32) bool {
+		println("=================================================")
+
+		// Create bitmap with initial values set.
+		bm := roaring.NewBitmap(a0...)
+
+		// Write snapshot to buffer.
+		var buf bytes.Buffer
+		if n, err := bm.WriteTo(&buf); err != nil {
+			t.Fatal(err)
+		} else if n != int64(buf.Len()) {
+			t.Fatalf("size mismatch: %d != %d", n, buf.Len())
+		}
+
+		// Set buffer as the writer for the ops log.
+		bm.OpWriter = &buf
+
+		// Add more values to bitmap.
+		for _, v := range a1 {
+			if err := bm.Add(v); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create new bitmap from ops log data.
+			bm2 := roaring.NewBitmap()
+			if err := bm2.UnmarshalBinary(buf.Bytes()); err != nil {
+				t.Fatal(err)
+			}
+
+			// Verify the two bitmaps match.
+			if x, y := bm.Slice(), bm2.Slice(); !reflect.DeepEqual(x, y) {
+				t.Fatalf("mismatch: %s\n\nbm1=%+v\n\nbm2=%+v\n\n", diff(x, y), x, y)
+			}
+		}
+
+		return true
+	}, &quick.Config{
+		Values: func(values []reflect.Value, rand *rand.Rand) {
+			values[0] = reflect.ValueOf(GenerateUint32Slice(n, min, max, rand))
+			values[1] = reflect.ValueOf(GenerateUint32Slice(100, min, max, rand))
+		},
+	})
+}
+
+// GenerateUint32Slice generates between [0, n) random uint32 numbers between min and max.
+func GenerateUint32Slice(n int, min, max uint32, rand *rand.Rand) []uint32 {
+	a := make([]uint32, rand.Intn(n))
+	for i := range a {
+		a[i] = min + uint32(rand.Intn(int(max-min)))
+	}
+	return a
+}
+
+// uint32SetSlice returns the values in a uint32 set.
+func uint32SetSlice(m map[uint32]struct{}) []uint32 {
+	a := make([]uint32, 0, len(m))
+	for v := range m {
+		a = append(a, v)
+	}
+	sort.Sort(uint32Slice(a))
+	return a
+}
+
+// uint32Slice represents a sortable slice of uint32 numbers.
+type uint32Slice []uint32
+
+func (p uint32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p uint32Slice) Len() int           { return len(p) }
+func (p uint32Slice) Less(i, j int) bool { return p[i] < p[j] }
+
+func diff(a, b []uint32) string {
+	if len(a) != len(b) {
+		return fmt.Sprintf("len: %d != %d", len(a), len(b))
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i])
+		}
+	}
+	return ""
+}