merge

Godeps/Godeps.json

@@ -28,10 +28,6 @@
 			"ImportPath": "github.com/golang/groupcache/lru",
 			"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
 		},
-		{
-			"ImportPath": "github.com/yasushi-saito/rbtree",
-			"Rev": "571e2538414bf914c7e2909b61217b4e3e5508f4"
-		},
 		{
 			"ImportPath": "golang.org/x/sys/unix",
 			"Rev": "50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e"

bitmap.go

@@ -3,33 +3,16 @@ package pilosa
 // #cgo  CFLAGS:-mpopcnt
 
 import (
-	"bytes"
-	"compress/gzip"
-	"encoding/base64"
-	"encoding/binary"
-	"encoding/gob"
 	"encoding/json"
-	"io"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/umbel/pilosa/internal"
-	"github.com/yasushi-saito/rbtree"
+	"github.com/umbel/pilosa/roaring"
 )
 
-const (
-	CounterMask = uint64(0xffffffffffffffff)
-)
-
-var (
-	CounterKey  = int64(-1)
-	EMPTY_BLOCK = make(Blocks, 32)
-)
-
-// Bitmap represents a bitmap broken up into Chunks.
-// Internally it is represented as a red-black tree of chunks.
+// Bitmap represents a set of bits.
 type Bitmap struct {
-	tree   *rbtree.Tree
-	bcount uint64
+	data roaring.Bitmap
+	n    uint64
 
 	// Attributes associated with the bitmap.
 	Attrs map[string]interface{}
@@ -37,166 +20,98 @@ type Bitmap struct {
 
 // NewBitmap returns a new instance of Bitmap.
 func NewBitmap(bits ...uint64) *Bitmap {
-	bm := &Bitmap{tree: rbtree.NewTree(rbtreeItemCompare)}
+	bm := &Bitmap{}
 	for _, i := range bits {
 		bm.SetBit(i)
 	}
 	return bm
 }
 
-// Chunk returns the chunk within the bitmap.
-// Returns nil if the chunk key does not exist.
-func (b *Bitmap) Chunk(c *Chunk) *Chunk {
-	if n := b.tree.Get(c); n != nil {
-		return n.(*Chunk)
-	}
-	return nil
-}
-
-// AddChunk adds c to the bitmap.
-func (b *Bitmap) AddChunk(c *Chunk) { b.tree.Insert(c) }
-
-// Chunks returns a list of all chunks.
-func (b *Bitmap) Chunks() []*Chunk {
-	var a []*Chunk
-	for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
-		a = append(a, itr.Item().Clone())
-	}
-	return a
-}
-
-// ChunkIterator returns an iterator for looping over the bitmap's chunks.
-func (b *Bitmap) ChunkIterator() *ChunkIterator {
-	return &ChunkIterator{b.tree.Min()}
-}
-
-// Clone returns a copy of b.
-func (b *Bitmap) Clone() *Bitmap {
-	itr := b.ChunkIterator()
-	other := NewBitmap()
-
-	for {
-		if itr.Limit() {
-			break
-		}
-
-		other.AddChunk(itr.Item().Clone())
-		itr = itr.Next()
-	}
-	return other
-}
-
 // Merge adds chunks from other to b.
 // Chunks in b are overwritten if they exist in other.
 func (b *Bitmap) Merge(other *Bitmap) {
-	for itr := other.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
-		b.AddChunk(itr.Item().Clone())
+	itr := other.data.Iterator()
+	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
+		b.SetBit(v)
 	}
 }
 
 // IntersectionCount returns the number of intersections between b and other.
 func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
-	itr0 := b.ChunkIterator()
-	itr1 := other.ChunkIterator()
+	// OPTIMIZE: Implement roaring.Bitmap.IntersectionCount()
 
-	results := uint64(0)
+	itr0 := roaring.NewBufIterator(b.data.Iterator())
+	itr1 := roaring.NewBufIterator(other.data.Iterator())
+
+	var n uint64
 	for {
-		if itr1.Limit() || itr0.Limit() {
+		v0, eof0 := itr0.Next()
+		v1, eof1 := itr1.Next()
+
+		if eof0 || eof1 {
 			break
-		} else if itr0.Item().Key < itr1.Item().Key {
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key > itr1.Item().Key {
-			itr1 = itr1.Next()
-		} else if itr0.Item().Key == itr1.Item().Key {
-			results += itr0.Item().Value.andcount(itr1.Item().Value)
-			itr0 = itr0.Next()
-			itr1 = itr1.Next()
+		} else if v0 < v1 {
+			itr1.Unread()
+		} else if v0 > v1 {
+			itr0.Unread()
+		} else {
+			n++
 		}
 	}
-	return results
+	return n
 }
 
 // Intersect returns the itersection of b and other.
 func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
-	itr0 := b.ChunkIterator()
-	itr1 := other.ChunkIterator()
+	// OPTIMIZE: Implement roaring.Bitmap.Intersect()
+
+	itr0 := roaring.NewBufIterator(b.data.Iterator())
+	itr1 := roaring.NewBufIterator(other.data.Iterator())
 
 	output := NewBitmap()
 	for {
-		if itr1.Limit() || itr0.Limit() {
+		v0, eof0 := itr0.Next()
+		v1, eof1 := itr1.Next()
+
+		if eof0 || eof1 {
 			break
-		} else if itr0.Item().Key < itr1.Item().Key {
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key > itr1.Item().Key {
-			itr1 = itr1.Next()
-		} else if itr0.Item().Key == itr1.Item().Key {
-			output.AddChunk(&Chunk{
-				Key:   itr0.Item().Key,
-				Value: itr0.Item().Value.intersect(itr1.Item().Value),
-			})
-			itr0 = itr0.Next()
-			itr1 = itr1.Next()
+		} else if v0 < v1 {
+			itr1.Unread()
+		} else if v0 > v1 {
+			itr0.Unread()
+		} else {
+			output.SetBit(v0)
 		}
 	}
 	return output
 }
 
-// Invert returns a bitwise inversion of b.
-func (b *Bitmap) Invert() *Bitmap {
-	other := NewBitmap()
-	for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
-		other.AddChunk(&Chunk{
-			Key:   i.Item().Key,
-			Value: i.Item().Value.invert(),
-		})
-	}
-	return other
-
-}
-
 // Union returns the bitwise union of b and other.
 func (b *Bitmap) Union(other *Bitmap) *Bitmap {
-	itr0 := b.ChunkIterator()
-	itr1 := other.ChunkIterator()
+	// OPTIMIZE: Implement roaring.Bitmap.Union()
+
+	itr0 := roaring.NewBufIterator(b.data.Iterator())
+	itr1 := roaring.NewBufIterator(other.data.Iterator())
 
 	output := NewBitmap()
-	eof := uint64(0xdeadbeef)
-
 	for {
-		if itr0.Limit() && itr1.Limit() {
+		v0, eof0 := itr0.Next()
+		v1, eof1 := itr1.Next()
+
+		if eof0 && eof1 {
 			break
-		} else if itr0.Limit() {
-			if eof == itr1.Item().Key {
-				break
-			}
-			output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value})
-			eof = itr1.Item().Key
-			itr1 = itr1.Next()
-		} else if itr1.Limit() {
-			if eof == itr0.Item().Key {
-				break
-			}
-			output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
-			eof = itr0.Item().Key
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key < itr1.Item().Key {
-			output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
-			eof = itr0.Item().Key
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key > itr1.Item().Key {
-			output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value})
-			eof = itr1.Item().Key
-			itr1 = itr1.Next()
-		} else if itr0.Item().Key == itr1.Item().Key {
-			output.AddChunk(&Chunk{
-				Key:   itr0.Item().Key,
-				Value: itr0.Item().Value.union(itr1.Item().Value),
-			})
-			eof = itr0.Item().Key
-			itr0 = itr0.Next()
-			itr1 = itr1.Next()
+		} else if eof0 {
+			output.SetBit(v1)
+		} else if eof1 {
+			output.SetBit(v0)
+		} else if v0 < v1 {
+			output.SetBit(v0)
+			itr1.Unread()
+		} else if v0 > v1 {
+			output.SetBit(v1)
+			itr0.Unread()
 		} else {
-			panic("unreachable")
+			output.SetBit(v0)
 		}
 	}
 	return output
@@ -204,118 +119,30 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap {
 
 // Difference returns the diff of b and other.
 func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
-	itr0 := b.ChunkIterator()
-	itr1 := other.ChunkIterator()
+	// OPTIMIZE: Implement roaring.Bitmap.Difference()
+
+	itr0 := roaring.NewBufIterator(b.data.Iterator())
+	itr1 := roaring.NewBufIterator(other.data.Iterator())
 
 	output := NewBitmap()
 	for {
-		if itr0.Limit() && itr1.Limit() {
-			break
-		} else if itr0.Limit() {
-			break
-		} else if itr1.Limit() {
-			output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key < itr1.Item().Key {
-			output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
-			itr0 = itr0.Next()
-		} else if itr0.Item().Key > itr1.Item().Key {
-			itr1 = itr1.Next()
-		} else if itr0.Item().Key == itr1.Item().Key {
-			chunk := &Chunk{
-				Key:   itr0.Item().Key,
-				Value: itr0.Item().Value.difference(itr1.Item().Value),
-			}
+		v0, eof0 := itr0.Next()
+		v1, eof1 := itr1.Next()
 
-			// Do not add if all bits are zeroed.
-			if chunk.Value.bitcount() > 0 {
-				output.AddChunk(chunk)
-			}
-
-			itr0 = itr0.Next()
-			itr1 = itr1.Next()
-		} else {
-			panic("unreachable")
+		if eof0 {
+			break
+		} else if eof1 {
+			output.SetBit(v0)
+		} else if v0 < v1 {
+			output.SetBit(v0)
+			itr1.Unread()
+		} else if v0 > v1 {
+			itr0.Unread()
 		}
 	}
 	return output
 }
 
-// ToRawCompressString returns a compressed, hex-encoded string of b.
-func (b *Bitmap) ToRawCompressString() (string, int) {
-	var bt bytes.Buffer
-	buf := gzip.NewWriter(&bt)
-	binary.Write(buf, binary.LittleEndian, uint64(b.tree.Len()))
-	max_slice := 0
-	for i := b.tree.Min(); !i.Limit(); i = i.Next() {
-		obj := i.Item().(*Chunk)
-		max_slice = int(obj.Key)
-		binary.Write(buf, binary.LittleEndian, obj.Key)
-		for _, v := range obj.Value {
-			binary.Write(buf, binary.LittleEndian, v)
-		}
-	}
-	buf.Flush()
-	//buf.Close()
-	max_slice = max_slice / 32
-	return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice
-}
-
-// WriteTo writes the encoded bitmap to w.
-func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
-	// Wrap output in gzip compression.
-	z := gzip.NewWriter(w)
-
-	// Encode chunk count.
-	enc := gob.NewEncoder(w)
-	if err := enc.Encode(b.tree.Len()); err != nil {
-		return 0, err
-	}
-
-	// Encode all chunks.
-	for i := b.tree.Min(); !i.Limit(); i = i.Next() {
-		if err := enc.Encode(i.Item().(*Chunk)); err != nil {
-			return 0, err
-		}
-	}
-
-	// Flush and close.
-	if err := z.Close(); err != nil {
-		return 0, err
-	}
-
-	return 0, nil
-}
-
-// ReadFrom reads encoded bitmap data from r into b.
-func (b *Bitmap) ReadFrom(r io.Reader) (n int64, err error) {
-	// Uncompress from gzip format.
-	z, err := gzip.NewReader(r)
-	if err != nil {
-		return 0, err
-	}
-	dec := gob.NewDecoder(z)
-
-	// Read size from data.
-	var size int
-	if err := dec.Decode(&size); err != nil {
-		return 0, err
-	}
-
-	// Read chunks into bitmap.
-	b.tree = rbtree.NewTree(rbtreeItemCompare)
-	for i := 0; i < size; i++ {
-		var chunk Chunk
-		if err := dec.Decode(&chunk); err != nil {
-			return 0, err
-		}
-		b.AddChunk(&chunk)
-	}
-	b.SetCount(b.BitCount())
-
-	return 0, nil
-}
-
 // MarshalJSON returns a JSON-encoded byte slice of b.
 func (b *Bitmap) MarshalJSON() ([]byte, error) {
 	var o struct {
@@ -332,94 +159,45 @@ func (b *Bitmap) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&o)
 }
 
-// MarshalBinary returns a gob-encoded byte slice of b.
-func (b *Bitmap) MarshalBinary() ([]byte, error) {
-	var buf bytes.Buffer
-	if _, err := b.WriteTo(&buf); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// UnmarshalBinary decodes a gob-encoded byte slice into b.
-func (b *Bitmap) UnmarshalBinary(data []byte) error {
-	_, err := b.ReadFrom(bytes.NewReader(data))
-	return err
-}
-
 // Bits returns the bits in b as a slice of ints.
 func (b *Bitmap) Bits() []uint64 {
-	result := make([]uint64, 0, b.Count())
-
-	for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
-		item := i.Item()
-		chunk := item.Key
-		for bi, block := range item.Value {
-			for bit := uint(0); bit < 64; bit++ {
-				if (block & (1 << bit)) != 0 {
-					idx := chunk << 11
-					idx = idx | uint64((uint(bi)<<6)|bit)
-					result = append(result, idx)
-				}
-			}
-		}
-
+	a := make([]uint64, 0, b.Count())
+	itr := b.data.Iterator()
+	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
+		a = append(a, v)
 	}
-	return result
+	return a
 }
 
-// setBit sets the i-th bit of the bitmap.
+// SetBit sets the i-th bit of the bitmap.
 func (b *Bitmap) SetBit(i uint64) (changed bool) {
-	address := deref(i)
-
-	chunk := b.Chunk(&Chunk{address.ChunkKey, EMPTY_BLOCK})
-	if chunk == nil {
-		chunk = &Chunk{address.ChunkKey, make(Blocks, 32)}
-		b.AddChunk(chunk)
-	}
-
-	changed = chunk.Value.setBit(address.BlockIndex, address.Bit)
+	changed, _ = b.data.Add(i)
 	if changed {
-		b.bcount++
+		b.n++
 	}
-
 	return changed
 }
 
-// clearBit clears the i-th bit of the bitmap.
+// ClearBit clears the i-th bit of the bitmap.
 func (b *Bitmap) ClearBit(i uint64) (changed bool) {
-	address := deref(i)
-
-	chunk := b.Chunk(&Chunk{address.ChunkKey, EMPTY_BLOCK})
-	if chunk == nil {
-		return false
+	changed, _ = b.data.Remove(i)
+	if changed {
+		b.n--
 	}
-
-	changed = chunk.Value.clearBit(address.BlockIndex, address.Bit)
-	if changed && b.bcount > 0 {
-		b.bcount--
-	}
-
 	return changed
 }
 
-// Len returns the number of chunks in b.
-func (b *Bitmap) Len() int { return b.tree.Len() }
-
-// SetCount sets the number of set bits in the bitmap.
-func (b *Bitmap) SetCount(c uint64) { b.bcount = c }
+// InvalidateCount updates the cached count in the bitmap.
+func (b *Bitmap) InvalidateCount() {
+	itr, n := b.data.Iterator(), uint64(0)
+	for _, eof := itr.Next(); !eof; _, eof = itr.Next() {
+		n++
+	}
+	b.n = n
+}
 
 // Count returns the number of set bits in the bitmap.
-func (b *Bitmap) Count() uint64 { return b.bcount }
-
-// BitCount calculates the number of set bits in the bitmap from raw chunk data.
-func (b *Bitmap) BitCount() uint64 {
-	var n uint64
-	for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
-		n += i.Item().Value.bitcount()
-	}
-	return n
-}
+func (b *Bitmap) Count() uint64 { return b.n }
 
 // encodeBitmap converts b into its internal representation.
 func encodeBitmap(b *Bitmap) *internal.Bitmap {
@@ -427,13 +205,10 @@ func encodeBitmap(b *Bitmap) *internal.Bitmap {
 		return nil
 	}
 
-	pb := &internal.Bitmap{
+	return &internal.Bitmap{
+		Bits:  b.Bits(),
 		Attrs: encodeAttrs(b.Attrs),
 	}
-	for i := b.tree.Min(); !i.Limit(); i = i.Next() {
-		pb.Chunks = append(pb.Chunks, encodeChunk(i.Item().(*Chunk)))
-	}
-	return pb
 }
 
 // decodeBitmap converts b from its internal representation.
@@ -444,10 +219,9 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap {
 
 	b := NewBitmap()
 	b.Attrs = decodeAttrs(pb.GetAttrs())
-	for _, chunk := range pb.GetChunks() {
-		b.AddChunk(decodeChunk(chunk))
+	for _, v := range pb.GetBits() {
+		b.SetBit(v)
 	}
-	b.SetCount(b.BitCount())
 	return b
 }
 
@@ -459,149 +233,3 @@ func Union(bitmaps []*Bitmap) *Bitmap {
 	}
 	return other
 }
-
-// Chunk represents a set of blocks in a Bitmap.
-type Chunk struct {
-	Key   uint64
-	Value Blocks
-}
-
-// Clone returns a copy of c.
-func (c *Chunk) Clone() *Chunk {
-	return &Chunk{
-		Key:   c.Key,
-		Value: c.Value.copy(),
-	}
-}
-
-// encodeChunks encodes c into its internal representation.
-func encodeChunk(c *Chunk) *internal.Chunk {
-	return &internal.Chunk{
-		Key:   proto.Uint64(c.Key),
-		Value: []uint64(c.Value),
-	}
-}
-
-// decodeChunk decodes c from its internal representation.
-func decodeChunk(pb *internal.Chunk) *Chunk {
-	return &Chunk{
-		Key:   pb.GetKey(),
-		Value: Blocks(pb.GetValue()),
-	}
-}
-
-// ChunkIterator represents an object for iterating over chunks in a bitmap.
-type ChunkIterator struct {
-	itr rbtree.Iterator
-}
-
-// Limit return true when the iterator is at the end of iteration.
-func (r *ChunkIterator) Limit() bool {
-	return r.itr.Limit()
-}
-
-// Next moves the iterator to the next chunk.
-func (r *ChunkIterator) Next() *ChunkIterator {
-	r.itr = r.itr.Next()
-	return r
-}
-
-// Item returns the current item that the iterator is pointing at.
-func (r *ChunkIterator) Item() *Chunk {
-	if r.itr.Item() != nil {
-		return r.itr.Item().(*Chunk)
-	}
-	return nil
-}
-
-func rbtreeItemCompare(a, b rbtree.Item) int {
-	aKey, bKey := a.(*Chunk).Key, b.(*Chunk).Key
-	if aKey < bKey {
-		return -1
-	} else if aKey > bKey {
-		return 1
-	}
-	return 0
-}
-
-type Blocks []uint64
-
-// NewBlocks returns a 32-length Block.
-func NewBlocks() Blocks {
-	return make(Blocks, 32)
-}
-
-func (a Blocks) bitcount() uint64 {
-	return popcntSlice(a)
-}
-
-func (a Blocks) union(other Blocks) Blocks {
-	ret := NewBlocks()
-	for i, _ := range a {
-		ret[i] = a[i] | other[i]
-	}
-	return ret
-}
-
-func (a Blocks) invert() Blocks {
-	other := NewBlocks()
-	for i, _ := range a {
-		other[i] = ^a[i]
-	}
-	return other
-}
-
-func (a Blocks) copy() Blocks {
-	other := NewBlocks()
-	for i, _ := range a {
-		other[i] = a[i]
-	}
-	return other
-}
-
-func (a Blocks) andcount(other Blocks) uint64 {
-	return popcntAndSliceAsm(a, other)
-}
-
-func (a Blocks) intersect(other Blocks) Blocks {
-	ret := NewBlocks()
-	for i, _ := range a {
-		ret[i] = a[i] & other[i]
-	}
-	return ret
-}
-
-func (a Blocks) difference(other Blocks) Blocks {
-	ret := NewBlocks()
-	for i, _ := range a {
-		ret[i] = a[i] &^ other[i]
-	}
-	return ret
-}
-
-func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
-	val := a[i] & (1 << bit)
-	a[i] |= 1 << bit
-	return val == 0
-}
-
-func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
-	val := a[i] & (1 << bit)
-	a[i] &= ^(1 << bit)
-	return val != 0
-}
-
-// Address represents a location for a given chunk/block/bit.
-type Address struct {
-	ChunkKey   uint64
-	BlockIndex uint8
-	Bit        uint8
-}
-
-func deref(pos uint64) Address {
-	chunkKey := pos >> 11              // div by 2048
-	offset := pos & 0x7FF              // mod by 2048
-	blockIndex := uint8(offset >> 6)   // div by 64
-	bit_offset := uint8(offset & 0x3F) // mod by 64
-	return Address{chunkKey, blockIndex, bit_offset}
-}

cache.go

@@ -10,10 +10,10 @@ import (
 	"github.com/umbel/pilosa/internal"
 )
 
-// Cache represents a cache for bitmaps.
+// Cache represents a cache for bitmap counts.
 type Cache interface {
-	Add(bitmapID uint64, bm *Bitmap)
-	Get(bitmapID uint64) *Bitmap
+	Add(bitmapID uint64, n uint64)
+	Get(bitmapID uint64) uint64
 	Len() int
 
 	// Returns a list of all bitmap IDs.
@@ -28,33 +28,30 @@ type Cache interface {
 
 // LRUCache represents a least recently used Cache implemenation.
 type LRUCache struct {
-	cache   *lru.Cache
-	bitmaps map[uint64]*Bitmap
+	cache  *lru.Cache
+	counts map[uint64]uint64
 }
 
 // NewLRUCache returns a new instance of LRUCache.
 func NewLRUCache(maxEntries int) *LRUCache {
 	c := &LRUCache{
-		cache:   lru.New(maxEntries),
-		bitmaps: make(map[uint64]*Bitmap),
+		cache:  lru.New(maxEntries),
+		counts: make(map[uint64]uint64),
 	}
 	c.cache.OnEvicted = c.onEvicted
 	return c
 }
 
 // Add adds a bitmap to the cache.
-func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) {
-	c.cache.Add(bitmapID, bm)
-	c.bitmaps[bitmapID] = bm
+func (c *LRUCache) Add(bitmapID, n uint64) {
+	c.cache.Add(bitmapID, n)
+	c.counts[bitmapID] = n
 }
 
 // Get returns a bitmap with a given id.
-func (c *LRUCache) Get(bitmapID uint64) *Bitmap {
-	bm, ok := c.cache.Get(bitmapID)
-	if !ok {
-		return nil
-	}
-	return bm.(*Bitmap)
+func (c *LRUCache) Get(bitmapID uint64) uint64 {
+	n, _ := c.cache.Get(bitmapID)
+	return n.(uint64)
 }
 
 // Len returns the number of items in the cache.
@@ -65,35 +62,35 @@ func (c *LRUCache) Invalidate() {}
 
 // BitmapIDs returns a list of all bitmap IDs in the cache.
 func (c *LRUCache) BitmapIDs() []uint64 {
-	a := make([]uint64, 0, len(c.bitmaps))
-	for id := range c.bitmaps {
+	a := make([]uint64, 0, len(c.counts))
+	for id := range c.counts {
 		a = append(a, id)
 	}
 	sort.Sort(uint64Slice(a))
 	return a
 }
 
-// Top returns all bitmaps in the cache.
+// Top returns all counts in the cache.
 func (c *LRUCache) Top() []BitmapPair {
-	a := make([]BitmapPair, 0, len(c.bitmaps))
-	for id, bm := range c.bitmaps {
+	a := make([]BitmapPair, 0, len(c.counts))
+	for id, n := range c.counts {
 		a = append(a, BitmapPair{
-			ID:     id,
-			Bitmap: bm,
+			ID:    id,
+			Count: uint64(n),
 		})
 	}
 	sort.Sort(BitmapPairs(a))
 	return a
 }
 
-func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) }
+func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
 
 // Ensure LRUCache implements Cache.
 var _ Cache = &LRUCache{}
 
 // RankCache represents a cache with sorted entries.
 type RankCache struct {
-	entries  map[uint64]*Bitmap
+	entries  map[uint64]uint64
 	rankings []BitmapPair // cached, ordered list
 
 	updateN    int
@@ -107,25 +104,25 @@ type RankCache struct {
 // NewRankCache returns a new instance of RankCache.
 func NewRankCache() *RankCache {
 	return &RankCache{
-		entries: make(map[uint64]*Bitmap),
+		entries: make(map[uint64]uint64),
 	}
 }
 
 // Add adds a bitmap to the cache.
-func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
+func (c *RankCache) Add(bitmapID uint64, n uint64) {
 	// Ignore if the bit count on the bitmap is below the threshold.
-	if bm.Count() < c.ThresholdValue {
+	if n < c.ThresholdValue {
 		return
 	}
 
 	// Add to cache.
-	c.entries[bitmapID] = bm
+	c.entries[bitmapID] = n
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.ThresholdLength {
 		c.update()
-		for id, bm := range c.entries {
-			if bm.Count() <= c.ThresholdValue {
+		for id, n := range c.entries {
+			if n <= c.ThresholdValue {
 				delete(c.entries, id)
 			}
 		}
@@ -133,7 +130,7 @@ func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
 }
 
 // Get returns a bitmap with a given id.
-func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] }
+func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] }
 
 // Len returns the number of items in the cache.
 func (c *RankCache) Len() int { return len(c.entries) }
@@ -160,10 +157,10 @@ func (c *RankCache) Invalidate() {
 func (c *RankCache) update() {
 	// Convert cache to a sorted list.
 	rankings := make([]BitmapPair, 0, len(c.entries))
-	for id, bm := range c.entries {
+	for id, n := range c.entries {
 		rankings = append(rankings, BitmapPair{
-			ID:     id,
-			Bitmap: bm,
+			ID:    id,
+			Count: n,
 		})
 	}
 	sort.Sort(BitmapPairs(rankings))
@@ -171,7 +168,7 @@ func (c *RankCache) update() {
 	// Store the count of the item at the threshold index.
 	c.rankings = rankings
 	if len(c.rankings) > c.ThresholdIndex {
-		c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count()
+		c.ThresholdValue = rankings[c.ThresholdIndex].Count
 	} else {
 		c.ThresholdValue = 1
 	}
@@ -198,8 +195,8 @@ var _ Cache = &RankCache{}
 
 // BitmapPair represents a bitmap with an associated identifier.
 type BitmapPair struct {
-	ID     uint64
-	Bitmap *Bitmap
+	ID    uint64
+	Count uint64
 }
 
 // BitmapPairs is a sortable list of BitmapPair objects.
@@ -207,7 +204,7 @@ type BitmapPairs []BitmapPair
 
 func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p BitmapPairs) Len() int           { return len(p) }
-func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() }
+func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
 type Pair struct {
 	Key   uint64 `json:"key"`

cmd/pilosa-bench/main.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/umbel/pilosa"
+)
+
+// BasePort is the initial port used when generating a cluster.
+const BasePort = 16000
+
+// Default settings for command line arguments.
+const (
+	DefaultServerN  = 1
+	DefaultReplicaN = 1
+)
+
+func main() {
+	m := NewMain()
+
+	// Parse command line arguments.
+	if err := m.ParseFlags(os.Args[1:]); err == flag.ErrHelp {
+		fmt.Fprintln(m.Stderr, m.Usage())
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(m.Stderr, err)
+		os.Exit(1)
+	}
+
+	// Execute the program.
+	if err := m.Run(); err != nil {
+		fmt.Fprintln(m.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Main represents the main program execution.
+type Main struct {
+	ServerN  int
+	ReplicaN int
+
+	Verbose bool
+	Work    bool
+
+	// Standard input/output
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewMain returns a new instance of Main.
+func NewMain() *Main {
+	return &Main{
+		ServerN:  DefaultServerN,
+		ReplicaN: DefaultReplicaN,
+
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Usage returns usage documentation.
+func (m *Main) Usage() string {
+	return `
+pilosa-bench is a tool for testing and benchmarking pilosa clusters.
+
+Usage:
+
+	pilosa-bench [arguments]
+
+The following arguments are available:
+
+	-servers N
+	    The number of servers to generate for the cluster.
+	    Defaults to 1 server.
+
+	-replicas N
+	    Replication factor for data in the cluster.
+	    Defaults to 1 replica.
+
+	-v
+	    Logs all server output to stderr in addition to server log.
+
+	-work
+	    Prints the temporary work directory and does not delete it
+	    after the benchmark has completed.
+
+`[1:]
+}
+
+// ParseFlags parses command line flags from args.
+func (m *Main) ParseFlags(args []string) error {
+	fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.IntVar(&m.ServerN, "servers", DefaultServerN, "server count")
+	fs.IntVar(&m.ReplicaN, "replicas", DefaultReplicaN, "replication factor")
+	fs.BoolVar(&m.Verbose, "v", false, "verbose")
+	fs.BoolVar(&m.Work, "work", false, "verbose")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Run executes the main program execution.
+func (m *Main) Run(args ...string) error {
+	// Validate arguments.
+	if m.ServerN <= 0 {
+		return errors.New("server count must be at least 1")
+	} else if m.ReplicaN <= 0 {
+		return errors.New("replication factor must be at least 1")
+	} else if m.ReplicaN > m.ServerN {
+		return errors.New("replication factor must be less than server count")
+	}
+
+	logger := m.logger()
+	var logFiles []*os.File
+
+	// Create a base temp path.
+	path, err := ioutil.TempDir("", "pilosa-bench-")
+	if err != nil {
+		return err
+	}
+	if m.Work {
+		fmt.Printf("WORK=%s\n\n", path)
+	}
+
+	// Build cluster configuration.
+	cluster := pilosa.Cluster{
+		ReplicaN: m.ReplicaN,
+	}
+	for i := 0; i < m.ServerN; i++ {
+		cluster.Nodes = append(cluster.Nodes, &pilosa.Node{
+			Host: fmt.Sprintf("localhost:%d", BasePort+i),
+		})
+	}
+
+	// Build servers.
+	servers := make([]*pilosa.Server, m.ServerN)
+	for i := range servers {
+		// Make server work directory.
+		if err := os.MkdirAll(filepath.Join(path, strconv.Itoa(i)), 0777); err != nil {
+			return err
+		}
+
+		// Build server.
+		s := pilosa.NewServer()
+		s.Host = fmt.Sprintf("localhost:%d", BasePort+i)
+		s.Cluster = &cluster
+		s.Index.Path = filepath.Join(path, strconv.Itoa(i), "data")
+
+		// Create log file.
+		f, err := os.Create(filepath.Join(path, strconv.Itoa(i), "log"))
+		if err != nil {
+			return err
+		}
+		logFiles = append(logFiles, f)
+
+		// Set log and optionally write out to stderr as well.
+		s.LogOutput = f
+		if m.Verbose {
+			s.LogOutput = io.MultiWriter(m.Stderr, s.LogOutput)
+		}
+
+		servers[i] = s
+	}
+
+	// Open all servers.
+	for i, s := range servers {
+		logger.Printf("starting server #%d: %s", i, s.Host)
+		if err := s.Open(); err != nil {
+			return err
+		}
+	}
+
+	// FIXME(benbjohnson): Execute benchmark testing.
+
+	// Close all servers.
+	for i, s := range servers {
+		logger.Printf("closing server #%d", i)
+		if err := s.Close(); err != nil {
+			logger.Printf("error closing server: %s", err)
+		}
+	}
+
+	// Close all logs.
+	for _, f := range logFiles {
+		f.Close()
+	}
+
+	// If work flag is not set then delete all data & logs.
+	if !m.Work {
+		if err := os.RemoveAll(path); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Close gracefully closes the program.
+func (m *Main) Close() error { return nil }
+
+func (m *Main) logger() *log.Logger { return log.New(m.Stderr, "", log.LstdFlags) }

cmd/pilosa-bench/main_test.go

@@ -0,0 +1,74 @@
+package main_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	main "github.com/umbel/pilosa/cmd/pilosa-bench"
+)
+
+// Ensure server count flag can be parsed.
+func TestMain_ParseFlags_ServerN(t *testing.T) {
+	m := NewMain()
+	if err := m.ParseFlags([]string{"-servers", "2"}); err != nil {
+		t.Fatal(err)
+	} else if m.ServerN != 2 {
+		t.Fatalf("unexpected server count: %d", m.ServerN)
+	}
+}
+
+// Ensure replication factor flag can be parsed.
+func TestMain_ParseFlags_ReplicaN(t *testing.T) {
+	m := NewMain()
+	if err := m.ParseFlags([]string{"-servers=4", "-replicas", "3"}); err != nil {
+		t.Fatal(err)
+	} else if m.ReplicaN != 3 {
+		t.Fatalf("unexpected replica count: %d", m.ReplicaN)
+	}
+}
+
+// Ensure verbose flag can be parsed.
+func TestMain_ParseFlags_Verbose(t *testing.T) {
+	m := NewMain()
+	if err := m.ParseFlags([]string{"-v"}); err != nil {
+		t.Fatal(err)
+	} else if !m.Verbose {
+		t.Fatalf("expected verbose flag")
+	}
+}
+
+// Ensure work flag can be parsed.
+func TestMain_ParseFlags_Work(t *testing.T) {
+	m := NewMain()
+	if err := m.ParseFlags([]string{"-work"}); err != nil {
+		t.Fatal(err)
+	} else if !m.Work {
+		t.Fatalf("expected work flag")
+	}
+}
+
+// Main represents a test wrapper for main.Main.
+type Main struct {
+	*main.Main
+
+	Stdin  bytes.Buffer
+	Stdout bytes.Buffer
+	Stderr bytes.Buffer
+}
+
+// NewMain returns a new instance of Main.
+func NewMain() *Main {
+	m := &Main{Main: main.NewMain()}
+	m.Main.Stdin = &m.Stdin
+	m.Main.Stdout = &m.Stdout
+	m.Main.Stderr = &m.Stderr
+
+	if testing.Verbose() {
+		m.Main.Stdout = io.MultiWriter(os.Stdout, m.Main.Stdout)
+		m.Main.Stderr = io.MultiWriter(os.Stderr, m.Main.Stderr)
+	}
+
+	return m
+}

executor.go

@@ -262,7 +262,7 @@ func (e *Executor) executeDifferenceSlice(db string, c *pql.Difference, slice ui
 			other = other.Difference(bm)
 		}
 	}
-	other.SetCount(other.BitCount())
+	other.InvalidateCount()
 	return other, nil
 }
 
@@ -294,7 +294,7 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint
 			other = other.Intersect(bm)
 		}
 	}
-	other.SetCount(other.BitCount())
+	other.InvalidateCount()
 	return other, nil
 }
 
@@ -327,7 +327,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi
 			other = other.Union(bm)
 		}
 	}
-	other.SetCount(other.BitCount())
+	other.InvalidateCount()
 	return other, nil
 }
 
@@ -486,7 +486,6 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op
 	}
 
 	// Create HTTP request.
-	println("dbg.host?", node.Host)
 	req, err := http.NewRequest("POST", (&url.URL{
 		Scheme: "http",
 		Host:   node.Host,

executor_test.go

@@ -24,12 +24,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
 		t.Fatal(err)
-	} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
-		t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
-	} else if chunks[0].Value[0] != 8 {
-		t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
-	} else if chunks[1].Value[0] != 2 {
-		t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
 		t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
 	}
@@ -43,14 +39,13 @@ func TestExecutor_Execute_Difference(t *testing.T) {
 	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2)
 	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3)
 	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4)
 
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
 		t.Fatal(err)
-	} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
-		t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
-	} else if chunks[0].Value[0] != 10 { // b1010
-		t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	}
 }
 
@@ -69,12 +64,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
 		t.Fatal(err)
-	} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
-		t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
-	} else if chunks[0].Value[0] != 2 {
-		t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
-	} else if chunks[1].Value[0] != 4 {
-		t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	}
 }
 
@@ -92,12 +83,8 @@ func TestExecutor_Execute_Union(t *testing.T) {
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
 		t.Fatal(err)
-	} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
-		t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
-	} else if chunks[0].Value[0] != 5 {
-		t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
-	} else if chunks[1].Value[0] != 6 {
-		t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	}
 }
 
@@ -315,12 +302,8 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
 	e := NewExecutor(idx.Index, c)
 	if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
 		t.Fatal(err)
-	} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
-		t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
-	} else if chunks[0].Value[0] != 6 {
-		t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
-	} else if chunks[1].Value[0] != 2 {
-		t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	}
 }
 

fragment.go

@@ -323,11 +323,6 @@ func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
 }
 
 func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
-	// Read from cache.
-	if bm := f.cache.Get(bitmapID); bm != nil {
-		return bm
-	}
-
 	// Read bitmap from storage.
 	bm := NewBitmap()
 	f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) {
@@ -335,8 +330,8 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
 		bm.SetBit(profileID)
 	})
 
-	// Add to the cache.
-	f.cache.Add(bitmapID, bm)
+	// Update cache.
+	f.cache.Add(bitmapID, bm.Count())
 
 	return bm
 }
@@ -462,10 +457,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 	// Iterate over rankings and add to results until we have enough.
 	results := make([]Pair, 0, opt.N)
 	for _, pair := range pairs {
-		bitmapID, bm := pair.ID, pair.Bitmap
+		bitmapID, n := pair.ID, pair.Count
 
 		// Ignore empty bitmaps.
-		if bm.Count() <= 0 {
+		if n <= 0 {
 			continue
 		}
 
@@ -486,9 +481,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 		// The initial n pairs should simply be added to the results.
 		if opt.N == 0 || len(results) < opt.N {
 			// Calculate count and append.
-			count := bm.Count()
+			count := n
 			if opt.Src != nil {
-				count = opt.Src.IntersectionCount(bm)
+				count = opt.Src.IntersectionCount(f.Bitmap(bitmapID))
 			}
 			if count == 0 {
 				continue
@@ -516,13 +511,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 
 		// If the bitmap doesn't have enough bits set before the intersection
 		// then we can assume that any remaing bitmaps also have a count too low.
-		if bm.Count() < threshold {
+		if n < threshold {
 			break
 		}
 
 		// Calculate the intersecting bit count and skip if it's below our
 		// last bitmap in our current result set.
-		count := opt.Src.IntersectionCount(bm)
+		count := opt.Src.IntersectionCount(f.Bitmap(bitmapID))
 		if count < threshold {
 			continue
 		}
@@ -553,8 +548,8 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
 	pairs := make([]BitmapPair, len(bitmapIDs))
 	for i, bitmapID := range bitmapIDs {
 		pairs[i] = BitmapPair{
-			ID:     bitmapID,
-			Bitmap: f.Bitmap(bitmapID),
+			ID:    bitmapID,
+			Count: f.Bitmap(bitmapID).Count(),
 		}
 	}
 	return pairs

handler_test.go

@@ -229,8 +229,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
 	var resp internal.QueryResponse
 	if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 		t.Fatal(err)
-	} else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
-		t.Fatalf("unexpected bitmap chunk length: %d", len(a))
+	} else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
 		t.Fatalf("unexpected attr length: %d", len(attrs))
 	} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
@@ -286,8 +286,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
 	if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 		t.Fatal(err)
 	}
-	if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
-		t.Fatalf("unexpected bitmap chunk length: %d", len(a))
+	if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
+		t.Fatalf("unexpected bits: %+v", bits)
 	} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
 		t.Fatalf("unexpected attr length: %d", len(attrs))
 	} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {

internal/internal.pb.go

@@ -10,7 +10,6 @@ It is generated from these files:
 
 It has these top-level messages:
 	Bitmap
-	Chunk
 	Pair
 	Bit
 	Profile
@@ -42,7 +41,7 @@ var _ = math.Inf
 const _ = proto.GoGoProtoPackageIsVersion1
 
 type Bitmap struct {
-	Chunks           []*Chunk `protobuf:"bytes,1,rep,name=Chunks" json:"Chunks,omitempty"`
+	Bits             []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"`
 	Attrs            []*Attr  `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
 	XXX_unrecognized []byte   `json:"-"`
 }
@@ -52,9 +51,9 @@ func (m *Bitmap) String() string            { return proto.CompactTextString(m)
 func (*Bitmap) ProtoMessage()               {}
 func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} }
 
-func (m *Bitmap) GetChunks() []*Chunk {
+func (m *Bitmap) GetBits() []uint64 {
 	if m != nil {
-		return m.Chunks
+		return m.Bits
 	}
 	return nil
 }
@@ -66,31 +65,6 @@ func (m *Bitmap) GetAttrs() []*Attr {
 	return nil
 }
 
-type Chunk struct {
-	Key              *uint64  `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"`
-	Value            []uint64 `protobuf:"varint,2,rep,name=Value" json:"Value,omitempty"`
-	XXX_unrecognized []byte   `json:"-"`
-}
-
-func (m *Chunk) Reset()                    { *m = Chunk{} }
-func (m *Chunk) String() string            { return proto.CompactTextString(m) }
-func (*Chunk) ProtoMessage()               {}
-func (*Chunk) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} }
-
-func (m *Chunk) GetKey() uint64 {
-	if m != nil && m.Key != nil {
-		return *m.Key
-	}
-	return 0
-}
-
-func (m *Chunk) GetValue() []uint64 {
-	if m != nil {
-		return m.Value
-	}
-	return nil
-}
-
 type Pair struct {
 	Key              *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"`
 	Count            *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"`
@@ -100,7 +74,7 @@ type Pair struct {
 func (m *Pair) Reset()                    { *m = Pair{} }
 func (m *Pair) String() string            { return proto.CompactTextString(m) }
 func (*Pair) ProtoMessage()               {}
-func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} }
+func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} }
 
 func (m *Pair) GetKey() uint64 {
 	if m != nil && m.Key != nil {
@@ -125,7 +99,7 @@ type Bit struct {
 func (m *Bit) Reset()                    { *m = Bit{} }
 func (m *Bit) String() string            { return proto.CompactTextString(m) }
 func (*Bit) ProtoMessage()               {}
-func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} }
+func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} }
 
 func (m *Bit) GetBitmapID() uint64 {
 	if m != nil && m.BitmapID != nil {
@@ -150,7 +124,7 @@ type Profile struct {
 func (m *Profile) Reset()                    { *m = Profile{} }
 func (m *Profile) String() string            { return proto.CompactTextString(m) }
 func (*Profile) ProtoMessage()               {}
-func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} }
+func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} }
 
 func (m *Profile) GetID() uint64 {
 	if m != nil && m.ID != nil {
@@ -177,7 +151,7 @@ type Attr struct {
 func (m *Attr) Reset()                    { *m = Attr{} }
 func (m *Attr) String() string            { return proto.CompactTextString(m) }
 func (*Attr) ProtoMessage()               {}
-func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} }
+func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} }
 
 func (m *Attr) GetKey() string {
 	if m != nil && m.Key != nil {
@@ -215,7 +189,7 @@ type AttrMap struct {
 func (m *AttrMap) Reset()                    { *m = AttrMap{} }
 func (m *AttrMap) String() string            { return proto.CompactTextString(m) }
 func (*AttrMap) ProtoMessage()               {}
-func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} }
+func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} }
 
 func (m *AttrMap) GetAttrs() []*Attr {
 	if m != nil {
@@ -238,7 +212,7 @@ type QueryRequest struct {
 func (m *QueryRequest) Reset()                    { *m = QueryRequest{} }
 func (m *QueryRequest) String() string            { return proto.CompactTextString(m) }
 func (*QueryRequest) ProtoMessage()               {}
-func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} }
+func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} }
 
 func (m *QueryRequest) GetDB() string {
 	if m != nil && m.DB != nil {
@@ -299,7 +273,7 @@ type QueryResponse struct {
 func (m *QueryResponse) Reset()                    { *m = QueryResponse{} }
 func (m *QueryResponse) String() string            { return proto.CompactTextString(m) }
 func (*QueryResponse) ProtoMessage()               {}
-func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} }
+func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} }
 
 func (m *QueryResponse) GetErr() string {
 	if m != nil && m.Err != nil {
@@ -333,7 +307,7 @@ type QueryResult struct {
 func (m *QueryResult) Reset()                    { *m = QueryResult{} }
 func (m *QueryResult) String() string            { return proto.CompactTextString(m) }
 func (*QueryResult) ProtoMessage()               {}
-func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} }
+func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} }
 
 func (m *QueryResult) GetBitmap() *Bitmap {
 	if m != nil {
@@ -375,7 +349,7 @@ type ImportRequest struct {
 func (m *ImportRequest) Reset()                    { *m = ImportRequest{} }
 func (m *ImportRequest) String() string            { return proto.CompactTextString(m) }
 func (*ImportRequest) ProtoMessage()               {}
-func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} }
+func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} }
 
 func (m *ImportRequest) GetDB() string {
 	if m != nil && m.DB != nil {
@@ -420,7 +394,7 @@ type ImportResponse struct {
 func (m *ImportResponse) Reset()                    { *m = ImportResponse{} }
 func (m *ImportResponse) String() string            { return proto.CompactTextString(m) }
 func (*ImportResponse) ProtoMessage()               {}
-func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} }
+func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} }
 
 func (m *ImportResponse) GetErr() string {
 	if m != nil && m.Err != nil {
@@ -440,7 +414,7 @@ type BlockDataRequest struct {
 func (m *BlockDataRequest) Reset()                    { *m = BlockDataRequest{} }
 func (m *BlockDataRequest) String() string            { return proto.CompactTextString(m) }
 func (*BlockDataRequest) ProtoMessage()               {}
-func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} }
+func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} }
 
 func (m *BlockDataRequest) GetDB() string {
 	if m != nil && m.DB != nil {
@@ -479,7 +453,7 @@ type BlockDataResponse struct {
 func (m *BlockDataResponse) Reset()                    { *m = BlockDataResponse{} }
 func (m *BlockDataResponse) String() string            { return proto.CompactTextString(m) }
 func (*BlockDataResponse) ProtoMessage()               {}
-func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} }
+func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} }
 
 func (m *BlockDataResponse) GetBitmapIDs() []uint64 {
 	if m != nil {
@@ -503,7 +477,7 @@ type Cache struct {
 func (m *Cache) Reset()                    { *m = Cache{} }
 func (m *Cache) String() string            { return proto.CompactTextString(m) }
 func (*Cache) ProtoMessage()               {}
-func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} }
+func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} }
 
 func (m *Cache) GetBitmapIDs() []uint64 {
 	if m != nil {
@@ -520,7 +494,7 @@ type SliceMaxResponse struct {
 func (m *SliceMaxResponse) Reset()                    { *m = SliceMaxResponse{} }
 func (m *SliceMaxResponse) String() string            { return proto.CompactTextString(m) }
 func (*SliceMaxResponse) ProtoMessage()               {}
-func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} }
+func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} }
 
 func (m *SliceMaxResponse) GetSliceMax() uint64 {
 	if m != nil && m.SliceMax != nil {
@@ -531,7 +505,6 @@ func (m *SliceMaxResponse) GetSliceMax() uint64 {
 
 func init() {
 	proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
-	proto.RegisterType((*Chunk)(nil), "internal.Chunk")
 	proto.RegisterType((*Pair)(nil), "internal.Pair")
 	proto.RegisterType((*Bit)(nil), "internal.Bit")
 	proto.RegisterType((*Profile)(nil), "internal.Profile")
@@ -549,38 +522,37 @@ func init() {
 }
 
 var fileDescriptorInternal = []byte{
-	// 514 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5b, 0x8b, 0xd3, 0x40,
-	0x18, 0x25, 0x4d, 0xda, 0xb4, 0x5f, 0x6c, 0xb7, 0x1d, 0x11, 0x83, 0xb0, 0xb8, 0xcc, 0x8a, 0x14,
-	0x1f, 0x56, 0x58, 0x7c, 0xf2, 0xcd, 0xb6, 0x8a, 0xcb, 0xb2, 0xcb, 0x5e, 0xd4, 0x67, 0x87, 0x3a,
-	0x6e, 0xe3, 0x26, 0x99, 0x38, 0x99, 0x80, 0x7d, 0xf2, 0xaf, 0xfb, 0xcd, 0x2d, 0x8d, 0x58, 0x11,
-	0x9f, 0xda, 0x39, 0xdf, 0xe5, 0x9c, 0x39, 0x39, 0x03, 0x8f, 0xb3, 0x52, 0x71, 0x59, 0xb2, 0xfc,
-	0xa5, 0xff, 0x73, 0x52, 0x49, 0xa1, 0x04, 0x19, 0xfa, 0x33, 0x7d, 0x0f, 0x83, 0x45, 0xa6, 0x0a,
-	0x56, 0x91, 0xa7, 0x30, 0x58, 0x6e, 0x9a, 0xf2, 0xbe, 0x4e, 0x83, 0xa3, 0x70, 0x9e, 0x9c, 0x1e,
-	0x9c, 0xb4, 0x43, 0x06, 0x27, 0x87, 0xd0, 0x7f, 0xa3, 0x94, 0xac, 0xd3, 0x9e, 0xa9, 0x4f, 0x76,
-	0x75, 0x0d, 0xd3, 0x63, 0xe8, 0xdb, 0xbe, 0x04, 0xc2, 0x73, 0xbe, 0xc5, 0x2d, 0xbd, 0x79, 0x44,
-	0xc6, 0xd0, 0xff, 0xc4, 0xf2, 0x86, 0x9b, 0xa1, 0x88, 0x52, 0x88, 0xae, 0x58, 0x26, 0xff, 0xe8,
-	0x59, 0x8a, 0xa6, 0x54, 0xd8, 0x83, 0x47, 0xfa, 0x02, 0x42, 0x94, 0x44, 0xa6, 0x30, 0xb4, 0xca,
-	0xce, 0x56, 0xae, 0x6f, 0x06, 0xa3, 0x2b, 0x29, 0xbe, 0x66, 0x39, 0x47, 0xc8, 0xf6, 0xbe, 0x82,
-	0xd8, 0x41, 0x04, 0xa0, 0xd7, 0x76, 0xfe, 0x43, 0xea, 0x25, 0x44, 0xfa, 0xb7, 0xab, 0x62, 0x44,
-	0x1e, 0x42, 0x72, 0xab, 0x64, 0x56, 0xde, 0x79, 0xbd, 0x01, 0x82, 0x48, 0xf9, 0x11, 0x67, 0x2d,
-	0x14, 0x22, 0x64, 0x54, 0x2c, 0x84, 0xc8, 0x2d, 0x14, 0x21, 0x34, 0xa4, 0x73, 0x88, 0xf5, 0xbe,
-	0x0b, 0x74, 0xb1, 0x65, 0x0e, 0xf6, 0x32, 0xff, 0x84, 0x07, 0xd7, 0x0d, 0x97, 0xdb, 0x1b, 0xfe,
-	0xbd, 0xe1, 0xb5, 0xd2, 0xa2, 0x57, 0x0b, 0x27, 0x00, 0x6d, 0x30, 0x35, 0x73, 0xb5, 0x11, 0x99,
-	0xc0, 0xe0, 0x36, 0xcf, 0xd6, 0xbc, 0x46, 0x5e, 0xb4, 0x4e, 0xfb, 0xe1, 0xae, 0x5a, 0x5b, 0x5a,
-	0xad, 0xe4, 0x43, 0x56, 0xe0, 0x1a, 0x56, 0x54, 0x69, 0x1f, 0xa1, 0x90, 0x1c, 0x40, 0x7c, 0xdd,
-	0xb0, 0x52, 0x35, 0x45, 0x3a, 0x40, 0x60, 0xac, 0xb7, 0xdc, 0xf0, 0x42, 0x28, 0x9e, 0xc6, 0x46,
-	0x6a, 0x06, 0x63, 0x27, 0xa0, 0xae, 0x44, 0x59, 0x73, 0xed, 0xc1, 0x5b, 0x29, 0x51, 0x82, 0xbe,
-	0xee, 0x73, 0x88, 0xb1, 0xd0, 0xe4, 0xca, 0x3b, 0xf7, 0x68, 0xa7, 0xdf, 0x8f, 0x61, 0x95, 0x1c,
-	0x77, 0xb4, 0x84, 0xa6, 0x71, 0xb6, 0x6b, 0x74, 0x15, 0xfa, 0x0d, 0x92, 0xee, 0xcc, 0x91, 0x4f,
-	0x9a, 0xe1, 0x4a, 0x4e, 0xa7, 0xbb, 0x09, 0x97, 0xc0, 0x11, 0x04, 0x97, 0xc6, 0x77, 0xf3, 0x01,
-	0x75, 0x4e, 0xfc, 0xf6, 0x8e, 0x8d, 0x26, 0x3e, 0x78, 0xcd, 0xe5, 0x86, 0x95, 0x77, 0xfc, 0x8b,
-	0xfb, 0x02, 0x9f, 0x61, 0x7c, 0x56, 0x54, 0x42, 0xaa, 0xbf, 0x18, 0xfb, 0x4e, 0xb2, 0x82, 0x3b,
-	0x63, 0xf1, 0x68, 0x8c, 0xc5, 0xdd, 0x2e, 0x55, 0x3e, 0x67, 0xda, 0x58, 0x6d, 0x35, 0x4e, 0xb7,
-	0x41, 0xab, 0xd1, 0x59, 0x9d, 0xdc, 0x43, 0x98, 0x78, 0x86, 0x3d, 0xce, 0xd1, 0x73, 0x98, 0x2e,
-	0x72, 0xb1, 0xbe, 0x5f, 0x31, 0xc5, 0xfe, 0x5f, 0x03, 0x1e, 0xcd, 0x34, 0xf2, 0xeb, 0x54, 0xbf,
-	0x86, 0x59, 0x67, 0x99, 0xa3, 0xfb, 0x4d, 0x67, 0xb0, 0x47, 0xa7, 0x7d, 0x61, 0x4f, 0xf0, 0x31,
-	0xb1, 0xf5, 0x66, 0x5f, 0x3f, 0x7d, 0x06, 0x53, 0xc3, 0x7a, 0xc1, 0x7e, 0xb4, 0x6b, 0x31, 0x56,
-	0x1e, 0xb3, 0x8f, 0xe7, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0xc6, 0x93, 0x16, 0x36, 0x04,
-	0x00, 0x00,
+	// 499 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40,
+	0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4,
+	0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb,
+	0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58,
+	0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95,
+	0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca,
+	0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46,
+	0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a,
+	0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2,
+	0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb,
+	0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31,
+	0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8,
+	0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6,
+	0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6,
+	0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b,
+	0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9,
+	0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65,
+	0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa,
+	0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6,
+	0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c,
+	0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f,
+	0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf,
+	0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07,
+	0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c,
+	0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08,
+	0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd,
+	0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1,
+	0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf,
+	0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f,
+	0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac,
+	0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe,
+	0x03, 0x00, 0x00,
 }

internal/internal.proto

@@ -1,13 +1,8 @@
 package internal;
 
 message Bitmap {
-	repeated Chunk Chunks = 1;
-	repeated Attr Attrs   = 2;
-}
-
-message Chunk {
-	required uint64 Key   = 1;
-	repeated uint64 Value = 2;
+	repeated uint64 Bits  = 1;
+	repeated Attr   Attrs = 2;
 }
 
 message Pair {

roaring/roaring.go

@@ -308,7 +308,11 @@ func (b *Bitmap) writeOp(op *op) error {
 }
 
 // Iterator returns a new iterator for the bitmap.
-func (b *Bitmap) Iterator() *Iterator { return &Iterator{bitmap: b} }
+func (b *Bitmap) Iterator() *Iterator {
+	itr := &Iterator{bitmap: b}
+	itr.Seek(0)
+	return itr
+}
 
 // Iterator represents an iterator over a Bitmap.
 type Iterator struct {
@@ -410,6 +414,56 @@ func (itr *Iterator) peek() uint64 {
 	return uint64(key)<<16 | uint64(itr.j)
 }
 
+// BufIterator wraps an iterator to provide the ability to unread values.
+type BufIterator struct {
+	buf struct {
+		v    uint64
+		eof  bool
+		full bool
+	}
+	itr *Iterator
+}
+
+// NewBufIterator returns a buffered iterator that wraps itr.
+func NewBufIterator(itr *Iterator) *BufIterator {
+	return &BufIterator{itr: itr}
+}
+
+// Seek moves to the first pair equal to or greater than pseek/bseek.
+func (itr *BufIterator) Seek(v uint64) {
+	itr.buf.full = false
+	itr.itr.Seek(v)
+}
+
+// Next returns the next pair in the bitmap.
+// If a value has been buffered then it is returned and the buffer is cleared.
+func (itr *BufIterator) Next() (v uint64, eof bool) {
+	if itr.buf.full {
+		itr.buf.full = false
+		return itr.buf.v, itr.buf.eof
+	}
+
+	// Read value onto buffer in case of unread.
+	itr.buf.v, itr.buf.eof = itr.itr.Next()
+	return itr.buf.v, itr.buf.eof
+}
+
+// Peek reads the next value but leaves it on the buffer.
+func (itr *BufIterator) Peek() (v uint64, eof bool) {
+	v, eof = itr.Next()
+	itr.Unread()
+	return
+}
+
+// Unread pushes previous pair on to the buffer.
+// Panics if the buffer is already full.
+func (itr *BufIterator) Unread() {
+	if itr.buf.full {
+		panic("roaring.BufIterator: buffer full")
+	}
+	itr.buf.full = true
+}
+
 // The maximum size of array containers.
 const arrayMaxSize = 4096