scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 00:25:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #82 from benbjohnson/bitmap-refactor

Browse source 
Refactor in-memory bitmap storage
This commit is contained in:
tgruben 2016-05-24 16:31:55 -05:00
commit 4103b5e7d2
13 changed files with 268 additions and 1512 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Godeps/Godeps.json generated
View file

@ -28,10 +28,6 @@
"ImportPath": "github.com/golang/groupcache/lru",
"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
},
{
"ImportPath": "github.com/yasushi-saito/rbtree",
"Rev": "571e2538414bf914c7e2909b61217b4e3e5508f4"
},
{
"ImportPath": "golang.org/x/sys/unix",
"Rev": "50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e"

561
bitmap.go
View file

@ -3,28 +3,16 @@ package pilosa
// #cgo CFLAGS:-mpopcnt
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"encoding/gob"
"encoding/json"
"io"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/yasushi-saito/rbtree"
"github.com/umbel/pilosa/roaring"
)
const CounterMask = uint64(0xffffffffffffffff)
var CounterKey = int64(-1)
// Bitmap represents a bitmap broken up into Chunks.
// Internally it is represented as a red-black tree of chunks.
// Bitmap represents a set of bits.
type Bitmap struct {
tree *rbtree.Tree
bcount uint64
data roaring.Bitmap
n uint64
// Attributes associated with the bitmap.
Attrs map[string]interface{}
@ -32,166 +20,98 @@ type Bitmap struct {
// NewBitmap returns a new instance of Bitmap.
func NewBitmap(bits ...uint64) *Bitmap {
bm := &Bitmap{tree: rbtree.NewTree(rbtreeItemCompare)}
bm := &Bitmap{}
for _, i := range bits {
bm.SetBit(i)
}
return bm
}
// Chunk returns the chunk within the bitmap.
// Returns nil if the chunk key does not exist.
func (b *Bitmap) Chunk(c *Chunk) *Chunk {
if n := b.tree.Get(c); n != nil {
return n.(*Chunk)
}
return nil
}
// AddChunk adds c to the bitmap.
func (b *Bitmap) AddChunk(c *Chunk) { b.tree.Insert(c) }
// Chunks returns a list of all chunks.
func (b *Bitmap) Chunks() []*Chunk {
var a []*Chunk
for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
a = append(a, itr.Item().Clone())
}
return a
}
// ChunkIterator returns an iterator for looping over the bitmap's chunks.
func (b *Bitmap) ChunkIterator() *ChunkIterator {
return &ChunkIterator{b.tree.Min()}
}
// Clone returns a copy of b.
func (b *Bitmap) Clone() *Bitmap {
itr := b.ChunkIterator()
other := NewBitmap()
for {
if itr.Limit() {
break
}
other.AddChunk(itr.Item().Clone())
itr = itr.Next()
}
return other
}
// Merge adds chunks from other to b.
// Chunks in b are overwritten if they exist in other.
func (b *Bitmap) Merge(other *Bitmap) {
for itr := other.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
b.AddChunk(itr.Item().Clone())
itr := other.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
b.SetBit(v)
}
}
// IntersectionCount returns the number of intersections between b and other.
func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
// OPTIMIZE: Implement roaring.Bitmap.IntersectionCount()
results := uint64(0)
itr0 := roaring.NewBufIterator(b.data.Iterator())
itr1 := roaring.NewBufIterator(other.data.Iterator())
var n uint64
for {
if itr1.Limit() || itr0.Limit() {
v0, eof0 := itr0.Next()
v1, eof1 := itr1.Next()
if eof0 || eof1 {
break
} else if itr0.Item().Key < itr1.Item().Key {
itr0 = itr0.Next()
} else if itr0.Item().Key > itr1.Item().Key {
itr1 = itr1.Next()
} else if itr0.Item().Key == itr1.Item().Key {
results += itr0.Item().Value.andcount(itr1.Item().Value)
itr0 = itr0.Next()
itr1 = itr1.Next()
} else if v0 < v1 {
itr1.Unread()
} else if v0 > v1 {
itr0.Unread()
} else {
n++
}
}
return results
return n
}
// Intersect returns the itersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
// OPTIMIZE: Implement roaring.Bitmap.Intersect()
itr0 := roaring.NewBufIterator(b.data.Iterator())
itr1 := roaring.NewBufIterator(other.data.Iterator())
output := NewBitmap()
for {
if itr1.Limit() || itr0.Limit() {
v0, eof0 := itr0.Next()
v1, eof1 := itr1.Next()
if eof0 || eof1 {
break
} else if itr0.Item().Key < itr1.Item().Key {
itr0 = itr0.Next()
} else if itr0.Item().Key > itr1.Item().Key {
itr1 = itr1.Next()
} else if itr0.Item().Key == itr1.Item().Key {
output.AddChunk(&Chunk{
Key: itr0.Item().Key,
Value: itr0.Item().Value.intersect(itr1.Item().Value),
})
itr0 = itr0.Next()
itr1 = itr1.Next()
} else if v0 < v1 {
itr1.Unread()
} else if v0 > v1 {
itr0.Unread()
} else {
output.SetBit(v0)
}
}
return output
}
// Invert returns a bitwise inversion of b.
func (b *Bitmap) Invert() *Bitmap {
other := NewBitmap()
for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
other.AddChunk(&Chunk{
Key: i.Item().Key,
Value: i.Item().Value.invert(),
})
}
return other
}
// Union returns the bitwise union of b and other.
func (b *Bitmap) Union(other *Bitmap) *Bitmap {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
// OPTIMIZE: Implement roaring.Bitmap.Union()
itr0 := roaring.NewBufIterator(b.data.Iterator())
itr1 := roaring.NewBufIterator(other.data.Iterator())
output := NewBitmap()
eof := uint64(0xdeadbeef)
for {
if itr0.Limit() && itr1.Limit() {
v0, eof0 := itr0.Next()
v1, eof1 := itr1.Next()
if eof0 && eof1 {
break
} else if itr0.Limit() {
if eof == itr1.Item().Key {
break
}
output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value})
eof = itr1.Item().Key
itr1 = itr1.Next()
} else if itr1.Limit() {
if eof == itr0.Item().Key {
break
}
output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
eof = itr0.Item().Key
itr0 = itr0.Next()
} else if itr0.Item().Key < itr1.Item().Key {
output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
eof = itr0.Item().Key
itr0 = itr0.Next()
} else if itr0.Item().Key > itr1.Item().Key {
output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value})
eof = itr1.Item().Key
itr1 = itr1.Next()
} else if itr0.Item().Key == itr1.Item().Key {
output.AddChunk(&Chunk{
Key: itr0.Item().Key,
Value: itr0.Item().Value.union(itr1.Item().Value),
})
eof = itr0.Item().Key
itr0 = itr0.Next()
itr1 = itr1.Next()
} else if eof0 {
output.SetBit(v1)
} else if eof1 {
output.SetBit(v0)
} else if v0 < v1 {
output.SetBit(v0)
itr1.Unread()
} else if v0 > v1 {
output.SetBit(v1)
itr0.Unread()
} else {
panic("unreachable")
output.SetBit(v0)
}
}
return output
@ -199,118 +119,30 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap {
// Difference returns the diff of b and other.
func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
// OPTIMIZE: Implement roaring.Bitmap.Difference()
itr0 := roaring.NewBufIterator(b.data.Iterator())
itr1 := roaring.NewBufIterator(other.data.Iterator())
output := NewBitmap()
for {
if itr0.Limit() && itr1.Limit() {
break
} else if itr0.Limit() {
break
} else if itr1.Limit() {
output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
itr0 = itr0.Next()
} else if itr0.Item().Key < itr1.Item().Key {
output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value})
itr0 = itr0.Next()
} else if itr0.Item().Key > itr1.Item().Key {
itr1 = itr1.Next()
} else if itr0.Item().Key == itr1.Item().Key {
chunk := &Chunk{
Key: itr0.Item().Key,
Value: itr0.Item().Value.difference(itr1.Item().Value),
}
v0, eof0 := itr0.Next()
v1, eof1 := itr1.Next()
// Do not add if all bits are zeroed.
if chunk.Value.bitcount() > 0 {
output.AddChunk(chunk)
}
itr0 = itr0.Next()
itr1 = itr1.Next()
} else {
panic("unreachable")
if eof0 {
break
} else if eof1 {
output.SetBit(v0)
} else if v0 < v1 {
output.SetBit(v0)
itr1.Unread()
} else if v0 > v1 {
itr0.Unread()
}
}
return output
}
// ToRawCompressString returns a compressed, hex-encoded string of b.
func (b *Bitmap) ToRawCompressString() (string, int) {
var bt bytes.Buffer
buf := gzip.NewWriter(&bt)
binary.Write(buf, binary.LittleEndian, uint64(b.tree.Len()))
max_slice := 0
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
obj := i.Item().(*Chunk)
max_slice = int(obj.Key)
binary.Write(buf, binary.LittleEndian, obj.Key)
for _, v := range obj.Value {
binary.Write(buf, binary.LittleEndian, v)
}
}
buf.Flush()
//buf.Close()
max_slice = max_slice / 32
return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice
}
// WriteTo writes the encoded bitmap to w.
func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
// Wrap output in gzip compression.
z := gzip.NewWriter(w)
// Encode chunk count.
enc := gob.NewEncoder(w)
if err := enc.Encode(b.tree.Len()); err != nil {
return 0, err
}
// Encode all chunks.
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
if err := enc.Encode(i.Item().(*Chunk)); err != nil {
return 0, err
}
}
// Flush and close.
if err := z.Close(); err != nil {
return 0, err
}
return 0, nil
}
// ReadFrom reads encoded bitmap data from r into b.
func (b *Bitmap) ReadFrom(r io.Reader) (n int64, err error) {
// Uncompress from gzip format.
z, err := gzip.NewReader(r)
if err != nil {
return 0, err
}
dec := gob.NewDecoder(z)
// Read size from data.
var size int
if err := dec.Decode(&size); err != nil {
return 0, err
}
// Read chunks into bitmap.
b.tree = rbtree.NewTree(rbtreeItemCompare)
for i := 0; i < size; i++ {
var chunk Chunk
if err := dec.Decode(&chunk); err != nil {
return 0, err
}
b.AddChunk(&chunk)
}
b.SetCount(b.BitCount())
return 0, nil
}
// MarshalJSON returns a JSON-encoded byte slice of b.
func (b *Bitmap) MarshalJSON() ([]byte, error) {
var o struct {
@ -327,94 +159,45 @@ func (b *Bitmap) MarshalJSON() ([]byte, error) {
return json.Marshal(&o)
}
// MarshalBinary returns a gob-encoded byte slice of b.
func (b *Bitmap) MarshalBinary() ([]byte, error) {
var buf bytes.Buffer
if _, err := b.WriteTo(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary decodes a gob-encoded byte slice into b.
func (b *Bitmap) UnmarshalBinary(data []byte) error {
_, err := b.ReadFrom(bytes.NewReader(data))
return err
}
// Bits returns the bits in b as a slice of ints.
func (b *Bitmap) Bits() []uint64 {
result := make([]uint64, 0, b.Count())
for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
item := i.Item()
chunk := item.Key
for bi, block := range item.Value {
for bit := uint(0); bit < 64; bit++ {
if (block & (1 << bit)) != 0 {
idx := chunk << 11
idx = idx | uint64((uint(bi)<<6)|bit)
result = append(result, idx)
}
}
}
a := make([]uint64, 0, b.Count())
itr := b.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
a = append(a, v)
}
return result
return a
}
// setBit sets the i-th bit of the bitmap.
// SetBit sets the i-th bit of the bitmap.
func (b *Bitmap) SetBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
if chunk == nil {
chunk = &Chunk{address.ChunkKey, make(Blocks, 32)}
b.AddChunk(chunk)
}
changed = chunk.Value.setBit(address.BlockIndex, address.Bit)
changed, _ = b.data.Add(i)
if changed {
b.bcount++
b.n++
}
return changed
}
// clearBit clears the i-th bit of the bitmap.
// ClearBit clears the i-th bit of the bitmap.
func (b *Bitmap) ClearBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
if chunk == nil {
return false
changed, _ = b.data.Remove(i)
if changed {
b.n--
}
changed = chunk.Value.clearBit(address.BlockIndex, address.Bit)
if changed && b.bcount > 0 {
b.bcount--
}
return changed
}
// Len returns the number of chunks in b.
func (b *Bitmap) Len() int { return b.tree.Len() }
// SetCount sets the number of set bits in the bitmap.
func (b *Bitmap) SetCount(c uint64) { b.bcount = c }
// InvalidateCount updates the cached count in the bitmap.
func (b *Bitmap) InvalidateCount() {
itr, n := b.data.Iterator(), uint64(0)
for _, eof := itr.Next(); !eof; _, eof = itr.Next() {
n++
}
b.n = n
}
// Count returns the number of set bits in the bitmap.
func (b *Bitmap) Count() uint64 { return b.bcount }
// BitCount calculates the number of set bits in the bitmap from raw chunk data.
func (b *Bitmap) BitCount() uint64 {
var n uint64
for i := b.ChunkIterator(); !i.Limit(); i = i.Next() {
n += i.Item().Value.bitcount()
}
return n
}
func (b *Bitmap) Count() uint64 { return b.n }
// encodeBitmap converts b into its internal representation.
func encodeBitmap(b *Bitmap) *internal.Bitmap {
@ -422,13 +205,10 @@ func encodeBitmap(b *Bitmap) *internal.Bitmap {
return nil
}
pb := &internal.Bitmap{
return &internal.Bitmap{
Bits: b.Bits(),
Attrs: encodeAttrs(b.Attrs),
}
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
pb.Chunks = append(pb.Chunks, encodeChunk(i.Item().(*Chunk)))
}
return pb
}
// decodeBitmap converts b from its internal representation.
@ -439,10 +219,9 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap {
b := NewBitmap()
b.Attrs = decodeAttrs(pb.GetAttrs())
for _, chunk := range pb.GetChunks() {
b.AddChunk(decodeChunk(chunk))
for _, v := range pb.GetBits() {
b.SetBit(v)
}
b.SetCount(b.BitCount())
return b
}
@ -454,149 +233,3 @@ func Union(bitmaps []*Bitmap) *Bitmap {
}
return other
}
// Chunk represents a set of blocks in a Bitmap.
type Chunk struct {
Key uint64
Value Blocks
}
// Clone returns a copy of c.
func (c *Chunk) Clone() *Chunk {
return &Chunk{
Key: c.Key,
Value: c.Value.copy(),
}
}
// encodeChunks encodes c into its internal representation.
func encodeChunk(c *Chunk) *internal.Chunk {
return &internal.Chunk{
Key: proto.Uint64(c.Key),
Value: []uint64(c.Value),
}
}
// decodeChunk decodes c from its internal representation.
func decodeChunk(pb *internal.Chunk) *Chunk {
return &Chunk{
Key: pb.GetKey(),
Value: Blocks(pb.GetValue()),
}
}
// ChunkIterator represents an object for iterating over chunks in a bitmap.
type ChunkIterator struct {
itr rbtree.Iterator
}
// Limit return true when the iterator is at the end of iteration.
func (r *ChunkIterator) Limit() bool {
return r.itr.Limit()
}
// Next moves the iterator to the next chunk.
func (r *ChunkIterator) Next() *ChunkIterator {
r.itr = r.itr.Next()
return r
}
// Item returns the current item that the iterator is pointing at.
func (r *ChunkIterator) Item() *Chunk {
if r.itr.Item() != nil {
return r.itr.Item().(*Chunk)
}
return nil
}
func rbtreeItemCompare(a, b rbtree.Item) int {
aKey, bKey := a.(*Chunk).Key, b.(*Chunk).Key
if aKey < bKey {
return -1
} else if aKey > bKey {
return 1
}
return 0
}
type Blocks []uint64
// NewBlocks returns a 32-length Block.
func NewBlocks() Blocks {
return make(Blocks, 32)
}
func (a Blocks) bitcount() uint64 {
return popcntSlice(a)
}
func (a Blocks) union(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] | other[i]
}
return ret
}
func (a Blocks) invert() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = ^a[i]
}
return other
}
func (a Blocks) copy() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = a[i]
}
return other
}
func (a Blocks) andcount(other Blocks) uint64 {
return popcntAndSliceAsm(a, other)
}
func (a Blocks) intersect(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] & other[i]
}
return ret
}
func (a Blocks) difference(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] &^ other[i]
}
return ret
}
func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] |= 1 << bit
return val == 0
}
func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] &= ^(1 << bit)
return val != 0
}
// Address represents a location for a given chunk/block/bit.
type Address struct {
ChunkKey uint64
BlockIndex uint8
Bit uint8
}
func deref(pos uint64) Address {
chunkKey := pos >> 11 // div by 2048
offset := pos & 0x7FF // mod by 2048
blockIndex := uint8(offset >> 6) // div by 64
bit_offset := uint8(offset & 0x3F) // mod by 64
return Address{chunkKey, blockIndex, bit_offset}
}

75
cache.go
View file

@ -10,10 +10,10 @@ import (
"github.com/umbel/pilosa/internal"
)
// Cache represents a cache for bitmaps.
// Cache represents a cache for bitmap counts.
type Cache interface {
Add(bitmapID uint64, bm *Bitmap)
Get(bitmapID uint64) *Bitmap
Add(bitmapID uint64, n uint64)
Get(bitmapID uint64) uint64
Len() int
// Returns a list of all bitmap IDs.
@ -28,33 +28,30 @@ type Cache interface {
// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
cache *lru.Cache
bitmaps map[uint64]*Bitmap
cache *lru.Cache
counts map[uint64]uint64
}
// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries int) *LRUCache {
c := &LRUCache{
cache: lru.New(maxEntries),
bitmaps: make(map[uint64]*Bitmap),
cache: lru.New(maxEntries),
counts: make(map[uint64]uint64),
}
c.cache.OnEvicted = c.onEvicted
return c
}
// Add adds a bitmap to the cache.
func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) {
c.cache.Add(bitmapID, bm)
c.bitmaps[bitmapID] = bm
func (c *LRUCache) Add(bitmapID, n uint64) {
c.cache.Add(bitmapID, n)
c.counts[bitmapID] = n
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) *Bitmap {
bm, ok := c.cache.Get(bitmapID)
if !ok {
return nil
}
return bm.(*Bitmap)
func (c *LRUCache) Get(bitmapID uint64) uint64 {
n, _ := c.cache.Get(bitmapID)
return n.(uint64)
}
// Len returns the number of items in the cache.
@ -65,35 +62,35 @@ func (c *LRUCache) Invalidate() {}
// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *LRUCache) BitmapIDs() []uint64 {
a := make([]uint64, 0, len(c.bitmaps))
for id := range c.bitmaps {
a := make([]uint64, 0, len(c.counts))
for id := range c.counts {
a = append(a, id)
}
sort.Sort(uint64Slice(a))
return a
}
// Top returns all bitmaps in the cache.
// Top returns all counts in the cache.
func (c *LRUCache) Top() []BitmapPair {
a := make([]BitmapPair, 0, len(c.bitmaps))
for id, bm := range c.bitmaps {
a := make([]BitmapPair, 0, len(c.counts))
for id, n := range c.counts {
a = append(a, BitmapPair{
ID: id,
Bitmap: bm,
ID: id,
Count: uint64(n),
})
}
sort.Sort(BitmapPairs(a))
return a
}
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) }
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}
// RankCache represents a cache with sorted entries.
type RankCache struct {
entries map[uint64]*Bitmap
entries map[uint64]uint64
rankings []BitmapPair // cached, ordered list
updateN int
@ -107,25 +104,25 @@ type RankCache struct {
// NewRankCache returns a new instance of RankCache.
func NewRankCache() *RankCache {
return &RankCache{
entries: make(map[uint64]*Bitmap),
entries: make(map[uint64]uint64),
}
}
// Add adds a bitmap to the cache.
func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
func (c *RankCache) Add(bitmapID uint64, n uint64) {
// Ignore if the bit count on the bitmap is below the threshold.
if bm.Count() < c.ThresholdValue {
if n < c.ThresholdValue {
return
}
// Add to cache.
c.entries[bitmapID] = bm
c.entries[bitmapID] = n
// If size is larger than the threshold then trim it.
if len(c.entries) > c.ThresholdLength {
c.update()
for id, bm := range c.entries {
if bm.Count() <= c.ThresholdValue {
for id, n := range c.entries {
if n <= c.ThresholdValue {
delete(c.entries, id)
}
}
@ -133,7 +130,7 @@ func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
}
// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] }
func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] }
// Len returns the number of items in the cache.
func (c *RankCache) Len() int { return len(c.entries) }
@ -160,10 +157,10 @@ func (c *RankCache) Invalidate() {
func (c *RankCache) update() {
// Convert cache to a sorted list.
rankings := make([]BitmapPair, 0, len(c.entries))
for id, bm := range c.entries {
for id, n := range c.entries {
rankings = append(rankings, BitmapPair{
ID: id,
Bitmap: bm,
ID: id,
Count: n,
})
}
sort.Sort(BitmapPairs(rankings))
@ -171,7 +168,7 @@ func (c *RankCache) update() {
// Store the count of the item at the threshold index.
c.rankings = rankings
if len(c.rankings) > c.ThresholdIndex {
c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count()
c.ThresholdValue = rankings[c.ThresholdIndex].Count
} else {
c.ThresholdValue = 1
}
@ -198,8 +195,8 @@ var _ Cache = &RankCache{}
// BitmapPair represents a bitmap with an associated identifier.
type BitmapPair struct {
ID uint64
Bitmap *Bitmap
ID uint64
Count uint64
}
// BitmapPairs is a sortable list of BitmapPair objects.
@ -207,7 +204,7 @@ type BitmapPairs []BitmapPair
func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitmapPairs) Len() int { return len(p) }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
type Pair struct {
Key uint64 `json:"key"`

7
executor.go
View file

@ -262,7 +262,7 @@ func (e *Executor) executeDifferenceSlice(db string, c *pql.Difference, slice ui
other = other.Difference(bm)
}
}
other.SetCount(other.BitCount())
other.InvalidateCount()
return other, nil
}
@ -294,7 +294,7 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint
other = other.Intersect(bm)
}
}
other.SetCount(other.BitCount())
other.InvalidateCount()
return other, nil
}
@ -327,7 +327,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi
other = other.Union(bm)
}
}
other.SetCount(other.BitCount())
other.InvalidateCount()
return other, nil
}
@ -486,7 +486,6 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op
}
// Create HTTP request.
println("dbg.host?", node.Host)
req, err := http.NewRequest("POST", (&url.URL{
Scheme: "http",
Host: node.Host,

39
executor_test.go
View file

@ -24,12 +24,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 8 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
@ -43,14 +39,13 @@ func TestExecutor_Execute_Difference(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 10 { // b1010
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -69,12 +64,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 2 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 4 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -92,12 +83,8 @@ func TestExecutor_Execute_Union(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 5 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 6 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -315,12 +302,8 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 6 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}

25
fragment.go
View file

@ -323,11 +323,6 @@ func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
}
func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
// Read from cache.
if bm := f.cache.Get(bitmapID); bm != nil {
return bm
}
// Read bitmap from storage.
bm := NewBitmap()
f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) {
@ -335,8 +330,8 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
bm.SetBit(profileID)
})
// Add to the cache.
f.cache.Add(bitmapID, bm)
// Update cache.
f.cache.Add(bitmapID, bm.Count())
return bm
}
@ -462,10 +457,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Iterate over rankings and add to results until we have enough.
results := make([]Pair, 0, opt.N)
for _, pair := range pairs {
bitmapID, bm := pair.ID, pair.Bitmap
bitmapID, n := pair.ID, pair.Count
// Ignore empty bitmaps.
if bm.Count() <= 0 {
if n <= 0 {
continue
}
@ -486,9 +481,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// The initial n pairs should simply be added to the results.
if opt.N == 0 || len(results) < opt.N {
// Calculate count and append.
count := bm.Count()
count := n
if opt.Src != nil {
count = opt.Src.IntersectionCount(bm)
count = opt.Src.IntersectionCount(f.Bitmap(bitmapID))
}
if count == 0 {
continue
@ -516,13 +511,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// If the bitmap doesn't have enough bits set before the intersection
// then we can assume that any remaing bitmaps also have a count too low.
if bm.Count() < threshold {
if n < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// last bitmap in our current result set.
count := opt.Src.IntersectionCount(bm)
count := opt.Src.IntersectionCount(f.Bitmap(bitmapID))
if count < threshold {
continue
}
@ -553,8 +548,8 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
pairs := make([]BitmapPair, len(bitmapIDs))
for i, bitmapID := range bitmapIDs {
pairs[i] = BitmapPair{
ID: bitmapID,
Bitmap: f.Bitmap(bitmapID),
ID: bitmapID,
Count: f.Bitmap(bitmapID).Count(),
}
}
return pairs

8
handler_test.go
View file

@ -229,8 +229,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
} else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {
@ -286,8 +286,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" {

128
internal/internal.pb.go
View file

@ -10,7 +10,6 @@ It is generated from these files:
It has these top-level messages:
Bitmap
Chunk
Pair
Bit
Profile
@ -42,7 +41,7 @@ var _ = math.Inf
const _ = proto.GoGoProtoPackageIsVersion1
type Bitmap struct {
Chunks []*Chunk `protobuf:"bytes,1,rep,name=Chunks" json:"Chunks,omitempty"`
Bits []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -52,9 +51,9 @@ func (m *Bitmap) String() string { return proto.CompactTextString(m)
func (*Bitmap) ProtoMessage() {}
func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} }
func (m *Bitmap) GetChunks() []*Chunk {
func (m *Bitmap) GetBits() []uint64 {
if m != nil {
return m.Chunks
return m.Bits
}
return nil
}
@ -66,31 +65,6 @@ func (m *Bitmap) GetAttrs() []*Attr {
return nil
}
type Chunk struct {
Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"`
Value []uint64 `protobuf:"varint,2,rep,name=Value" json:"Value,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Chunk) Reset() { *m = Chunk{} }
func (m *Chunk) String() string { return proto.CompactTextString(m) }
func (*Chunk) ProtoMessage() {}
func (*Chunk) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} }
func (m *Chunk) GetKey() uint64 {
if m != nil && m.Key != nil {
return *m.Key
}
return 0
}
func (m *Chunk) GetValue() []uint64 {
if m != nil {
return m.Value
}
return nil
}
type Pair struct {
Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"`
Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"`
@ -100,7 +74,7 @@ type Pair struct {
func (m *Pair) Reset() { *m = Pair{} }
func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} }
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} }
func (m *Pair) GetKey() uint64 {
if m != nil && m.Key != nil {
@ -125,7 +99,7 @@ type Bit struct {
func (m *Bit) Reset() { *m = Bit{} }
func (m *Bit) String() string { return proto.CompactTextString(m) }
func (*Bit) ProtoMessage() {}
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} }
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} }
func (m *Bit) GetBitmapID() uint64 {
if m != nil && m.BitmapID != nil {
@ -150,7 +124,7 @@ type Profile struct {
func (m *Profile) Reset() { *m = Profile{} }
func (m *Profile) String() string { return proto.CompactTextString(m) }
func (*Profile) ProtoMessage() {}
func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} }
func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} }
func (m *Profile) GetID() uint64 {
if m != nil && m.ID != nil {
@ -177,7 +151,7 @@ type Attr struct {
func (m *Attr) Reset() { *m = Attr{} }
func (m *Attr) String() string { return proto.CompactTextString(m) }
func (*Attr) ProtoMessage() {}
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} }
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} }
func (m *Attr) GetKey() string {
if m != nil && m.Key != nil {
@ -215,7 +189,7 @@ type AttrMap struct {
func (m *AttrMap) Reset() { *m = AttrMap{} }
func (m *AttrMap) String() string { return proto.CompactTextString(m) }
func (*AttrMap) ProtoMessage() {}
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} }
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} }
func (m *AttrMap) GetAttrs() []*Attr {
if m != nil {
@ -238,7 +212,7 @@ type QueryRequest struct {
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} }
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} }
func (m *QueryRequest) GetDB() string {
if m != nil && m.DB != nil {
@ -299,7 +273,7 @@ type QueryResponse struct {
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} }
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} }
func (m *QueryResponse) GetErr() string {
if m != nil && m.Err != nil {
@ -333,7 +307,7 @@ type QueryResult struct {
func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} }
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} }
func (m *QueryResult) GetBitmap() *Bitmap {
if m != nil {
@ -375,7 +349,7 @@ type ImportRequest struct {
func (m *ImportRequest) Reset() { *m = ImportRequest{} }
func (m *ImportRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRequest) ProtoMessage() {}
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} }
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} }
func (m *ImportRequest) GetDB() string {
if m != nil && m.DB != nil {
@ -420,7 +394,7 @@ type ImportResponse struct {
func (m *ImportResponse) Reset() { *m = ImportResponse{} }
func (m *ImportResponse) String() string { return proto.CompactTextString(m) }
func (*ImportResponse) ProtoMessage() {}
func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} }
func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} }
func (m *ImportResponse) GetErr() string {
if m != nil && m.Err != nil {
@ -440,7 +414,7 @@ type BlockDataRequest struct {
func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} }
func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) }
func (*BlockDataRequest) ProtoMessage() {}
func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} }
func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} }
func (m *BlockDataRequest) GetDB() string {
if m != nil && m.DB != nil {
@ -479,7 +453,7 @@ type BlockDataResponse struct {
func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} }
func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) }
func (*BlockDataResponse) ProtoMessage() {}
func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} }
func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} }
func (m *BlockDataResponse) GetBitmapIDs() []uint64 {
if m != nil {
@ -503,7 +477,7 @@ type Cache struct {
func (m *Cache) Reset() { *m = Cache{} }
func (m *Cache) String() string { return proto.CompactTextString(m) }
func (*Cache) ProtoMessage() {}
func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} }
func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} }
func (m *Cache) GetBitmapIDs() []uint64 {
if m != nil {
@ -520,7 +494,7 @@ type SliceMaxResponse struct {
func (m *SliceMaxResponse) Reset() { *m = SliceMaxResponse{} }
func (m *SliceMaxResponse) String() string { return proto.CompactTextString(m) }
func (*SliceMaxResponse) ProtoMessage() {}
func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} }
func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} }
func (m *SliceMaxResponse) GetSliceMax() uint64 {
if m != nil && m.SliceMax != nil {
@ -531,7 +505,6 @@ func (m *SliceMaxResponse) GetSliceMax() uint64 {
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Chunk)(nil), "internal.Chunk")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*Profile)(nil), "internal.Profile")
@ -549,38 +522,37 @@ func init() {
}
var fileDescriptorInternal = []byte{
// 514 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5b, 0x8b, 0xd3, 0x40,
0x18, 0x25, 0x4d, 0xda, 0xb4, 0x5f, 0x6c, 0xb7, 0x1d, 0x11, 0x83, 0xb0, 0xb8, 0xcc, 0x8a, 0x14,
0x1f, 0x56, 0x58, 0x7c, 0xf2, 0xcd, 0xb6, 0x8a, 0xcb, 0xb2, 0xcb, 0x5e, 0xd4, 0x67, 0x87, 0x3a,
0x6e, 0xe3, 0x26, 0x99, 0x38, 0x99, 0x80, 0x7d, 0xf2, 0xaf, 0xfb, 0xcd, 0x2d, 0x8d, 0x58, 0x11,
0x9f, 0xda, 0x39, 0xdf, 0xe5, 0x9c, 0x39, 0x39, 0x03, 0x8f, 0xb3, 0x52, 0x71, 0x59, 0xb2, 0xfc,
0xa5, 0xff, 0x73, 0x52, 0x49, 0xa1, 0x04, 0x19, 0xfa, 0x33, 0x7d, 0x0f, 0x83, 0x45, 0xa6, 0x0a,
0x56, 0x91, 0xa7, 0x30, 0x58, 0x6e, 0x9a, 0xf2, 0xbe, 0x4e, 0x83, 0xa3, 0x70, 0x9e, 0x9c, 0x1e,
0x9c, 0xb4, 0x43, 0x06, 0x27, 0x87, 0xd0, 0x7f, 0xa3, 0x94, 0xac, 0xd3, 0x9e, 0xa9, 0x4f, 0x76,
0x75, 0x0d, 0xd3, 0x63, 0xe8, 0xdb, 0xbe, 0x04, 0xc2, 0x73, 0xbe, 0xc5, 0x2d, 0xbd, 0x79, 0x44,
0xc6, 0xd0, 0xff, 0xc4, 0xf2, 0x86, 0x9b, 0xa1, 0x88, 0x52, 0x88, 0xae, 0x58, 0x26, 0xff, 0xe8,
0x59, 0x8a, 0xa6, 0x54, 0xd8, 0x83, 0x47, 0xfa, 0x02, 0x42, 0x94, 0x44, 0xa6, 0x30, 0xb4, 0xca,
0xce, 0x56, 0xae, 0x6f, 0x06, 0xa3, 0x2b, 0x29, 0xbe, 0x66, 0x39, 0x47, 0xc8, 0xf6, 0xbe, 0x82,
0xd8, 0x41, 0x04, 0xa0, 0xd7, 0x76, 0xfe, 0x43, 0xea, 0x25, 0x44, 0xfa, 0xb7, 0xab, 0x62, 0x44,
0x1e, 0x42, 0x72, 0xab, 0x64, 0x56, 0xde, 0x79, 0xbd, 0x01, 0x82, 0x48, 0xf9, 0x11, 0x67, 0x2d,
0x14, 0x22, 0x64, 0x54, 0x2c, 0x84, 0xc8, 0x2d, 0x14, 0x21, 0x34, 0xa4, 0x73, 0x88, 0xf5, 0xbe,
0x0b, 0x74, 0xb1, 0x65, 0x0e, 0xf6, 0x32, 0xff, 0x84, 0x07, 0xd7, 0x0d, 0x97, 0xdb, 0x1b, 0xfe,
0xbd, 0xe1, 0xb5, 0xd2, 0xa2, 0x57, 0x0b, 0x27, 0x00, 0x6d, 0x30, 0x35, 0x73, 0xb5, 0x11, 0x99,
0xc0, 0xe0, 0x36, 0xcf, 0xd6, 0xbc, 0x46, 0x5e, 0xb4, 0x4e, 0xfb, 0xe1, 0xae, 0x5a, 0x5b, 0x5a,
0xad, 0xe4, 0x43, 0x56, 0xe0, 0x1a, 0x56, 0x54, 0x69, 0x1f, 0xa1, 0x90, 0x1c, 0x40, 0x7c, 0xdd,
0xb0, 0x52, 0x35, 0x45, 0x3a, 0x40, 0x60, 0xac, 0xb7, 0xdc, 0xf0, 0x42, 0x28, 0x9e, 0xc6, 0x46,
0x6a, 0x06, 0x63, 0x27, 0xa0, 0xae, 0x44, 0x59, 0x73, 0xed, 0xc1, 0x5b, 0x29, 0x51, 0x82, 0xbe,
0xee, 0x73, 0x88, 0xb1, 0xd0, 0xe4, 0xca, 0x3b, 0xf7, 0x68, 0xa7, 0xdf, 0x8f, 0x61, 0x95, 0x1c,
0x77, 0xb4, 0x84, 0xa6, 0x71, 0xb6, 0x6b, 0x74, 0x15, 0xfa, 0x0d, 0x92, 0xee, 0xcc, 0x91, 0x4f,
0x9a, 0xe1, 0x4a, 0x4e, 0xa7, 0xbb, 0x09, 0x97, 0xc0, 0x11, 0x04, 0x97, 0xc6, 0x77, 0xf3, 0x01,
0x75, 0x4e, 0xfc, 0xf6, 0x8e, 0x8d, 0x26, 0x3e, 0x78, 0xcd, 0xe5, 0x86, 0x95, 0x77, 0xfc, 0x8b,
0xfb, 0x02, 0x9f, 0x61, 0x7c, 0x56, 0x54, 0x42, 0xaa, 0xbf, 0x18, 0xfb, 0x4e, 0xb2, 0x82, 0x3b,
0x63, 0xf1, 0x68, 0x8c, 0xc5, 0xdd, 0x2e, 0x55, 0x3e, 0x67, 0xda, 0x58, 0x6d, 0x35, 0x4e, 0xb7,
0x41, 0xab, 0xd1, 0x59, 0x9d, 0xdc, 0x43, 0x98, 0x78, 0x86, 0x3d, 0xce, 0xd1, 0x73, 0x98, 0x2e,
0x72, 0xb1, 0xbe, 0x5f, 0x31, 0xc5, 0xfe, 0x5f, 0x03, 0x1e, 0xcd, 0x34, 0xf2, 0xeb, 0x54, 0xbf,
0x86, 0x59, 0x67, 0x99, 0xa3, 0xfb, 0x4d, 0x67, 0xb0, 0x47, 0xa7, 0x7d, 0x61, 0x4f, 0xf0, 0x31,
0xb1, 0xf5, 0x66, 0x5f, 0x3f, 0x7d, 0x06, 0x53, 0xc3, 0x7a, 0xc1, 0x7e, 0xb4, 0x6b, 0x31, 0x56,
0x1e, 0xb3, 0x8f, 0xe7, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0xc6, 0x93, 0x16, 0x36, 0x04,
0x00, 0x00,
// 499 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40,
0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4,
0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb,
0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58,
0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95,
0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca,
0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46,
0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a,
0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2,
0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb,
0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31,
0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8,
0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6,
0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6,
0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b,
0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9,
0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65,
0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa,
0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6,
0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c,
0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f,
0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf,
0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07,
0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c,
0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08,
0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd,
0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1,
0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf,
0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f,
0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac,
0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe,
0x03, 0x00, 0x00,
}

9
internal/internal.proto
View file

@ -1,13 +1,8 @@
package internal;
message Bitmap {
repeated Chunk Chunks = 1;
repeated Attr Attrs = 2;
}
message Chunk {
required uint64 Key = 1;
repeated uint64 Value = 2;
repeated uint64 Bits = 1;
repeated Attr Attrs = 2;
}
message Pair {

56
roaring/roaring.go
View file

@ -308,7 +308,11 @@ func (b *Bitmap) writeOp(op *op) error {
}
// Iterator returns a new iterator for the bitmap.
func (b *Bitmap) Iterator() *Iterator { return &Iterator{bitmap: b} }
func (b *Bitmap) Iterator() *Iterator {
itr := &Iterator{bitmap: b}
itr.Seek(0)
return itr
}
// Iterator represents an iterator over a Bitmap.
type Iterator struct {
@ -410,6 +414,56 @@ func (itr *Iterator) peek() uint64 {
return uint64(key)<<16 | uint64(itr.j)
}
// BufIterator wraps an iterator to provide the ability to unread values.
type BufIterator struct {
buf struct {
v uint64
eof bool
full bool
}
itr *Iterator
}
// NewBufIterator returns a buffered iterator that wraps itr.
func NewBufIterator(itr *Iterator) *BufIterator {
return &BufIterator{itr: itr}
}
// Seek moves to the first pair equal to or greater than pseek/bseek.
func (itr *BufIterator) Seek(v uint64) {
itr.buf.full = false
itr.itr.Seek(v)
}
// Next returns the next pair in the bitmap.
// If a value has been buffered then it is returned and the buffer is cleared.
func (itr *BufIterator) Next() (v uint64, eof bool) {
if itr.buf.full {
itr.buf.full = false
return itr.buf.v, itr.buf.eof
}
// Read value onto buffer in case of unread.
itr.buf.v, itr.buf.eof = itr.itr.Next()
return itr.buf.v, itr.buf.eof
}
// Peek reads the next value but leaves it on the buffer.
func (itr *BufIterator) Peek() (v uint64, eof bool) {
v, eof = itr.Next()
itr.Unread()
return
}
// Unread pushes previous pair on to the buffer.
// Panics if the buffer is already full.
func (itr *BufIterator) Unread() {
if itr.buf.full {
panic("roaring.BufIterator: buffer full")
}
itr.buf.full = true
}
// The maximum size of array containers.
const arrayMaxSize = 4096

19
vendor/github.com/yasushi-saito/rbtree/LICENSE generated vendored
View file

@ -1,19 +0,0 @@
Copyright (C) 2012 Yasushi Saito
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

136
vendor/github.com/yasushi-saito/rbtree/README generated vendored
View file

@ -1,136 +0,0 @@
A red-black tree with an API similar to C++ STL's.
INSTALLATION
go get github.com/yasushi-saito/rbtree
EXAMPLE
More examples can be found in rbtree_test.go
import "github.com/yasushi-saito/rbtree"
type MyItem struct {
key int
value string
}
tree := rbtree.NewTree(func(a, b Item) int { return a.(MyItem).key - b.(MyItem).key })
tree.Insert(MyItem{10, "value10"})
tree.Insert(MyItem{12, "value12"})
fmt.Println("Get(10) ->", tree.Get(MyItem{10, ""}))
fmt.Println("Get(11) ->", tree.Get(MyItem{11, ""}))
// Find an element >= 11
iter := tree.FindGE(MyItem{11, ""})
fmt.Println("FindGE(11) ->", iter.Item())
// Find an element >= 13
iter = tree.FindGE(MyItem{13, ""})
if !iter.End() { panic("There should be no element >= 13") }
// Output:
// Get(10) -> {10 value10}
// Get(11) -> <nil>
// FindGE(11) -> {12 value12}
TYPES
type CompareFunc func(a, b Item) int
CompareFunc returns 0 if a==b, <0 if a<b, >0 if a>b.
type Item interface{}
Item is the object stored in each tree node.
type Iterator struct {
// contains filtered or unexported fields
}
Iterator allows scanning tree elements in sort order.
Iterator invalidation rule is the same as C++ std::map<>'s. That is, if
you delete the element that an iterator points to, the iterator becomes
invalid. For other operation types, the iterator remains valid.
func (iter Iterator) Equal(iter2 Iterator) bool
func (iter Iterator) Item() interface{}
Return the current element.
REQUIRES: !iter.Limit() && !iter.NegativeLimit()
func (iter Iterator) Limit() bool
Check if the iterator points beyond the max element in the tree
func (iter Iterator) Max() bool
Check if the iterator points to the maximum element in the tree
func (iter Iterator) Min() bool
Check if the iterator points to the minimum element in the tree
func (iter Iterator) NegativeLimit() bool
Check if the iterator points before the minumum element in the tree
func (iter Iterator) Next() Iterator
Create a new iterator that points to the successor of the current
element.
REQUIRES: !iter.Limit()
func (iter Iterator) Prev() Iterator
Create a new iterator that points to the predecessor of the current
node.
REQUIRES: !iter.NegativeLimit()
type Tree struct {
// contains filtered or unexported fields
}
func NewTree(compare CompareFunc) *Tree
Create a new empty tree.
func (root *Tree) DeleteWithIterator(iter Iterator)
Delete the current item.
REQUIRES: !iter.Limit() && !iter.NegativeLimit()
func (root *Tree) DeleteWithKey(key Item) bool
Delete an item with the given key. Return true iff the item was found.
func (root *Tree) FindGE(key Item) Iterator
Find the smallest element N such that N >= key, and return the iterator
pointing to the element. If no such element is found, return
root.Limit().
func (root *Tree) FindLE(key Item) Iterator
Find the largest element N such that N <= key, and return the iterator
pointing to the element. If no such element is found, return
iter.NegativeLimit().
func (root *Tree) Get(key Item) Item
A convenience function for finding an element equal to key. Return nil
if not found.
func (root *Tree) Insert(item Item) bool
Insert an item. If the item is already in the tree, do nothing and
return false. Else return true.
func (root *Tree) Len() int
Return the number of elements in the tree.
func (root *Tree) Limit() Iterator
Create an iterator that points beyond the maximum item in the tree
func (root *Tree) Max() Iterator
Create an iterator that points at the maximum item in the tree
If the tree is empty, return NegativeLimit()
func (root *Tree) Min() Iterator
Create an iterator that points to the minimum item in the tree If the
tree is empty, return Limit()
func (root *Tree) NegativeLimit() Iterator
Create an iterator that points before the minimum item in the tree

713
vendor/github.com/yasushi-saito/rbtree/rbtree.go generated vendored
View file

@ -1,713 +0,0 @@
//
// Created by Yaz Saito on 06/10/12.
//
// A red-black tree with an API similar to C++ STL's.
//
// The implementation is inspired (read: stolen) from:
// http://en.literateprograms.org/Red-black_tree_(C)#chunk use:private function prototypes.
//
package rbtree
//
// Public definitions
//
// Item is the object stored in each tree node.
type Item interface{}
// CompareFunc returns 0 if a==b, <0 if a<b, >0 if a>b.
type CompareFunc func(a, b Item) int
type Tree struct {
// Root of the tree
root *node
// The minimum and maximum nodes under the root.
minNode, maxNode *node
// Number of nodes under root, including the root
count int
compare CompareFunc
}
// Create a new empty tree.
func NewTree(compare CompareFunc) *Tree {
return &Tree{compare: compare}
}
// Return the number of elements in the tree.
func (root *Tree) Len() int {
return root.count
}
// A convenience function for finding an element equal to key. Return
// nil if not found.
func (root *Tree) Get(key Item) Item {
n, exact := root.findGE(key)
if exact {
return n.item
}
return nil
}
// Create an iterator that points to the minimum item in the tree
// If the tree is empty, return Limit()
func (root *Tree) Min() Iterator {
return Iterator{root, root.minNode}
}
// Create an iterator that points at the maximum item in the tree
//
// If the tree is empty, return NegativeLimit()
func (root *Tree) Max() Iterator {
if root.maxNode == nil {
// TODO: there are a few checks of this form.
// Perhaps set maxNode=negativeLimit when the tree is empty
return Iterator{root, negativeLimitNode}
}
return Iterator{root, root.maxNode}
}
// Create an iterator that points beyond the maximum item in the tree
func (root *Tree) Limit() Iterator {
return Iterator{root, nil}
}
// Create an iterator that points before the minimum item in the tree
func (root *Tree) NegativeLimit() Iterator {
return Iterator{root, negativeLimitNode}
}
// Find the smallest element N such that N >= key, and return the
// iterator pointing to the element. If no such element is found,
// return root.Limit().
func (root *Tree) FindGE(key Item) Iterator {
n, _ := root.findGE(key)
return Iterator{root, n}
}
// Find the largest element N such that N <= key, and return the
// iterator pointing to the element. If no such element is found,
// return iter.NegativeLimit().
func (root *Tree) FindLE(key Item) Iterator {
n, exact := root.findGE(key)
if exact {
return Iterator{root, n}
}
if n != nil {
return Iterator{root, n.doPrev()}
}
if root.maxNode == nil {
return Iterator{root, negativeLimitNode}
}
return Iterator{root, root.maxNode}
}
// Insert an item. If the item is already in the tree, do nothing and
// return false. Else return true.
func (root *Tree) Insert(item Item) bool {
// TODO: delay creating n until it is found to be inserted
n := root.doInsert(item)
if n == nil {
return false
}
n.color = red
for true {
// Case 1: N is at the root
if n.parent == nil {
n.color = black
break
}
// Case 2: The parent is black, so the tree already
// satisfies the RB properties
if n.parent.color == black {
break
}
// Case 3: parent and uncle are both red.
// Then paint both black and make grandparent red.
grandparent := n.parent.parent
var uncle *node
if n.parent.isLeftChild() {
uncle = grandparent.right
} else {
uncle = grandparent.left
}
if uncle != nil && uncle.color == red {
n.parent.color = black
uncle.color = black
grandparent.color = red
n = grandparent
continue
}
// Case 4: parent is red, uncle is black (1)
if n.isRightChild() && n.parent.isLeftChild() {
root.rotateLeft(n.parent)
n = n.left
continue
}
if n.isLeftChild() && n.parent.isRightChild() {
root.rotateRight(n.parent)
n = n.right
continue
}
// Case 5: parent is read, uncle is black (2)
n.parent.color = black
grandparent.color = red
if n.isLeftChild() {
root.rotateRight(grandparent)
} else {
root.rotateLeft(grandparent)
}
break
}
return true
}
// Delete an item with the given key. Return true iff the item was
// found.
func (root *Tree) DeleteWithKey(key Item) bool {
iter := root.FindGE(key)
if iter.node != nil {
root.DeleteWithIterator(iter)
return true
}
return false
}
// Delete the current item.
//
// REQUIRES: !iter.Limit() && !iter.NegativeLimit()
func (root *Tree) DeleteWithIterator(iter Iterator) {
doAssert(!iter.Limit() && !iter.NegativeLimit())
root.doDelete(iter.node)
}
// Iterator allows scanning tree elements in sort order.
//
// Iterator invalidation rule is the same as C++ std::map<>'s. That
// is, if you delete the element that an iterator points to, the
// iterator becomes invalid. For other operation types, the iterator
// remains valid.
type Iterator struct {
root *Tree
node *node
}
func (iter Iterator) Equal(iter2 Iterator) bool {
return iter.node == iter2.node
}
// Check if the iterator points beyond the max element in the tree
func (iter Iterator) Limit() bool {
return iter.node == nil
}
// Check if the iterator points to the minimum element in the tree
func (iter Iterator) Min() bool {
return iter.node == iter.root.minNode
}
// Check if the iterator points to the maximum element in the tree
func (iter Iterator) Max() bool {
return iter.node == iter.root.maxNode
}
// Check if the iterator points before the minumum element in the tree
func (iter Iterator) NegativeLimit() bool {
return iter.node == negativeLimitNode
}
// Return the current element.
//
// REQUIRES: !iter.Limit() && !iter.NegativeLimit()
func (iter Iterator) Item() interface{} {
return iter.node.item
}
// Create a new iterator that points to the successor of the current element.
//
// REQUIRES: !iter.Limit()
func (iter Iterator) Next() Iterator {
doAssert(!iter.Limit())
if iter.NegativeLimit() {
return Iterator{iter.root, iter.root.minNode}
}
return Iterator{iter.root, iter.node.doNext()}
}
// Create a new iterator that points to the predecessor of the current
// node.
//
// REQUIRES: !iter.NegativeLimit()
func (iter Iterator) Prev() Iterator {
doAssert(!iter.NegativeLimit())
if !iter.Limit() {
return Iterator{iter.root, iter.node.doPrev()}
}
if iter.root.maxNode == nil {
return Iterator{iter.root, negativeLimitNode}
}
return Iterator{iter.root, iter.root.maxNode}
}
func doAssert(b bool) {
if !b {
panic("rbtree internal assertion failed")
}
}
const red = iota
const black = 1 + iota
type node struct {
item Item
parent, left, right *node
color int // black or red
}
var negativeLimitNode *node
//
// Internal node attribute accessors
//
func getColor(n *node) int {
if n == nil {
return black
}
return n.color
}
func (n *node) isLeftChild() bool {
return n == n.parent.left
}
func (n *node) isRightChild() bool {
return n == n.parent.right
}
func (n *node) sibling() *node {
doAssert(n.parent != nil)
if n.isLeftChild() {
return n.parent.right
}
return n.parent.left
}
// Return the minimum node that's larger than N. Return nil if no such
// node is found.
func (n *node) doNext() *node {
if n.right != nil {
m := n.right
for m.left != nil {
m = m.left
}
return m
}
for n != nil {
p := n.parent
if p == nil {
return nil
}
if n.isLeftChild() {
return p
}
n = p
}
return nil
}
// Return the maximum node that's smaller than N. Return nil if no
// such node is found.
func (n *node) doPrev() *node {
if n.left != nil {
return maxPredecessor(n)
}
for n != nil {
p := n.parent
if p == nil {
break
}
if n.isRightChild() {
return p
}
n = p
}
return negativeLimitNode
}
// Return the predecessor of "n".
func maxPredecessor(n *node) *node {
doAssert(n.left != nil)
m := n.left
for m.right != nil {
m = m.right
}
return m
}
//
// Tree methods
//
//
// Private methods
//
func (root *Tree) recomputeMinNode() {
root.minNode = root.root
if root.minNode != nil {
for root.minNode.left != nil {
root.minNode = root.minNode.left
}
}
}
func (root *Tree) recomputeMaxNode() {
root.maxNode = root.root
if root.maxNode != nil {
for root.maxNode.right != nil {
root.maxNode = root.maxNode.right
}
}
}
func (root *Tree) maybeSetMinNode(n *node) {
if root.minNode == nil {
root.minNode = n
root.maxNode = n
} else if root.compare(n.item, root.minNode.item) < 0 {
root.minNode = n
}
}
func (root *Tree) maybeSetMaxNode(n *node) {
if root.maxNode == nil {
root.minNode = n
root.maxNode = n
} else if root.compare(n.item, root.maxNode.item) > 0 {
root.maxNode = n
}
}
// Try inserting "item" into the tree. Return nil if the item is
// already in the tree. Otherwise return a new (leaf) node.
func (root *Tree) doInsert(item Item) *node {
if root.root == nil {
n := &node{item: item}
root.root = n
root.minNode = n
root.maxNode = n
root.count++
return n
}
parent := root.root
for true {
comp := root.compare(item, parent.item)
if comp == 0 {
return nil
} else if comp < 0 {
if parent.left == nil {
n := &node{item: item, parent: parent}
parent.left = n
root.count++
root.maybeSetMinNode(n)
return n
} else {
parent = parent.left
}
} else {
if parent.right == nil {
n := &node{item: item, parent: parent}
parent.right = n
root.count++
root.maybeSetMaxNode(n)
return n
} else {
parent = parent.right
}
}
}
panic("should not reach here")
}
// Find a node whose item >= key. The 2nd return value is true iff the
// node.item==key. Returns (nil, false) if all nodes in the tree are <
// key.
func (root *Tree) findGE(key Item) (*node, bool) {
n := root.root
for true {
if n == nil {
return nil, false
}
comp := root.compare(key, n.item)
if comp == 0 {
return n, true
} else if comp < 0 {
if n.left != nil {
n = n.left
} else {
return n, false
}
} else {
if n.right != nil {
n = n.right
} else {
succ := n.doNext()
if succ == nil {
return nil, false
} else {
comp = root.compare(key, succ.item)
return succ, (comp == 0)
}
}
}
}
panic("should not reach here")
}
// Delete N from the tree.
func (root *Tree) doDelete(n *node) {
if n.left != nil && n.right != nil {
pred := maxPredecessor(n)
root.swapNodes(n, pred)
}
doAssert(n.left == nil || n.right == nil)
child := n.right
if child == nil {
child = n.left
}
if n.color == black {
n.color = getColor(child)
root.deleteCase1(n)
}
root.replaceNode(n, child)
if n.parent == nil && child != nil {
child.color = black
}
root.count--
if root.count == 0 {
root.minNode = nil
root.maxNode = nil
} else {
if root.minNode == n {
root.recomputeMinNode()
}
if root.maxNode == n {
root.recomputeMaxNode()
}
}
}
// Move n to the pred's place, and vice versa
//
// TODO: this code is overly convoluted
func (root *Tree) swapNodes(n, pred *node) {
doAssert(pred != n)
isLeft := pred.isLeftChild()
tmp := *pred
root.replaceNode(n, pred)
pred.color = n.color
if tmp.parent == n {
// swap the positions of n and pred
if isLeft {
pred.left = n
pred.right = n.right
if pred.right != nil {
pred.right.parent = pred
}
} else {
pred.left = n.left
if pred.left != nil {
pred.left.parent = pred
}
pred.right = n
}
n.item = tmp.item
n.parent = pred
n.left = tmp.left
if n.left != nil {
n.left.parent = n
}
n.right = tmp.right
if n.right != nil {
n.right.parent = n
}
} else {
pred.left = n.left
if pred.left != nil {
pred.left.parent = pred
}
pred.right = n.right
if pred.right != nil {
pred.right.parent = pred
}
if isLeft {
tmp.parent.left = n
} else {
tmp.parent.right = n
}
n.item = tmp.item
n.parent = tmp.parent
n.left = tmp.left
if n.left != nil {
n.left.parent = n
}
n.right = tmp.right
if n.right != nil {
n.right.parent = n
}
}
n.color = tmp.color
}
func (root *Tree) deleteCase1(n *node) {
for true {
if n.parent != nil {
if getColor(n.sibling()) == red {
n.parent.color = red
n.sibling().color = black
if n == n.parent.left {
root.rotateLeft(n.parent)
} else {
root.rotateRight(n.parent)
}
}
if getColor(n.parent) == black &&
getColor(n.sibling()) == black &&
getColor(n.sibling().left) == black &&
getColor(n.sibling().right) == black {
n.sibling().color = red
n = n.parent
continue
} else {
// case 4
if getColor(n.parent) == red &&
getColor(n.sibling()) == black &&
getColor(n.sibling().left) == black &&
getColor(n.sibling().right) == black {
n.sibling().color = red
n.parent.color = black
} else {
root.deleteCase5(n)
}
}
}
break
}
}
func (root *Tree) deleteCase5(n *node) {
if n == n.parent.left &&
getColor(n.sibling()) == black &&
getColor(n.sibling().left) == red &&
getColor(n.sibling().right) == black {
n.sibling().color = red
n.sibling().left.color = black
root.rotateRight(n.sibling())
} else if n == n.parent.right &&
getColor(n.sibling()) == black &&
getColor(n.sibling().right) == red &&
getColor(n.sibling().left) == black {
n.sibling().color = red
n.sibling().right.color = black
root.rotateLeft(n.sibling())
}
// case 6
n.sibling().color = getColor(n.parent)
n.parent.color = black
if n == n.parent.left {
doAssert(getColor(n.sibling().right) == red)
n.sibling().right.color = black
root.rotateLeft(n.parent)
} else {
doAssert(getColor(n.sibling().left) == red)
n.sibling().left.color = black
root.rotateRight(n.parent)
}
}
func (root *Tree) replaceNode(oldn, newn *node) {
if oldn.parent == nil {
root.root = newn
} else {
if oldn == oldn.parent.left {
oldn.parent.left = newn
} else {
oldn.parent.right = newn
}
}
if newn != nil {
newn.parent = oldn.parent
}
}
/*
X Y
A Y => X C
B C A B
*/
func (root *Tree) rotateLeft(x *node) {
y := x.right
x.right = y.left
if y.left != nil {
y.left.parent = x
}
y.parent = x.parent
if x.parent == nil {
root.root = y
} else {
if x.isLeftChild() {
x.parent.left = y
} else {
x.parent.right = y
}
}
y.left = x
x.parent = y
}
/*
Y X
X C => A Y
A B B C
*/
func (root *Tree) rotateRight(y *node) {
x := y.left
// Move "B"
y.left = x.right
if x.right != nil {
x.right.parent = y
}
x.parent = y.parent
if y.parent == nil {
root.root = x
} else {
if y.isLeftChild() {
y.parent.left = x
} else {
y.parent.right = x
}
}
x.right = y
y.parent = x
}
func init() {
negativeLimitNode = &node{}
}