diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index f302fa89c..6447da71b 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -28,10 +28,6 @@ "ImportPath": "github.com/golang/groupcache/lru", "Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52" }, - { - "ImportPath": "github.com/yasushi-saito/rbtree", - "Rev": "571e2538414bf914c7e2909b61217b4e3e5508f4" - }, { "ImportPath": "golang.org/x/sys/unix", "Rev": "50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e" diff --git a/bitmap.go b/bitmap.go index 41bd98e6c..36f6c73c3 100644 --- a/bitmap.go +++ b/bitmap.go @@ -3,28 +3,16 @@ package pilosa // #cgo CFLAGS:-mpopcnt import ( - "bytes" - "compress/gzip" - "encoding/base64" - "encoding/binary" - "encoding/gob" "encoding/json" - "io" - "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" - "github.com/yasushi-saito/rbtree" + "github.com/umbel/pilosa/roaring" ) -const CounterMask = uint64(0xffffffffffffffff) - -var CounterKey = int64(-1) - -// Bitmap represents a bitmap broken up into Chunks. -// Internally it is represented as a red-black tree of chunks. +// Bitmap represents a set of bits. type Bitmap struct { - tree *rbtree.Tree - bcount uint64 + data roaring.Bitmap + n uint64 // Attributes associated with the bitmap. Attrs map[string]interface{} @@ -32,166 +20,98 @@ type Bitmap struct { // NewBitmap returns a new instance of Bitmap. func NewBitmap(bits ...uint64) *Bitmap { - bm := &Bitmap{tree: rbtree.NewTree(rbtreeItemCompare)} + bm := &Bitmap{} for _, i := range bits { bm.SetBit(i) } return bm } -// Chunk returns the chunk within the bitmap. -// Returns nil if the chunk key does not exist. -func (b *Bitmap) Chunk(c *Chunk) *Chunk { - if n := b.tree.Get(c); n != nil { - return n.(*Chunk) - } - return nil -} - -// AddChunk adds c to the bitmap. -func (b *Bitmap) AddChunk(c *Chunk) { b.tree.Insert(c) } - -// Chunks returns a list of all chunks. -func (b *Bitmap) Chunks() []*Chunk { - var a []*Chunk - for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() { - a = append(a, itr.Item().Clone()) - } - return a -} - -// ChunkIterator returns an iterator for looping over the bitmap's chunks. -func (b *Bitmap) ChunkIterator() *ChunkIterator { - return &ChunkIterator{b.tree.Min()} -} - -// Clone returns a copy of b. -func (b *Bitmap) Clone() *Bitmap { - itr := b.ChunkIterator() - other := NewBitmap() - - for { - if itr.Limit() { - break - } - - other.AddChunk(itr.Item().Clone()) - itr = itr.Next() - } - return other -} - // Merge adds chunks from other to b. // Chunks in b are overwritten if they exist in other. func (b *Bitmap) Merge(other *Bitmap) { - for itr := other.ChunkIterator(); !itr.Limit(); itr = itr.Next() { - b.AddChunk(itr.Item().Clone()) + itr := other.data.Iterator() + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + b.SetBit(v) } } // IntersectionCount returns the number of intersections between b and other. func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { - itr0 := b.ChunkIterator() - itr1 := other.ChunkIterator() + // OPTIMIZE: Implement roaring.Bitmap.IntersectionCount() - results := uint64(0) + itr0 := roaring.NewBufIterator(b.data.Iterator()) + itr1 := roaring.NewBufIterator(other.data.Iterator()) + + var n uint64 for { - if itr1.Limit() || itr0.Limit() { + v0, eof0 := itr0.Next() + v1, eof1 := itr1.Next() + + if eof0 || eof1 { break - } else if itr0.Item().Key < itr1.Item().Key { - itr0 = itr0.Next() - } else if itr0.Item().Key > itr1.Item().Key { - itr1 = itr1.Next() - } else if itr0.Item().Key == itr1.Item().Key { - results += itr0.Item().Value.andcount(itr1.Item().Value) - itr0 = itr0.Next() - itr1 = itr1.Next() + } else if v0 < v1 { + itr1.Unread() + } else if v0 > v1 { + itr0.Unread() + } else { + n++ } } - return results + return n } // Intersect returns the itersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { - itr0 := b.ChunkIterator() - itr1 := other.ChunkIterator() + // OPTIMIZE: Implement roaring.Bitmap.Intersect() + + itr0 := roaring.NewBufIterator(b.data.Iterator()) + itr1 := roaring.NewBufIterator(other.data.Iterator()) output := NewBitmap() for { - if itr1.Limit() || itr0.Limit() { + v0, eof0 := itr0.Next() + v1, eof1 := itr1.Next() + + if eof0 || eof1 { break - } else if itr0.Item().Key < itr1.Item().Key { - itr0 = itr0.Next() - } else if itr0.Item().Key > itr1.Item().Key { - itr1 = itr1.Next() - } else if itr0.Item().Key == itr1.Item().Key { - output.AddChunk(&Chunk{ - Key: itr0.Item().Key, - Value: itr0.Item().Value.intersect(itr1.Item().Value), - }) - itr0 = itr0.Next() - itr1 = itr1.Next() + } else if v0 < v1 { + itr1.Unread() + } else if v0 > v1 { + itr0.Unread() + } else { + output.SetBit(v0) } } return output } -// Invert returns a bitwise inversion of b. -func (b *Bitmap) Invert() *Bitmap { - other := NewBitmap() - for i := b.ChunkIterator(); !i.Limit(); i = i.Next() { - other.AddChunk(&Chunk{ - Key: i.Item().Key, - Value: i.Item().Value.invert(), - }) - } - return other - -} - // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { - itr0 := b.ChunkIterator() - itr1 := other.ChunkIterator() + // OPTIMIZE: Implement roaring.Bitmap.Union() + + itr0 := roaring.NewBufIterator(b.data.Iterator()) + itr1 := roaring.NewBufIterator(other.data.Iterator()) output := NewBitmap() - eof := uint64(0xdeadbeef) - for { - if itr0.Limit() && itr1.Limit() { + v0, eof0 := itr0.Next() + v1, eof1 := itr1.Next() + + if eof0 && eof1 { break - } else if itr0.Limit() { - if eof == itr1.Item().Key { - break - } - output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value}) - eof = itr1.Item().Key - itr1 = itr1.Next() - } else if itr1.Limit() { - if eof == itr0.Item().Key { - break - } - output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value}) - eof = itr0.Item().Key - itr0 = itr0.Next() - } else if itr0.Item().Key < itr1.Item().Key { - output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value}) - eof = itr0.Item().Key - itr0 = itr0.Next() - } else if itr0.Item().Key > itr1.Item().Key { - output.AddChunk(&Chunk{itr1.Item().Key, itr1.Item().Value}) - eof = itr1.Item().Key - itr1 = itr1.Next() - } else if itr0.Item().Key == itr1.Item().Key { - output.AddChunk(&Chunk{ - Key: itr0.Item().Key, - Value: itr0.Item().Value.union(itr1.Item().Value), - }) - eof = itr0.Item().Key - itr0 = itr0.Next() - itr1 = itr1.Next() + } else if eof0 { + output.SetBit(v1) + } else if eof1 { + output.SetBit(v0) + } else if v0 < v1 { + output.SetBit(v0) + itr1.Unread() + } else if v0 > v1 { + output.SetBit(v1) + itr0.Unread() } else { - panic("unreachable") + output.SetBit(v0) } } return output @@ -199,118 +119,30 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { // Difference returns the diff of b and other. func (b *Bitmap) Difference(other *Bitmap) *Bitmap { - itr0 := b.ChunkIterator() - itr1 := other.ChunkIterator() + // OPTIMIZE: Implement roaring.Bitmap.Difference() + + itr0 := roaring.NewBufIterator(b.data.Iterator()) + itr1 := roaring.NewBufIterator(other.data.Iterator()) output := NewBitmap() for { - if itr0.Limit() && itr1.Limit() { - break - } else if itr0.Limit() { - break - } else if itr1.Limit() { - output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value}) - itr0 = itr0.Next() - } else if itr0.Item().Key < itr1.Item().Key { - output.AddChunk(&Chunk{itr0.Item().Key, itr0.Item().Value}) - itr0 = itr0.Next() - } else if itr0.Item().Key > itr1.Item().Key { - itr1 = itr1.Next() - } else if itr0.Item().Key == itr1.Item().Key { - chunk := &Chunk{ - Key: itr0.Item().Key, - Value: itr0.Item().Value.difference(itr1.Item().Value), - } + v0, eof0 := itr0.Next() + v1, eof1 := itr1.Next() - // Do not add if all bits are zeroed. - if chunk.Value.bitcount() > 0 { - output.AddChunk(chunk) - } - - itr0 = itr0.Next() - itr1 = itr1.Next() - } else { - panic("unreachable") + if eof0 { + break + } else if eof1 { + output.SetBit(v0) + } else if v0 < v1 { + output.SetBit(v0) + itr1.Unread() + } else if v0 > v1 { + itr0.Unread() } } return output } -// ToRawCompressString returns a compressed, hex-encoded string of b. -func (b *Bitmap) ToRawCompressString() (string, int) { - var bt bytes.Buffer - buf := gzip.NewWriter(&bt) - binary.Write(buf, binary.LittleEndian, uint64(b.tree.Len())) - max_slice := 0 - for i := b.tree.Min(); !i.Limit(); i = i.Next() { - obj := i.Item().(*Chunk) - max_slice = int(obj.Key) - binary.Write(buf, binary.LittleEndian, obj.Key) - for _, v := range obj.Value { - binary.Write(buf, binary.LittleEndian, v) - } - } - buf.Flush() - //buf.Close() - max_slice = max_slice / 32 - return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice -} - -// WriteTo writes the encoded bitmap to w. -func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { - // Wrap output in gzip compression. - z := gzip.NewWriter(w) - - // Encode chunk count. - enc := gob.NewEncoder(w) - if err := enc.Encode(b.tree.Len()); err != nil { - return 0, err - } - - // Encode all chunks. - for i := b.tree.Min(); !i.Limit(); i = i.Next() { - if err := enc.Encode(i.Item().(*Chunk)); err != nil { - return 0, err - } - } - - // Flush and close. - if err := z.Close(); err != nil { - return 0, err - } - - return 0, nil -} - -// ReadFrom reads encoded bitmap data from r into b. -func (b *Bitmap) ReadFrom(r io.Reader) (n int64, err error) { - // Uncompress from gzip format. - z, err := gzip.NewReader(r) - if err != nil { - return 0, err - } - dec := gob.NewDecoder(z) - - // Read size from data. - var size int - if err := dec.Decode(&size); err != nil { - return 0, err - } - - // Read chunks into bitmap. - b.tree = rbtree.NewTree(rbtreeItemCompare) - for i := 0; i < size; i++ { - var chunk Chunk - if err := dec.Decode(&chunk); err != nil { - return 0, err - } - b.AddChunk(&chunk) - } - b.SetCount(b.BitCount()) - - return 0, nil -} - // MarshalJSON returns a JSON-encoded byte slice of b. func (b *Bitmap) MarshalJSON() ([]byte, error) { var o struct { @@ -327,94 +159,45 @@ func (b *Bitmap) MarshalJSON() ([]byte, error) { return json.Marshal(&o) } -// MarshalBinary returns a gob-encoded byte slice of b. -func (b *Bitmap) MarshalBinary() ([]byte, error) { - var buf bytes.Buffer - if _, err := b.WriteTo(&buf); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// UnmarshalBinary decodes a gob-encoded byte slice into b. -func (b *Bitmap) UnmarshalBinary(data []byte) error { - _, err := b.ReadFrom(bytes.NewReader(data)) - return err -} - // Bits returns the bits in b as a slice of ints. func (b *Bitmap) Bits() []uint64 { - result := make([]uint64, 0, b.Count()) - - for i := b.ChunkIterator(); !i.Limit(); i = i.Next() { - item := i.Item() - chunk := item.Key - for bi, block := range item.Value { - for bit := uint(0); bit < 64; bit++ { - if (block & (1 << bit)) != 0 { - idx := chunk << 11 - idx = idx | uint64((uint(bi)<<6)|bit) - result = append(result, idx) - } - } - } - + a := make([]uint64, 0, b.Count()) + itr := b.data.Iterator() + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + a = append(a, v) } - return result + return a } -// setBit sets the i-th bit of the bitmap. +// SetBit sets the i-th bit of the bitmap. func (b *Bitmap) SetBit(i uint64) (changed bool) { - address := deref(i) - - chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)}) - if chunk == nil { - chunk = &Chunk{address.ChunkKey, make(Blocks, 32)} - b.AddChunk(chunk) - } - - changed = chunk.Value.setBit(address.BlockIndex, address.Bit) + changed, _ = b.data.Add(i) if changed { - b.bcount++ + b.n++ } - return changed } -// clearBit clears the i-th bit of the bitmap. +// ClearBit clears the i-th bit of the bitmap. func (b *Bitmap) ClearBit(i uint64) (changed bool) { - address := deref(i) - - chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)}) - if chunk == nil { - return false + changed, _ = b.data.Remove(i) + if changed { + b.n-- } - - changed = chunk.Value.clearBit(address.BlockIndex, address.Bit) - if changed && b.bcount > 0 { - b.bcount-- - } - return changed } -// Len returns the number of chunks in b. -func (b *Bitmap) Len() int { return b.tree.Len() } - -// SetCount sets the number of set bits in the bitmap. -func (b *Bitmap) SetCount(c uint64) { b.bcount = c } +// InvalidateCount updates the cached count in the bitmap. +func (b *Bitmap) InvalidateCount() { + itr, n := b.data.Iterator(), uint64(0) + for _, eof := itr.Next(); !eof; _, eof = itr.Next() { + n++ + } + b.n = n +} // Count returns the number of set bits in the bitmap. -func (b *Bitmap) Count() uint64 { return b.bcount } - -// BitCount calculates the number of set bits in the bitmap from raw chunk data. -func (b *Bitmap) BitCount() uint64 { - var n uint64 - for i := b.ChunkIterator(); !i.Limit(); i = i.Next() { - n += i.Item().Value.bitcount() - } - return n -} +func (b *Bitmap) Count() uint64 { return b.n } // encodeBitmap converts b into its internal representation. func encodeBitmap(b *Bitmap) *internal.Bitmap { @@ -422,13 +205,10 @@ func encodeBitmap(b *Bitmap) *internal.Bitmap { return nil } - pb := &internal.Bitmap{ + return &internal.Bitmap{ + Bits: b.Bits(), Attrs: encodeAttrs(b.Attrs), } - for i := b.tree.Min(); !i.Limit(); i = i.Next() { - pb.Chunks = append(pb.Chunks, encodeChunk(i.Item().(*Chunk))) - } - return pb } // decodeBitmap converts b from its internal representation. @@ -439,10 +219,9 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap { b := NewBitmap() b.Attrs = decodeAttrs(pb.GetAttrs()) - for _, chunk := range pb.GetChunks() { - b.AddChunk(decodeChunk(chunk)) + for _, v := range pb.GetBits() { + b.SetBit(v) } - b.SetCount(b.BitCount()) return b } @@ -454,149 +233,3 @@ func Union(bitmaps []*Bitmap) *Bitmap { } return other } - -// Chunk represents a set of blocks in a Bitmap. -type Chunk struct { - Key uint64 - Value Blocks -} - -// Clone returns a copy of c. -func (c *Chunk) Clone() *Chunk { - return &Chunk{ - Key: c.Key, - Value: c.Value.copy(), - } -} - -// encodeChunks encodes c into its internal representation. -func encodeChunk(c *Chunk) *internal.Chunk { - return &internal.Chunk{ - Key: proto.Uint64(c.Key), - Value: []uint64(c.Value), - } -} - -// decodeChunk decodes c from its internal representation. -func decodeChunk(pb *internal.Chunk) *Chunk { - return &Chunk{ - Key: pb.GetKey(), - Value: Blocks(pb.GetValue()), - } -} - -// ChunkIterator represents an object for iterating over chunks in a bitmap. -type ChunkIterator struct { - itr rbtree.Iterator -} - -// Limit return true when the iterator is at the end of iteration. -func (r *ChunkIterator) Limit() bool { - return r.itr.Limit() -} - -// Next moves the iterator to the next chunk. -func (r *ChunkIterator) Next() *ChunkIterator { - r.itr = r.itr.Next() - return r -} - -// Item returns the current item that the iterator is pointing at. -func (r *ChunkIterator) Item() *Chunk { - if r.itr.Item() != nil { - return r.itr.Item().(*Chunk) - } - return nil -} - -func rbtreeItemCompare(a, b rbtree.Item) int { - aKey, bKey := a.(*Chunk).Key, b.(*Chunk).Key - if aKey < bKey { - return -1 - } else if aKey > bKey { - return 1 - } - return 0 -} - -type Blocks []uint64 - -// NewBlocks returns a 32-length Block. -func NewBlocks() Blocks { - return make(Blocks, 32) -} - -func (a Blocks) bitcount() uint64 { - return popcntSlice(a) -} - -func (a Blocks) union(other Blocks) Blocks { - ret := NewBlocks() - for i, _ := range a { - ret[i] = a[i] | other[i] - } - return ret -} - -func (a Blocks) invert() Blocks { - other := NewBlocks() - for i, _ := range a { - other[i] = ^a[i] - } - return other -} - -func (a Blocks) copy() Blocks { - other := NewBlocks() - for i, _ := range a { - other[i] = a[i] - } - return other -} - -func (a Blocks) andcount(other Blocks) uint64 { - return popcntAndSliceAsm(a, other) -} - -func (a Blocks) intersect(other Blocks) Blocks { - ret := NewBlocks() - for i, _ := range a { - ret[i] = a[i] & other[i] - } - return ret -} - -func (a Blocks) difference(other Blocks) Blocks { - ret := NewBlocks() - for i, _ := range a { - ret[i] = a[i] &^ other[i] - } - return ret -} - -func (a Blocks) setBit(i uint8, bit uint8) (changed bool) { - val := a[i] & (1 << bit) - a[i] |= 1 << bit - return val == 0 -} - -func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) { - val := a[i] & (1 << bit) - a[i] &= ^(1 << bit) - return val != 0 -} - -// Address represents a location for a given chunk/block/bit. -type Address struct { - ChunkKey uint64 - BlockIndex uint8 - Bit uint8 -} - -func deref(pos uint64) Address { - chunkKey := pos >> 11 // div by 2048 - offset := pos & 0x7FF // mod by 2048 - blockIndex := uint8(offset >> 6) // div by 64 - bit_offset := uint8(offset & 0x3F) // mod by 64 - return Address{chunkKey, blockIndex, bit_offset} -} diff --git a/cache.go b/cache.go index 17a69f75b..03e02d15f 100644 --- a/cache.go +++ b/cache.go @@ -10,10 +10,10 @@ import ( "github.com/umbel/pilosa/internal" ) -// Cache represents a cache for bitmaps. +// Cache represents a cache for bitmap counts. type Cache interface { - Add(bitmapID uint64, bm *Bitmap) - Get(bitmapID uint64) *Bitmap + Add(bitmapID uint64, n uint64) + Get(bitmapID uint64) uint64 Len() int // Returns a list of all bitmap IDs. @@ -28,33 +28,30 @@ type Cache interface { // LRUCache represents a least recently used Cache implemenation. type LRUCache struct { - cache *lru.Cache - bitmaps map[uint64]*Bitmap + cache *lru.Cache + counts map[uint64]uint64 } // NewLRUCache returns a new instance of LRUCache. func NewLRUCache(maxEntries int) *LRUCache { c := &LRUCache{ - cache: lru.New(maxEntries), - bitmaps: make(map[uint64]*Bitmap), + cache: lru.New(maxEntries), + counts: make(map[uint64]uint64), } c.cache.OnEvicted = c.onEvicted return c } // Add adds a bitmap to the cache. -func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) { - c.cache.Add(bitmapID, bm) - c.bitmaps[bitmapID] = bm +func (c *LRUCache) Add(bitmapID, n uint64) { + c.cache.Add(bitmapID, n) + c.counts[bitmapID] = n } // Get returns a bitmap with a given id. -func (c *LRUCache) Get(bitmapID uint64) *Bitmap { - bm, ok := c.cache.Get(bitmapID) - if !ok { - return nil - } - return bm.(*Bitmap) +func (c *LRUCache) Get(bitmapID uint64) uint64 { + n, _ := c.cache.Get(bitmapID) + return n.(uint64) } // Len returns the number of items in the cache. @@ -65,35 +62,35 @@ func (c *LRUCache) Invalidate() {} // BitmapIDs returns a list of all bitmap IDs in the cache. func (c *LRUCache) BitmapIDs() []uint64 { - a := make([]uint64, 0, len(c.bitmaps)) - for id := range c.bitmaps { + a := make([]uint64, 0, len(c.counts)) + for id := range c.counts { a = append(a, id) } sort.Sort(uint64Slice(a)) return a } -// Top returns all bitmaps in the cache. +// Top returns all counts in the cache. func (c *LRUCache) Top() []BitmapPair { - a := make([]BitmapPair, 0, len(c.bitmaps)) - for id, bm := range c.bitmaps { + a := make([]BitmapPair, 0, len(c.counts)) + for id, n := range c.counts { a = append(a, BitmapPair{ - ID: id, - Bitmap: bm, + ID: id, + Count: uint64(n), }) } sort.Sort(BitmapPairs(a)) return a } -func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) } +func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } // Ensure LRUCache implements Cache. var _ Cache = &LRUCache{} // RankCache represents a cache with sorted entries. type RankCache struct { - entries map[uint64]*Bitmap + entries map[uint64]uint64 rankings []BitmapPair // cached, ordered list updateN int @@ -107,25 +104,25 @@ type RankCache struct { // NewRankCache returns a new instance of RankCache. func NewRankCache() *RankCache { return &RankCache{ - entries: make(map[uint64]*Bitmap), + entries: make(map[uint64]uint64), } } // Add adds a bitmap to the cache. -func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) { +func (c *RankCache) Add(bitmapID uint64, n uint64) { // Ignore if the bit count on the bitmap is below the threshold. - if bm.Count() < c.ThresholdValue { + if n < c.ThresholdValue { return } // Add to cache. - c.entries[bitmapID] = bm + c.entries[bitmapID] = n // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { c.update() - for id, bm := range c.entries { - if bm.Count() <= c.ThresholdValue { + for id, n := range c.entries { + if n <= c.ThresholdValue { delete(c.entries, id) } } @@ -133,7 +130,7 @@ func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) { } // Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] } +func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] } // Len returns the number of items in the cache. func (c *RankCache) Len() int { return len(c.entries) } @@ -160,10 +157,10 @@ func (c *RankCache) Invalidate() { func (c *RankCache) update() { // Convert cache to a sorted list. rankings := make([]BitmapPair, 0, len(c.entries)) - for id, bm := range c.entries { + for id, n := range c.entries { rankings = append(rankings, BitmapPair{ - ID: id, - Bitmap: bm, + ID: id, + Count: n, }) } sort.Sort(BitmapPairs(rankings)) @@ -171,7 +168,7 @@ func (c *RankCache) update() { // Store the count of the item at the threshold index. c.rankings = rankings if len(c.rankings) > c.ThresholdIndex { - c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count() + c.ThresholdValue = rankings[c.ThresholdIndex].Count } else { c.ThresholdValue = 1 } @@ -198,8 +195,8 @@ var _ Cache = &RankCache{} // BitmapPair represents a bitmap with an associated identifier. type BitmapPair struct { - ID uint64 - Bitmap *Bitmap + ID uint64 + Count uint64 } // BitmapPairs is a sortable list of BitmapPair objects. @@ -207,7 +204,7 @@ type BitmapPairs []BitmapPair func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitmapPairs) Len() int { return len(p) } -func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() } +func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } type Pair struct { Key uint64 `json:"key"` diff --git a/executor.go b/executor.go index 9e46258b8..c5ce6c457 100644 --- a/executor.go +++ b/executor.go @@ -262,7 +262,7 @@ func (e *Executor) executeDifferenceSlice(db string, c *pql.Difference, slice ui other = other.Difference(bm) } } - other.SetCount(other.BitCount()) + other.InvalidateCount() return other, nil } @@ -294,7 +294,7 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint other = other.Intersect(bm) } } - other.SetCount(other.BitCount()) + other.InvalidateCount() return other, nil } @@ -327,7 +327,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi other = other.Union(bm) } } - other.SetCount(other.BitCount()) + other.InvalidateCount() return other, nil } @@ -486,7 +486,6 @@ func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, op } // Create HTTP request. - println("dbg.host?", node.Host) req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", Host: node.Host, diff --git a/executor_test.go b/executor_test.go index b7af82276..8aeeaa335 100644 --- a/executor_test.go +++ b/executor_test.go @@ -24,12 +24,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { - t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) - } else if chunks[0].Value[0] != 8 { - t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) - } else if chunks[1].Value[0] != 2 { - t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } @@ -43,14 +39,13 @@ func TestExecutor_Execute_Difference(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2) idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3) idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 1 { - t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) - } else if chunks[0].Value[0] != 10 { // b1010 - t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { + t.Fatalf("unexpected bits: %+v", bits) } } @@ -69,12 +64,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { - t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) - } else if chunks[0].Value[0] != 2 { - t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) - } else if chunks[1].Value[0] != 4 { - t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { + t.Fatalf("unexpected bits: %+v", bits) } } @@ -92,12 +83,8 @@ func TestExecutor_Execute_Union(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { - t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) - } else if chunks[0].Value[0] != 5 { - t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) - } else if chunks[1].Value[0] != 6 { - t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + t.Fatalf("unexpected bits: %+v", bits) } } @@ -315,12 +302,8 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { e := NewExecutor(idx.Index, c) if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if chunks := res[0].(*pilosa.Bitmap).Chunks(); len(chunks) != 3 { - t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) - } else if chunks[0].Value[0] != 6 { - t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) - } else if chunks[1].Value[0] != 2 { - t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) { + t.Fatalf("unexpected bits: %+v", bits) } } diff --git a/fragment.go b/fragment.go index 1770d25c3..f4dcf4559 100644 --- a/fragment.go +++ b/fragment.go @@ -323,11 +323,6 @@ func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { } func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { - // Read from cache. - if bm := f.cache.Get(bitmapID); bm != nil { - return bm - } - // Read bitmap from storage. bm := NewBitmap() f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) { @@ -335,8 +330,8 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { bm.SetBit(profileID) }) - // Add to the cache. - f.cache.Add(bitmapID, bm) + // Update cache. + f.cache.Add(bitmapID, bm.Count()) return bm } @@ -462,10 +457,10 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Iterate over rankings and add to results until we have enough. results := make([]Pair, 0, opt.N) for _, pair := range pairs { - bitmapID, bm := pair.ID, pair.Bitmap + bitmapID, n := pair.ID, pair.Count // Ignore empty bitmaps. - if bm.Count() <= 0 { + if n <= 0 { continue } @@ -486,9 +481,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // The initial n pairs should simply be added to the results. if opt.N == 0 || len(results) < opt.N { // Calculate count and append. - count := bm.Count() + count := n if opt.Src != nil { - count = opt.Src.IntersectionCount(bm) + count = opt.Src.IntersectionCount(f.Bitmap(bitmapID)) } if count == 0 { continue @@ -516,13 +511,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If the bitmap doesn't have enough bits set before the intersection // then we can assume that any remaing bitmaps also have a count too low. - if bm.Count() < threshold { + if n < threshold { break } // Calculate the intersecting bit count and skip if it's below our // last bitmap in our current result set. - count := opt.Src.IntersectionCount(bm) + count := opt.Src.IntersectionCount(f.Bitmap(bitmapID)) if count < threshold { continue } @@ -553,8 +548,8 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { pairs := make([]BitmapPair, len(bitmapIDs)) for i, bitmapID := range bitmapIDs { pairs[i] = BitmapPair{ - ID: bitmapID, - Bitmap: f.Bitmap(bitmapID), + ID: bitmapID, + Count: f.Bitmap(bitmapID).Count(), } } return pairs diff --git a/handler_test.go b/handler_test.go index c2b3c885e..66a72f681 100644 --- a/handler_test.go +++ b/handler_test.go @@ -229,8 +229,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 { - t.Fatalf("unexpected bitmap chunk length: %d", len(a)) + } else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { @@ -286,8 +286,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if a := resp.Results[0].GetBitmap().GetChunks(); len(a) != 2 { - t.Fatalf("unexpected bitmap chunk length: %d", len(a)) + if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 32869d702..c61ad59c7 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -10,7 +10,6 @@ It is generated from these files: It has these top-level messages: Bitmap - Chunk Pair Bit Profile @@ -42,7 +41,7 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion1 type Bitmap struct { - Chunks []*Chunk `protobuf:"bytes,1,rep,name=Chunks" json:"Chunks,omitempty"` + Bits []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -52,9 +51,9 @@ func (m *Bitmap) String() string { return proto.CompactTextString(m) func (*Bitmap) ProtoMessage() {} func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } -func (m *Bitmap) GetChunks() []*Chunk { +func (m *Bitmap) GetBits() []uint64 { if m != nil { - return m.Chunks + return m.Bits } return nil } @@ -66,31 +65,6 @@ func (m *Bitmap) GetAttrs() []*Attr { return nil } -type Chunk struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Value []uint64 `protobuf:"varint,2,rep,name=Value" json:"Value,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Chunk) Reset() { *m = Chunk{} } -func (m *Chunk) String() string { return proto.CompactTextString(m) } -func (*Chunk) ProtoMessage() {} -func (*Chunk) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } - -func (m *Chunk) GetKey() uint64 { - if m != nil && m.Key != nil { - return *m.Key - } - return 0 -} - -func (m *Chunk) GetValue() []uint64 { - if m != nil { - return m.Value - } - return nil -} - type Pair struct { Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"` @@ -100,7 +74,7 @@ type Pair struct { func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } func (m *Pair) GetKey() uint64 { if m != nil && m.Key != nil { @@ -125,7 +99,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } func (m *Bit) GetBitmapID() uint64 { if m != nil && m.BitmapID != nil { @@ -150,7 +124,7 @@ type Profile struct { func (m *Profile) Reset() { *m = Profile{} } func (m *Profile) String() string { return proto.CompactTextString(m) } func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } +func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } func (m *Profile) GetID() uint64 { if m != nil && m.ID != nil { @@ -177,7 +151,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } func (m *Attr) GetKey() string { if m != nil && m.Key != nil { @@ -215,7 +189,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -238,7 +212,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } func (m *QueryRequest) GetDB() string { if m != nil && m.DB != nil { @@ -299,7 +273,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } func (m *QueryResponse) GetErr() string { if m != nil && m.Err != nil { @@ -333,7 +307,7 @@ type QueryResult struct { func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } func (m *QueryResult) GetBitmap() *Bitmap { if m != nil { @@ -375,7 +349,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } func (m *ImportRequest) GetDB() string { if m != nil && m.DB != nil { @@ -420,7 +394,7 @@ type ImportResponse struct { func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } func (m *ImportResponse) GetErr() string { if m != nil && m.Err != nil { @@ -440,7 +414,7 @@ type BlockDataRequest struct { func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } func (m *BlockDataRequest) GetDB() string { if m != nil && m.DB != nil { @@ -479,7 +453,7 @@ type BlockDataResponse struct { func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } func (m *BlockDataResponse) GetBitmapIDs() []uint64 { if m != nil { @@ -503,7 +477,7 @@ type Cache struct { func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } func (m *Cache) GetBitmapIDs() []uint64 { if m != nil { @@ -520,7 +494,7 @@ type SliceMaxResponse struct { func (m *SliceMaxResponse) Reset() { *m = SliceMaxResponse{} } func (m *SliceMaxResponse) String() string { return proto.CompactTextString(m) } func (*SliceMaxResponse) ProtoMessage() {} -func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } +func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } func (m *SliceMaxResponse) GetSliceMax() uint64 { if m != nil && m.SliceMax != nil { @@ -531,7 +505,6 @@ func (m *SliceMaxResponse) GetSliceMax() uint64 { func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") - proto.RegisterType((*Chunk)(nil), "internal.Chunk") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*Profile)(nil), "internal.Profile") @@ -549,38 +522,37 @@ func init() { } var fileDescriptorInternal = []byte{ - // 514 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5b, 0x8b, 0xd3, 0x40, - 0x18, 0x25, 0x4d, 0xda, 0xb4, 0x5f, 0x6c, 0xb7, 0x1d, 0x11, 0x83, 0xb0, 0xb8, 0xcc, 0x8a, 0x14, - 0x1f, 0x56, 0x58, 0x7c, 0xf2, 0xcd, 0xb6, 0x8a, 0xcb, 0xb2, 0xcb, 0x5e, 0xd4, 0x67, 0x87, 0x3a, - 0x6e, 0xe3, 0x26, 0x99, 0x38, 0x99, 0x80, 0x7d, 0xf2, 0xaf, 0xfb, 0xcd, 0x2d, 0x8d, 0x58, 0x11, - 0x9f, 0xda, 0x39, 0xdf, 0xe5, 0x9c, 0x39, 0x39, 0x03, 0x8f, 0xb3, 0x52, 0x71, 0x59, 0xb2, 0xfc, - 0xa5, 0xff, 0x73, 0x52, 0x49, 0xa1, 0x04, 0x19, 0xfa, 0x33, 0x7d, 0x0f, 0x83, 0x45, 0xa6, 0x0a, - 0x56, 0x91, 0xa7, 0x30, 0x58, 0x6e, 0x9a, 0xf2, 0xbe, 0x4e, 0x83, 0xa3, 0x70, 0x9e, 0x9c, 0x1e, - 0x9c, 0xb4, 0x43, 0x06, 0x27, 0x87, 0xd0, 0x7f, 0xa3, 0x94, 0xac, 0xd3, 0x9e, 0xa9, 0x4f, 0x76, - 0x75, 0x0d, 0xd3, 0x63, 0xe8, 0xdb, 0xbe, 0x04, 0xc2, 0x73, 0xbe, 0xc5, 0x2d, 0xbd, 0x79, 0x44, - 0xc6, 0xd0, 0xff, 0xc4, 0xf2, 0x86, 0x9b, 0xa1, 0x88, 0x52, 0x88, 0xae, 0x58, 0x26, 0xff, 0xe8, - 0x59, 0x8a, 0xa6, 0x54, 0xd8, 0x83, 0x47, 0xfa, 0x02, 0x42, 0x94, 0x44, 0xa6, 0x30, 0xb4, 0xca, - 0xce, 0x56, 0xae, 0x6f, 0x06, 0xa3, 0x2b, 0x29, 0xbe, 0x66, 0x39, 0x47, 0xc8, 0xf6, 0xbe, 0x82, - 0xd8, 0x41, 0x04, 0xa0, 0xd7, 0x76, 0xfe, 0x43, 0xea, 0x25, 0x44, 0xfa, 0xb7, 0xab, 0x62, 0x44, - 0x1e, 0x42, 0x72, 0xab, 0x64, 0x56, 0xde, 0x79, 0xbd, 0x01, 0x82, 0x48, 0xf9, 0x11, 0x67, 0x2d, - 0x14, 0x22, 0x64, 0x54, 0x2c, 0x84, 0xc8, 0x2d, 0x14, 0x21, 0x34, 0xa4, 0x73, 0x88, 0xf5, 0xbe, - 0x0b, 0x74, 0xb1, 0x65, 0x0e, 0xf6, 0x32, 0xff, 0x84, 0x07, 0xd7, 0x0d, 0x97, 0xdb, 0x1b, 0xfe, - 0xbd, 0xe1, 0xb5, 0xd2, 0xa2, 0x57, 0x0b, 0x27, 0x00, 0x6d, 0x30, 0x35, 0x73, 0xb5, 0x11, 0x99, - 0xc0, 0xe0, 0x36, 0xcf, 0xd6, 0xbc, 0x46, 0x5e, 0xb4, 0x4e, 0xfb, 0xe1, 0xae, 0x5a, 0x5b, 0x5a, - 0xad, 0xe4, 0x43, 0x56, 0xe0, 0x1a, 0x56, 0x54, 0x69, 0x1f, 0xa1, 0x90, 0x1c, 0x40, 0x7c, 0xdd, - 0xb0, 0x52, 0x35, 0x45, 0x3a, 0x40, 0x60, 0xac, 0xb7, 0xdc, 0xf0, 0x42, 0x28, 0x9e, 0xc6, 0x46, - 0x6a, 0x06, 0x63, 0x27, 0xa0, 0xae, 0x44, 0x59, 0x73, 0xed, 0xc1, 0x5b, 0x29, 0x51, 0x82, 0xbe, - 0xee, 0x73, 0x88, 0xb1, 0xd0, 0xe4, 0xca, 0x3b, 0xf7, 0x68, 0xa7, 0xdf, 0x8f, 0x61, 0x95, 0x1c, - 0x77, 0xb4, 0x84, 0xa6, 0x71, 0xb6, 0x6b, 0x74, 0x15, 0xfa, 0x0d, 0x92, 0xee, 0xcc, 0x91, 0x4f, - 0x9a, 0xe1, 0x4a, 0x4e, 0xa7, 0xbb, 0x09, 0x97, 0xc0, 0x11, 0x04, 0x97, 0xc6, 0x77, 0xf3, 0x01, - 0x75, 0x4e, 0xfc, 0xf6, 0x8e, 0x8d, 0x26, 0x3e, 0x78, 0xcd, 0xe5, 0x86, 0x95, 0x77, 0xfc, 0x8b, - 0xfb, 0x02, 0x9f, 0x61, 0x7c, 0x56, 0x54, 0x42, 0xaa, 0xbf, 0x18, 0xfb, 0x4e, 0xb2, 0x82, 0x3b, - 0x63, 0xf1, 0x68, 0x8c, 0xc5, 0xdd, 0x2e, 0x55, 0x3e, 0x67, 0xda, 0x58, 0x6d, 0x35, 0x4e, 0xb7, - 0x41, 0xab, 0xd1, 0x59, 0x9d, 0xdc, 0x43, 0x98, 0x78, 0x86, 0x3d, 0xce, 0xd1, 0x73, 0x98, 0x2e, - 0x72, 0xb1, 0xbe, 0x5f, 0x31, 0xc5, 0xfe, 0x5f, 0x03, 0x1e, 0xcd, 0x34, 0xf2, 0xeb, 0x54, 0xbf, - 0x86, 0x59, 0x67, 0x99, 0xa3, 0xfb, 0x4d, 0x67, 0xb0, 0x47, 0xa7, 0x7d, 0x61, 0x4f, 0xf0, 0x31, - 0xb1, 0xf5, 0x66, 0x5f, 0x3f, 0x7d, 0x06, 0x53, 0xc3, 0x7a, 0xc1, 0x7e, 0xb4, 0x6b, 0x31, 0x56, - 0x1e, 0xb3, 0x8f, 0xe7, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0xc6, 0x93, 0x16, 0x36, 0x04, - 0x00, 0x00, + // 499 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40, + 0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4, + 0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb, + 0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58, + 0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95, + 0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca, + 0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46, + 0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a, + 0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2, + 0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb, + 0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31, + 0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8, + 0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6, + 0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6, + 0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b, + 0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9, + 0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65, + 0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa, + 0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6, + 0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c, + 0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f, + 0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf, + 0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07, + 0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c, + 0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08, + 0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd, + 0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1, + 0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf, + 0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f, + 0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac, + 0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe, + 0x03, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 096620fcc..79e1d627d 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -1,13 +1,8 @@ package internal; message Bitmap { - repeated Chunk Chunks = 1; - repeated Attr Attrs = 2; -} - -message Chunk { - required uint64 Key = 1; - repeated uint64 Value = 2; + repeated uint64 Bits = 1; + repeated Attr Attrs = 2; } message Pair { diff --git a/roaring/roaring.go b/roaring/roaring.go index aba63077a..32a050438 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -308,7 +308,11 @@ func (b *Bitmap) writeOp(op *op) error { } // Iterator returns a new iterator for the bitmap. -func (b *Bitmap) Iterator() *Iterator { return &Iterator{bitmap: b} } +func (b *Bitmap) Iterator() *Iterator { + itr := &Iterator{bitmap: b} + itr.Seek(0) + return itr +} // Iterator represents an iterator over a Bitmap. type Iterator struct { @@ -410,6 +414,56 @@ func (itr *Iterator) peek() uint64 { return uint64(key)<<16 | uint64(itr.j) } +// BufIterator wraps an iterator to provide the ability to unread values. +type BufIterator struct { + buf struct { + v uint64 + eof bool + full bool + } + itr *Iterator +} + +// NewBufIterator returns a buffered iterator that wraps itr. +func NewBufIterator(itr *Iterator) *BufIterator { + return &BufIterator{itr: itr} +} + +// Seek moves to the first pair equal to or greater than pseek/bseek. +func (itr *BufIterator) Seek(v uint64) { + itr.buf.full = false + itr.itr.Seek(v) +} + +// Next returns the next pair in the bitmap. +// If a value has been buffered then it is returned and the buffer is cleared. +func (itr *BufIterator) Next() (v uint64, eof bool) { + if itr.buf.full { + itr.buf.full = false + return itr.buf.v, itr.buf.eof + } + + // Read value onto buffer in case of unread. + itr.buf.v, itr.buf.eof = itr.itr.Next() + return itr.buf.v, itr.buf.eof +} + +// Peek reads the next value but leaves it on the buffer. +func (itr *BufIterator) Peek() (v uint64, eof bool) { + v, eof = itr.Next() + itr.Unread() + return +} + +// Unread pushes previous pair on to the buffer. +// Panics if the buffer is already full. +func (itr *BufIterator) Unread() { + if itr.buf.full { + panic("roaring.BufIterator: buffer full") + } + itr.buf.full = true +} + // The maximum size of array containers. const arrayMaxSize = 4096 diff --git a/vendor/github.com/yasushi-saito/rbtree/LICENSE b/vendor/github.com/yasushi-saito/rbtree/LICENSE deleted file mode 100644 index fcf79ff80..000000000 --- a/vendor/github.com/yasushi-saito/rbtree/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2012 Yasushi Saito - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/yasushi-saito/rbtree/README b/vendor/github.com/yasushi-saito/rbtree/README deleted file mode 100644 index fdd1162c0..000000000 --- a/vendor/github.com/yasushi-saito/rbtree/README +++ /dev/null @@ -1,136 +0,0 @@ -A red-black tree with an API similar to C++ STL's. - -INSTALLATION - go get github.com/yasushi-saito/rbtree - -EXAMPLE - - More examples can be found in rbtree_test.go - - import "github.com/yasushi-saito/rbtree" - - type MyItem struct { - key int - value string - } - - tree := rbtree.NewTree(func(a, b Item) int { return a.(MyItem).key - b.(MyItem).key }) - tree.Insert(MyItem{10, "value10"}) - tree.Insert(MyItem{12, "value12"}) - - fmt.Println("Get(10) ->", tree.Get(MyItem{10, ""})) - fmt.Println("Get(11) ->", tree.Get(MyItem{11, ""})) - - // Find an element >= 11 - iter := tree.FindGE(MyItem{11, ""}) - fmt.Println("FindGE(11) ->", iter.Item()) - - // Find an element >= 13 - iter = tree.FindGE(MyItem{13, ""}) - if !iter.End() { panic("There should be no element >= 13") } - - // Output: - // Get(10) -> {10 value10} - // Get(11) -> - // FindGE(11) -> {12 value12} - -TYPES - -type CompareFunc func(a, b Item) int - CompareFunc returns 0 if a==b, <0 if a0 if a>b. - -type Item interface{} - Item is the object stored in each tree node. - -type Iterator struct { - // contains filtered or unexported fields -} - Iterator allows scanning tree elements in sort order. - - Iterator invalidation rule is the same as C++ std::map<>'s. That is, if - you delete the element that an iterator points to, the iterator becomes - invalid. For other operation types, the iterator remains valid. - -func (iter Iterator) Equal(iter2 Iterator) bool - -func (iter Iterator) Item() interface{} - Return the current element. - - REQUIRES: !iter.Limit() && !iter.NegativeLimit() - -func (iter Iterator) Limit() bool - Check if the iterator points beyond the max element in the tree - -func (iter Iterator) Max() bool - Check if the iterator points to the maximum element in the tree - -func (iter Iterator) Min() bool - Check if the iterator points to the minimum element in the tree - -func (iter Iterator) NegativeLimit() bool - Check if the iterator points before the minumum element in the tree - -func (iter Iterator) Next() Iterator - Create a new iterator that points to the successor of the current - element. - - REQUIRES: !iter.Limit() - -func (iter Iterator) Prev() Iterator - Create a new iterator that points to the predecessor of the current - node. - - REQUIRES: !iter.NegativeLimit() - -type Tree struct { - // contains filtered or unexported fields -} - -func NewTree(compare CompareFunc) *Tree - Create a new empty tree. - -func (root *Tree) DeleteWithIterator(iter Iterator) - Delete the current item. - - REQUIRES: !iter.Limit() && !iter.NegativeLimit() - -func (root *Tree) DeleteWithKey(key Item) bool - Delete an item with the given key. Return true iff the item was found. - -func (root *Tree) FindGE(key Item) Iterator - Find the smallest element N such that N >= key, and return the iterator - pointing to the element. If no such element is found, return - root.Limit(). - -func (root *Tree) FindLE(key Item) Iterator - Find the largest element N such that N <= key, and return the iterator - pointing to the element. If no such element is found, return - iter.NegativeLimit(). - -func (root *Tree) Get(key Item) Item - A convenience function for finding an element equal to key. Return nil - if not found. - -func (root *Tree) Insert(item Item) bool - Insert an item. If the item is already in the tree, do nothing and - return false. Else return true. - -func (root *Tree) Len() int - Return the number of elements in the tree. - -func (root *Tree) Limit() Iterator - Create an iterator that points beyond the maximum item in the tree - -func (root *Tree) Max() Iterator - Create an iterator that points at the maximum item in the tree - - If the tree is empty, return NegativeLimit() - -func (root *Tree) Min() Iterator - Create an iterator that points to the minimum item in the tree If the - tree is empty, return Limit() - -func (root *Tree) NegativeLimit() Iterator - Create an iterator that points before the minimum item in the tree - - diff --git a/vendor/github.com/yasushi-saito/rbtree/rbtree.go b/vendor/github.com/yasushi-saito/rbtree/rbtree.go deleted file mode 100644 index 3aeb6882e..000000000 --- a/vendor/github.com/yasushi-saito/rbtree/rbtree.go +++ /dev/null @@ -1,713 +0,0 @@ -// -// Created by Yaz Saito on 06/10/12. -// - -// A red-black tree with an API similar to C++ STL's. -// -// The implementation is inspired (read: stolen) from: -// http://en.literateprograms.org/Red-black_tree_(C)#chunk use:private function prototypes. -// -package rbtree - -// -// Public definitions -// - -// Item is the object stored in each tree node. -type Item interface{} - -// CompareFunc returns 0 if a==b, <0 if a0 if a>b. -type CompareFunc func(a, b Item) int - -type Tree struct { - // Root of the tree - root *node - - // The minimum and maximum nodes under the root. - minNode, maxNode *node - - // Number of nodes under root, including the root - count int - compare CompareFunc -} - -// Create a new empty tree. -func NewTree(compare CompareFunc) *Tree { - return &Tree{compare: compare} -} - -// Return the number of elements in the tree. -func (root *Tree) Len() int { - return root.count -} - -// A convenience function for finding an element equal to key. Return -// nil if not found. -func (root *Tree) Get(key Item) Item { - n, exact := root.findGE(key) - if exact { - return n.item - } - return nil -} - -// Create an iterator that points to the minimum item in the tree -// If the tree is empty, return Limit() -func (root *Tree) Min() Iterator { - return Iterator{root, root.minNode} -} - -// Create an iterator that points at the maximum item in the tree -// -// If the tree is empty, return NegativeLimit() -func (root *Tree) Max() Iterator { - if root.maxNode == nil { - // TODO: there are a few checks of this form. - // Perhaps set maxNode=negativeLimit when the tree is empty - return Iterator{root, negativeLimitNode} - } - return Iterator{root, root.maxNode} -} - -// Create an iterator that points beyond the maximum item in the tree -func (root *Tree) Limit() Iterator { - return Iterator{root, nil} -} - -// Create an iterator that points before the minimum item in the tree -func (root *Tree) NegativeLimit() Iterator { - return Iterator{root, negativeLimitNode} -} - -// Find the smallest element N such that N >= key, and return the -// iterator pointing to the element. If no such element is found, -// return root.Limit(). -func (root *Tree) FindGE(key Item) Iterator { - n, _ := root.findGE(key) - return Iterator{root, n} -} - -// Find the largest element N such that N <= key, and return the -// iterator pointing to the element. If no such element is found, -// return iter.NegativeLimit(). -func (root *Tree) FindLE(key Item) Iterator { - n, exact := root.findGE(key) - if exact { - return Iterator{root, n} - } - if n != nil { - return Iterator{root, n.doPrev()} - } - if root.maxNode == nil { - return Iterator{root, negativeLimitNode} - } - return Iterator{root, root.maxNode} -} - -// Insert an item. If the item is already in the tree, do nothing and -// return false. Else return true. -func (root *Tree) Insert(item Item) bool { - // TODO: delay creating n until it is found to be inserted - n := root.doInsert(item) - if n == nil { - return false - } - - n.color = red - - for true { - // Case 1: N is at the root - if n.parent == nil { - n.color = black - break - } - - // Case 2: The parent is black, so the tree already - // satisfies the RB properties - if n.parent.color == black { - break - } - - // Case 3: parent and uncle are both red. - // Then paint both black and make grandparent red. - grandparent := n.parent.parent - var uncle *node - if n.parent.isLeftChild() { - uncle = grandparent.right - } else { - uncle = grandparent.left - } - if uncle != nil && uncle.color == red { - n.parent.color = black - uncle.color = black - grandparent.color = red - n = grandparent - continue - } - - // Case 4: parent is red, uncle is black (1) - if n.isRightChild() && n.parent.isLeftChild() { - root.rotateLeft(n.parent) - n = n.left - continue - } - if n.isLeftChild() && n.parent.isRightChild() { - root.rotateRight(n.parent) - n = n.right - continue - } - - // Case 5: parent is read, uncle is black (2) - n.parent.color = black - grandparent.color = red - if n.isLeftChild() { - root.rotateRight(grandparent) - } else { - root.rotateLeft(grandparent) - } - break - } - return true -} - -// Delete an item with the given key. Return true iff the item was -// found. -func (root *Tree) DeleteWithKey(key Item) bool { - iter := root.FindGE(key) - if iter.node != nil { - root.DeleteWithIterator(iter) - return true - } - return false -} - -// Delete the current item. -// -// REQUIRES: !iter.Limit() && !iter.NegativeLimit() -func (root *Tree) DeleteWithIterator(iter Iterator) { - doAssert(!iter.Limit() && !iter.NegativeLimit()) - root.doDelete(iter.node) -} - -// Iterator allows scanning tree elements in sort order. -// -// Iterator invalidation rule is the same as C++ std::map<>'s. That -// is, if you delete the element that an iterator points to, the -// iterator becomes invalid. For other operation types, the iterator -// remains valid. -type Iterator struct { - root *Tree - node *node -} - -func (iter Iterator) Equal(iter2 Iterator) bool { - return iter.node == iter2.node -} - -// Check if the iterator points beyond the max element in the tree -func (iter Iterator) Limit() bool { - return iter.node == nil -} - -// Check if the iterator points to the minimum element in the tree -func (iter Iterator) Min() bool { - return iter.node == iter.root.minNode -} - -// Check if the iterator points to the maximum element in the tree -func (iter Iterator) Max() bool { - return iter.node == iter.root.maxNode -} - -// Check if the iterator points before the minumum element in the tree -func (iter Iterator) NegativeLimit() bool { - return iter.node == negativeLimitNode -} - -// Return the current element. -// -// REQUIRES: !iter.Limit() && !iter.NegativeLimit() -func (iter Iterator) Item() interface{} { - return iter.node.item -} - -// Create a new iterator that points to the successor of the current element. -// -// REQUIRES: !iter.Limit() -func (iter Iterator) Next() Iterator { - doAssert(!iter.Limit()) - if iter.NegativeLimit() { - return Iterator{iter.root, iter.root.minNode} - } - return Iterator{iter.root, iter.node.doNext()} -} - -// Create a new iterator that points to the predecessor of the current -// node. -// -// REQUIRES: !iter.NegativeLimit() -func (iter Iterator) Prev() Iterator { - doAssert(!iter.NegativeLimit()) - if !iter.Limit() { - return Iterator{iter.root, iter.node.doPrev()} - } - if iter.root.maxNode == nil { - return Iterator{iter.root, negativeLimitNode} - } - return Iterator{iter.root, iter.root.maxNode} -} - -func doAssert(b bool) { - if !b { - panic("rbtree internal assertion failed") - } -} - -const red = iota -const black = 1 + iota - -type node struct { - item Item - parent, left, right *node - color int // black or red -} - -var negativeLimitNode *node - -// -// Internal node attribute accessors -// -func getColor(n *node) int { - if n == nil { - return black - } - return n.color -} - -func (n *node) isLeftChild() bool { - return n == n.parent.left -} - -func (n *node) isRightChild() bool { - return n == n.parent.right -} - -func (n *node) sibling() *node { - doAssert(n.parent != nil) - if n.isLeftChild() { - return n.parent.right - } - return n.parent.left -} - -// Return the minimum node that's larger than N. Return nil if no such -// node is found. -func (n *node) doNext() *node { - if n.right != nil { - m := n.right - for m.left != nil { - m = m.left - } - return m - } - - for n != nil { - p := n.parent - if p == nil { - return nil - } - if n.isLeftChild() { - return p - } - n = p - } - return nil -} - -// Return the maximum node that's smaller than N. Return nil if no -// such node is found. -func (n *node) doPrev() *node { - if n.left != nil { - return maxPredecessor(n) - } - - for n != nil { - p := n.parent - if p == nil { - break - } - if n.isRightChild() { - return p - } - n = p - } - return negativeLimitNode -} - -// Return the predecessor of "n". -func maxPredecessor(n *node) *node { - doAssert(n.left != nil) - m := n.left - for m.right != nil { - m = m.right - } - return m -} - -// -// Tree methods -// - -// -// Private methods -// - -func (root *Tree) recomputeMinNode() { - root.minNode = root.root - if root.minNode != nil { - for root.minNode.left != nil { - root.minNode = root.minNode.left - } - } -} - -func (root *Tree) recomputeMaxNode() { - root.maxNode = root.root - if root.maxNode != nil { - for root.maxNode.right != nil { - root.maxNode = root.maxNode.right - } - } -} - -func (root *Tree) maybeSetMinNode(n *node) { - if root.minNode == nil { - root.minNode = n - root.maxNode = n - } else if root.compare(n.item, root.minNode.item) < 0 { - root.minNode = n - } -} - -func (root *Tree) maybeSetMaxNode(n *node) { - if root.maxNode == nil { - root.minNode = n - root.maxNode = n - } else if root.compare(n.item, root.maxNode.item) > 0 { - root.maxNode = n - } -} - -// Try inserting "item" into the tree. Return nil if the item is -// already in the tree. Otherwise return a new (leaf) node. -func (root *Tree) doInsert(item Item) *node { - if root.root == nil { - n := &node{item: item} - root.root = n - root.minNode = n - root.maxNode = n - root.count++ - return n - } - parent := root.root - for true { - comp := root.compare(item, parent.item) - if comp == 0 { - return nil - } else if comp < 0 { - if parent.left == nil { - n := &node{item: item, parent: parent} - parent.left = n - root.count++ - root.maybeSetMinNode(n) - return n - } else { - parent = parent.left - } - } else { - if parent.right == nil { - n := &node{item: item, parent: parent} - parent.right = n - root.count++ - root.maybeSetMaxNode(n) - return n - } else { - parent = parent.right - } - } - } - panic("should not reach here") -} - -// Find a node whose item >= key. The 2nd return value is true iff the -// node.item==key. Returns (nil, false) if all nodes in the tree are < -// key. -func (root *Tree) findGE(key Item) (*node, bool) { - n := root.root - for true { - if n == nil { - return nil, false - } - comp := root.compare(key, n.item) - if comp == 0 { - return n, true - } else if comp < 0 { - if n.left != nil { - n = n.left - } else { - return n, false - } - } else { - if n.right != nil { - n = n.right - } else { - succ := n.doNext() - if succ == nil { - return nil, false - } else { - comp = root.compare(key, succ.item) - return succ, (comp == 0) - } - } - } - } - panic("should not reach here") -} - - -// Delete N from the tree. -func (root *Tree) doDelete(n *node) { - if n.left != nil && n.right != nil { - pred := maxPredecessor(n) - root.swapNodes(n, pred) - } - - doAssert(n.left == nil || n.right == nil) - child := n.right - if child == nil { - child = n.left - } - if n.color == black { - n.color = getColor(child) - root.deleteCase1(n) - } - root.replaceNode(n, child) - if n.parent == nil && child != nil { - child.color = black - } - root.count-- - if root.count == 0 { - root.minNode = nil - root.maxNode = nil - } else { - if root.minNode == n { - root.recomputeMinNode() - } - if root.maxNode == n { - root.recomputeMaxNode() - } - } -} - -// Move n to the pred's place, and vice versa -// -// TODO: this code is overly convoluted -func (root *Tree) swapNodes(n, pred *node) { - doAssert(pred != n) - isLeft := pred.isLeftChild() - tmp := *pred - root.replaceNode(n, pred) - pred.color = n.color - - if tmp.parent == n { - // swap the positions of n and pred - if isLeft { - pred.left = n - pred.right = n.right - if pred.right != nil { - pred.right.parent = pred - } - } else { - pred.left = n.left - if pred.left != nil { - pred.left.parent = pred - } - pred.right = n - } - n.item = tmp.item - n.parent = pred - - n.left = tmp.left - if n.left != nil { - n.left.parent = n - } - n.right = tmp.right - if n.right != nil { - n.right.parent = n - } - } else { - pred.left = n.left - if pred.left != nil { - pred.left.parent = pred - } - pred.right = n.right - if pred.right != nil { - pred.right.parent = pred - } - if isLeft { - tmp.parent.left = n - } else { - tmp.parent.right = n - } - n.item = tmp.item - n.parent = tmp.parent - n.left = tmp.left - if n.left != nil { - n.left.parent = n - } - n.right = tmp.right - if n.right != nil { - n.right.parent = n - } - } - n.color = tmp.color -} - -func (root *Tree) deleteCase1(n *node) { - for true { - if n.parent != nil { - if getColor(n.sibling()) == red { - n.parent.color = red - n.sibling().color = black - if n == n.parent.left { - root.rotateLeft(n.parent) - } else { - root.rotateRight(n.parent) - } - } - if getColor(n.parent) == black && - getColor(n.sibling()) == black && - getColor(n.sibling().left) == black && - getColor(n.sibling().right) == black { - n.sibling().color = red - n = n.parent - continue - } else { - // case 4 - if getColor(n.parent) == red && - getColor(n.sibling()) == black && - getColor(n.sibling().left) == black && - getColor(n.sibling().right) == black { - n.sibling().color = red - n.parent.color = black - } else { - root.deleteCase5(n) - } - } - } - break - } -} - -func (root *Tree) deleteCase5(n *node) { - if n == n.parent.left && - getColor(n.sibling()) == black && - getColor(n.sibling().left) == red && - getColor(n.sibling().right) == black { - n.sibling().color = red - n.sibling().left.color = black - root.rotateRight(n.sibling()) - } else if n == n.parent.right && - getColor(n.sibling()) == black && - getColor(n.sibling().right) == red && - getColor(n.sibling().left) == black { - n.sibling().color = red - n.sibling().right.color = black - root.rotateLeft(n.sibling()) - } - - // case 6 - n.sibling().color = getColor(n.parent) - n.parent.color = black - if n == n.parent.left { - doAssert(getColor(n.sibling().right) == red) - n.sibling().right.color = black - root.rotateLeft(n.parent) - } else { - doAssert(getColor(n.sibling().left) == red) - n.sibling().left.color = black - root.rotateRight(n.parent) - } -} - -func (root *Tree) replaceNode(oldn, newn *node) { - if oldn.parent == nil { - root.root = newn - } else { - if oldn == oldn.parent.left { - oldn.parent.left = newn - } else { - oldn.parent.right = newn - } - } - if newn != nil { - newn.parent = oldn.parent - } -} - -/* - X Y - A Y => X C - B C A B -*/ -func (root *Tree) rotateLeft(x *node) { - y := x.right - x.right = y.left - if y.left != nil { - y.left.parent = x - } - y.parent = x.parent - if x.parent == nil { - root.root = y - } else { - if x.isLeftChild() { - x.parent.left = y - } else { - x.parent.right = y - } - } - y.left = x - x.parent = y -} - -/* - Y X - X C => A Y - A B B C -*/ -func (root *Tree) rotateRight(y *node) { - x := y.left - - // Move "B" - y.left = x.right - if x.right != nil { - x.right.parent = y - } - - x.parent = y.parent - if y.parent == nil { - root.root = x - } else { - if y.isLeftChild() { - y.parent.left = x - } else { - y.parent.right = x - } - } - x.right = y - y.parent = x -} - -func init() { - negativeLimitNode = &node{} -}