From cce3379abf616686914738e975220a42d610c4ae Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 13 Dec 2017 15:07:50 -0600 Subject: [PATCH 01/48] add Containers interface and modify Bitmap to use it there are no implementations of Containers, so everything is broken, but the code compiles --- roaring/roaring.go | 589 +++++++++++++++---------------- roaring/roaring_internal_test.go | 74 ++-- 2 files changed, 323 insertions(+), 340 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index a685e479c..6fa352a12 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -62,10 +62,42 @@ const ( maxContainerVal = 0xffff ) +type Containers interface { + // Get returns nil if the key does not exist. + Get(key uint64) *container + + // Put adds the container at key. + Put(key uint64, c *container) + + // Remove takes the container at key out. + Remove(key uint64) + + // GetOrCreate returns the container at key, creating a new empty container if necessary. + GetOrCreate(key uint64) *container + + // Clone does a deep copy of Containers, including cloning all containers contained. + Clone() Containers + + // Last returns the highest key and associated container. + Last() (key uint64, c *container) + + // Size returns the number of containers stored. + Size() int + + // Iterator returns a Contiterator which after a call to Next(), a call to Value() will + // return the first container at or after key. found will be true if a + // container is found at key. + Iterator(key uint64) (citer Contiterator, found bool) +} + +type Contiterator interface { + Next() bool + Value() (uint64, *container) +} + // Bitmap represents a roaring bitmap. type Bitmap struct { - keys []uint64 // keys for containers - containers []*container // array, bitmap and RLE containers + conts Containers // Number of operations written to the writer. opN int @@ -90,14 +122,7 @@ func (b *Bitmap) Clone() *Bitmap { // Create a copy of the bitmap structure. other := &Bitmap{ - keys: make([]uint64, len(b.keys)), - containers: make([]*container, len(b.containers)), - } - - // Copy keys & clone containers. - copy(other.keys, b.keys) - for i, c := range b.containers { - other.containers[i] = c.clone() + conts: b.conts.Clone(), } return other @@ -127,20 +152,13 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { func (b *Bitmap) add(v uint64) bool { hb := highbits(v) - i := search64(b.keys, hb) - - // If index is negative then there's not an exact match - // and a container needs to be added. - if i < 0 { - b.insertAt(hb, newContainer(), -i-1) - i = -i - 1 - } - return b.containers[i].add(lowbits(v)) + cont := b.conts.GetOrCreate(hb) + return cont.add(lowbits(v)) } // Contains returns true if v is in the bitmap. func (b *Bitmap) Contains(v uint64) bool { - c := b.container(highbits(v)) + c := b.conts.Get(highbits(v)) if c == nil { return false } @@ -168,76 +186,75 @@ func (b *Bitmap) Remove(a ...uint64) (changed bool, err error) { } func (b *Bitmap) remove(v uint64) bool { - hb := highbits(v) - i := search64(b.keys, hb) - if i < 0 { + c := b.conts.Get(highbits(v)) + if c == nil { return false } - return b.containers[i].remove(lowbits(v)) + // TODO - do nil check inside c.remove? + return c.remove(lowbits(v)) } // Max returns the highest value in the bitmap. // Returns zero if the bitmap is empty. func (b *Bitmap) Max() uint64 { - if len(b.keys) == 0 { + if b.conts.Size() == 0 { return 0 } - hb := b.keys[len(b.keys)-1] - lb := b.containers[len(b.containers)-1].max() + hb, c := b.conts.Last() + lb := c.max() return hb<<16 | uint64(lb) } // Count returns the number of bits set in the bitmap. func (b *Bitmap) Count() (n uint64) { - for _, container := range b.containers { - n += uint64(container.n) + citer, _ := b.conts.Iterator(0) + for citer.Next() { + _, c := citer.Value() + n += uint64(c.n) } return n } // CountRange returns the number of bits set between [start, end). func (b *Bitmap) CountRange(start, end uint64) (n uint64) { - if len(b.keys) == 0 { + if b.conts.Size() == 0 { return } skey := highbits(start) ekey := highbits(end) - i := search64(b.keys, skey) - j := search64(b.keys, ekey) - + citer, found := b.conts.Iterator(highbits(start)) // If range is entirely in one container then just count that range. - if i >= 0 && i == j { - return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end)))) + if found && skey == ekey { + citer.Next() + _, c := citer.Value() + return uint64(c.countRange(int(lowbits(start)), int(lowbits(end)))) } - if i < 0 { - // start's container did not exist - // set i to the index of the first container we have with values higher than start - i = -i - 1 - } else { - // Count first partial container and advance i so we don't recount it - n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1)) - i++ + for citer.Next() { + k, c := citer.Value() + if k < skey { + // TODO remove once we've validated this stuff works + panic("should be impossible for k to be less than skey") + } + if k == skey { + n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1)) + continue + } + if k < ekey { + n += uint64(c.n) + continue + } + if k == ekey { + n += uint64(c.countRange(0, int(lowbits(end)))) + break + } + if k > ekey { + break + } } - - // Count last container. - if j < 0 { - // end's container did not exist - // set j to the index of the first container with values higher than end (or len(containers)) - j = -j - 1 - } else { - // end's container exists, count it up to end - n += uint64(b.containers[j].countRange(0, int(lowbits(end)))) - } - - // Count containers in between. - for x := i; x < j; x++ { - n += uint64(b.containers[x].n) - } - return n } @@ -295,44 +312,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) - - // Find starting container. - n := len(b.containers) - i := sort.Search(n, func(i int) bool { return b.keys[i] >= hi0 }) - + citer, _ := b.conts.Iterator(hi0) var other Bitmap - for ; i < n; i++ { - key := b.keys[i] - - // If we've exceeded the upper bound then exit. - if key >= hi1 { + for citer.Next() { + k, c := citer.Value() + if k >= hi1 { break } - - // Otherwise append container with offset key. - other.keys = append(other.keys, off+(key-hi0)) - other.containers = append(other.containers, b.containers[i]) + other.conts.Put(off+(k-hi0), c) } return &other } // container returns the container with the given key. func (b *Bitmap) container(key uint64) *container { - i := search64(b.keys, key) - if i < 0 { - return nil - } - return b.containers[i] -} - -func (b *Bitmap) insertAt(key uint64, c *container, i int) { - b.keys = append(b.keys, 0) - copy(b.keys[i+1:], b.keys[i:]) - b.keys[i] = key - - b.containers = append(b.containers, nil) - copy(b.containers[i+1:], b.containers[i:]) - b.containers[i] = c + return b.conts.Get(key) } // IntersectionCount returns the number of set bits that would result in an @@ -340,15 +334,23 @@ func (b *Bitmap) insertAt(key uint64, c *container, i int) { // intersecting the two and counting the result. func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { var n uint64 - for i, j := 0, 0; i < len(b.containers) && j < len(other.containers); { - ki, kj := b.keys[i], other.keys[j] + iiter, _ := b.conts.Iterator(0) + jiter, _ := b.conts.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i && j { if ki < kj { - i++ + i = iiter.Next() + ki, ci = iiter.Value() } else if ki > kj { - j++ + j = jiter.Next() + kj, cj = jiter.Value() } else { - n += uint64(intersectionCount(b.containers[i], other.containers[j])) - i, j = i+1, j+1 + n += uint64(intersectionCount(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() } } return n @@ -357,30 +359,25 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { output := &Bitmap{} - - ki, ci := b.keys, b.containers - kj, cj := other.keys, other.containers - for { - var key uint64 - var container *container - - ni, nj := len(ki), len(kj) - if ni == 0 && nj == 0 { // eof(i,j) - break - } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j - kj, cj = kj[1:], cj[1:] - } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j - ki, ci = ki[1:], ci[1:] - } else { // i == j - key, container = ki[0], intersect(ci[0], cj[0]) - ki, ci = ki[1:], ci[1:] - kj, cj = kj[1:], cj[1:] - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) + iiter, _ := b.conts.Iterator(0) + jiter, _ := b.conts.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i && j { + if ki < kj { + i = iiter.Next() + ki, ci = iiter.Value() + } else if ki > kj { + j = jiter.Next() + kj, cj = jiter.Value() + } else { // ki == kj + output.conts.Put(ki, intersect(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() } - } - return output } @@ -388,32 +385,27 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { func (b *Bitmap) Union(other *Bitmap) *Bitmap { output := &Bitmap{} - ki, ci := b.keys, b.containers - kj, cj := other.keys, other.containers - - for { - var key uint64 - var container *container - - ni, nj := len(ki), len(kj) - if ni == 0 && nj == 0 { // eof(i,j) - break - } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j - key, container = kj[0], cj[0].clone() - kj, cj = kj[1:], cj[1:] - } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j - key, container = ki[0], ci[0].clone() - ki, ci = ki[1:], ci[1:] - } else { // i == j - key, container = ki[0], union(ci[0], cj[0]) - ki, ci = ki[1:], ci[1:] - kj, cj = kj[1:], cj[1:] + iiter, _ := b.conts.Iterator(0) + jiter, _ := b.conts.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i || j { + if !j || ki < kj { + output.conts.Put(ki, ci.clone()) + i = iiter.Next() + ki, ci = iiter.Value() + } else if !i || ki > kj { + output.conts.Put(kj, cj.clone()) + j = jiter.Next() + kj, cj = jiter.Value() + } else { // ki == kj + output.conts.Put(ki, union(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() } - - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) } - return output } @@ -421,30 +413,24 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { func (b *Bitmap) Difference(other *Bitmap) *Bitmap { output := &Bitmap{} - ki, ci := b.keys, b.containers - kj, cj := other.keys, other.containers - - ni, nj := len(ki), len(kj) - i, j := 0, 0 - for { - var key uint64 - var container *container - - if ni == i { // eof(i) - break - } else if nj == j || ki[i] < kj[j] { // eof(j) or i < j - key, container = ki[i], ci[i].clone() - i++ - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) - } else if nj > j && ki[i] > kj[j] { // i > j - j++ - } else { // i == j - key, container = ki[i], difference(ci[i], cj[j]) - i++ - j++ - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) + iiter, _ := b.conts.Iterator(0) + jiter, _ := b.conts.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i || j { + if !j || ki < kj { + output.conts.Put(ki, ci.clone()) + i = iiter.Next() + ki, ci = iiter.Value() + } else if !i || ki > kj { + j = jiter.Next() + kj, cj = jiter.Value() + } else { // ki == kj + output.conts.Put(ki, difference(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() } } return output @@ -454,68 +440,57 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { func (b *Bitmap) Xor(other *Bitmap) *Bitmap { output := &Bitmap{} - ki, ci := b.keys, b.containers - kj, cj := other.keys, other.containers - - for { - var key uint64 - var container *container - - ni, nj := len(ki), len(kj) - if ni == 0 && nj == 0 { // eof(i,j) - break - } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j - key, container = kj[0], cj[0].clone() - kj, cj = kj[1:], cj[1:] - } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j - key, container = ki[0], ci[0].clone() - ki, ci = ki[1:], ci[1:] - } else { // i == j - key, container = ki[0], xor(ci[0], cj[0]) - ki, ci = ki[1:], ci[1:] - kj, cj = kj[1:], cj[1:] + iiter, _ := b.conts.Iterator(0) + jiter, _ := b.conts.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i || j { + if !j || ki < kj { + output.conts.Put(ki, ci.clone()) + i = iiter.Next() + ki, ci = iiter.Value() + } else if !i || ki > kj { + output.conts.Put(kj, cj.clone()) + j = jiter.Next() + kj, cj = jiter.Value() + } else { // ki == kj + output.conts.Put(ki, xor(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() } - - output.keys = append(output.keys, key) - output.containers = append(output.containers, container) } - return output } // removeEmptyContainers deletes all containers that have a count of zero. func (b *Bitmap) removeEmptyContainers() { - for i := 0; i < len(b.containers); { - c := b.containers[i] - + citer, _ := b.conts.Iterator(0) + for citer.Next() { + k, c := citer.Value() if c.n == 0 { - b.keys = append(b.keys[:i], b.keys[i+1:]...) - - copy(b.containers[i:], b.containers[i+1:]) - b.containers[len(b.containers)-1] = nil - b.containers = b.containers[:len(b.containers)-1] - continue + b.conts.Remove(k) } - - i++ } } func (b *Bitmap) countEmptyContainers() int { result := 0 - for i := 0; i < len(b.containers); { - c := b.containers[i] - + citer, _ := b.conts.Iterator(0) + for citer.Next() { + _, c := citer.Value() if c.n == 0 { result++ } - i++ } return result } // Optimize converts array and bitmap containers to run containers as necessary. func (b *Bitmap) Optimize() { - for _, c := range b.containers { + citer, _ := b.conts.Iterator(0) + for citer.Next() { + _, c := citer.Value() c.Optimize() } } @@ -561,7 +536,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Remove empty containers before persisting. //b.removeEmptyContainers() - containerCount := len(b.keys) - b.countEmptyContainers() + containerCount := b.conts.Size() - b.countEmptyContainers() headerSize := headerBaseSize byte2 := make([]byte, 2) byte4 := make([]byte, 4) @@ -580,9 +555,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Descriptive header section: encode keys and cardinality. // Key and cardinality are stored interleaved here, 12 bytes per container. - for i, key := range b.keys { - c := b.containers[i] - + citer, _ := b.conts.Iterator(0) + for citer.Next() { + key, c := citer.Value() // Verify container count before writing. // TODO: instead of commenting this out, we need to make it a configuration option //count := c.count() @@ -592,17 +567,20 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { ew.WriteUint16(byte2, uint16(c.containerType)) ew.WriteUint16(byte2, uint16(c.n-1)) } + } // Offset header section: write the offset for each container block. // 4 bytes per container. offset := uint32(headerSize + (containerCount * (8 + 2 + 2 + 4))) - for _, c := range b.containers { - + citer, _ = b.conts.Iterator(0) + for citer.Next() { + _, c := citer.Value() if c.n > 0 { ew.WriteUint32(byte4, offset) offset += uint32(c.size()) } + } if ew.err != nil { return int64(ew.n), ew.err @@ -611,7 +589,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { n = int64(headerSize + (containerCount * (8 + 2 + 2 + 4))) // Container storage section: write each container block. - for _, c := range b.containers { + citer, _ = b.conts.Iterator(0) + for citer.Next() { + _, c := citer.Value() if c.n > 0 { nn, err := c.WriteTo(w) n += nn @@ -620,7 +600,6 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } } } - return n, nil } @@ -644,42 +623,20 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[4:8]) - if len(b.keys) == 0 { - b.keys = make([]uint64, 0, keyN) - b.containers = make([]*container, 0, keyN) - } else if int(keyN) < len(b.keys) { //shrink - // nil out to allow to be GCed - for i := range b.containers[keyN:] { - b.containers[int(keyN)+i] = nil - } - b.keys = b.keys[:keyN] - b.containers = b.containers[:keyN] - } - headerSize := headerBaseSize // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { - // Reuse memory if possible - if i >= len(b.keys) { - b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8])) - b.containers = append(b.containers, &container{ - containerType: byte(binary.LittleEndian.Uint16(buf[8:10])), - n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, - mapped: true, - }) - } else { - b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) - c := b.containers[i] - c.containerType = byte(binary.LittleEndian.Uint16(buf[8:10])) - c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1 - c.mapped = true - - } + b.conts.Put(binary.LittleEndian.Uint64(buf[0:8]), &container{ + containerType: byte(binary.LittleEndian.Uint16(buf[8:10])), + n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, + mapped: true, + }) } opsOffset := headerSize + int(keyN)*12 // Read container offsets and attach data. + citer, _ := b.conts.Iterator(0) for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { offset := binary.LittleEndian.Uint32(buf[0:4]) // Verify the offset is within the bounds of the input data. @@ -688,7 +645,8 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { } // Map byte slice directly to the container data. - c := b.containers[i] + citer.Next() + _, c := citer.Value() switch c.containerType { case ContainerRun: c.array = nil @@ -760,15 +718,16 @@ func (b *Bitmap) Iterator() *Iterator { func (b *Bitmap) Info() BitmapInfo { info := BitmapInfo{ OpN: b.opN, - Containers: make([]ContainerInfo, len(b.containers)), + Containers: make([]ContainerInfo, 0, b.conts.Size()), } - for i, c := range b.containers { + citer, _ := b.conts.Iterator(0) + for citer.Next() { + k, c := citer.Value() ci := c.info() - ci.Key = b.keys[i] - info.Containers[i] = ci + ci.Key = k + info.Containers = append(info.Containers, ci) } - return info } @@ -776,16 +735,12 @@ func (b *Bitmap) Info() BitmapInfo { func (b *Bitmap) Check() error { var a ErrorList - // Check keys/containers match. Return immediately if this happens. - if len(b.keys) != len(b.containers) { - a.Append(fmt.Errorf("key/container count mismatch: %d != %d", len(b.keys), len(b.containers))) - return a - } - // Check each container. - for i, c := range b.containers { + citer, _ := b.conts.Iterator(0) + for citer.Next() { + k, c := citer.Value() if err := c.check(); err != nil { - a.AppendWithPrefix(err, fmt.Sprintf("%d/", b.keys[i])) + a.AppendWithPrefix(err, fmt.Sprintf("%d/", k)) } } @@ -831,13 +786,13 @@ type BitmapInfo struct { // Iterator represents an iterator over a Bitmap. type Iterator struct { - bitmap *Bitmap - i, j, k int // i: container; j: array index, bit index, or run index; k: offset within the run + bitmap *Bitmap + citer Contiterator + key uint64 + c *container + j, k int // i: container; j: array index, bit index, or run index; k: offset within the run } -// eof returns true if the iterator is at the end of the bitmap. -func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) } - // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { // k should always be -1 unless we're seeking into a run container. Then the @@ -845,45 +800,45 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.i = search64(itr.bitmap.keys, highbits(seek)) - if itr.i < 0 { - itr.i = -itr.i - 1 - } - if itr.eof() { - return + itr.citer, _ = itr.bitmap.conts.Iterator(seek) + if !itr.citer.Next() { + itr.c = nil + return // eof } + itr.key, itr.c = itr.citer.Value() // Move to the correct value index inside the container. lb := lowbits(seek) - if itr.i >= len(itr.bitmap.containers) { - panic(fmt.Sprintf("data Corruption %d %d %d", itr.i, len(itr.bitmap.containers), seek)) - } - c := itr.bitmap.containers[itr.i] - if c.isArray() { + if itr.c.isArray() { // Find index in the container. - itr.j = search32(c.array, lb) + itr.j = search32(itr.c.array, lb) if itr.j < 0 { itr.j = -itr.j - 1 } - if itr.j < len(c.array) { + if itr.j < len(itr.c.array) { itr.j-- return } // If it's at the end of the container then move to the next one. - itr.i, itr.j = itr.i+1, -1 + if !itr.citer.Next() { + itr.c = nil + return + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 return } - if c.isRun() { + if itr.c.isRun() { if seek == 0 { - itr.i, itr.j, itr.k = 0, 0, -1 + itr.j, itr.k = 0, -1 } - j, contains := binSearchRuns(lb, c.runs) + j, contains := binSearchRuns(lb, itr.c.runs) if contains { itr.j = j - itr.k = int(lb) - int(c.runs[j].start) - 1 + itr.k = int(lb) - int(itr.c.runs[j].start) - 1 } else { // Set iterator to next value in the Bitmap. itr.j = j @@ -900,24 +855,27 @@ func (itr *Iterator) Seek(seek uint64) { // Next returns the next value in the bitmap. // Returns eof as true if there are no values left in the iterator. func (itr *Iterator) Next() (v uint64, eof bool) { + if itr.c == nil { + return + } // Iterate over containers until we find the next value or EOF. for { - if itr.eof() { - return 0, true - } - - c := itr.bitmap.containers[itr.i] - if c.isArray() { - if itr.j >= c.n-1 { + if itr.c.isArray() { + if itr.j >= itr.c.n-1 { // Reached end of array, move to the next container. - itr.i, itr.j = itr.i+1, -1 + if !itr.citer.Next() { + itr.c = nil + return 0, true + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 continue } itr.j++ return itr.peek(), false } - if c.isRun() { + if itr.c.isRun() { // Because itr.j for an array container defaults to -1 // but defaults to 0 for a run container, we need to // standardize on treating -1 as our default value for itr.j. @@ -930,12 +888,17 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } // If the container is empty, move to the next container. - if len(c.runs) == 0 { - itr.i, itr.j = itr.i+1, -1 + if len(itr.c.runs) == 0 { + if !itr.citer.Next() { + itr.c = nil + return 0, true + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 continue } - r := c.runs[itr.j] + r := itr.c.runs[itr.j] runLength := int(r.last - r.start) if itr.k >= runLength { @@ -943,9 +906,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { itr.j, itr.k = itr.j+1, -1 } - if itr.j >= len(c.runs) { + if itr.j >= len(itr.c.runs) { // Reached end of runs, move to the next container. - itr.i, itr.j = itr.i+1, -1 + if !itr.citer.Next() { + itr.c = nil + return 0, true + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 continue } @@ -959,40 +927,51 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Find first non-zero bit in current bitmap, if possible. hb := itr.j >> 6 - if hb >= len(c.bitmap) { - itr.i, itr.j = itr.i+1, -1 + if hb >= len(itr.c.bitmap) { + if !itr.citer.Next() { + itr.c = nil + return 0, true + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 continue } - lb := c.bitmap[hb] >> (uint(itr.j) % 64) + lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64) if lb != 0 { itr.j = itr.j + trailingZeroN(lb) return itr.peek(), false } // Otherwise iterate through remaining bitmaps to find next bit. - for hb++; hb < len(c.bitmap); hb++ { - if c.bitmap[hb] != 0 { - itr.j = hb<<6 + trailingZeroN(c.bitmap[hb]) + for hb++; hb < len(itr.c.bitmap); hb++ { + if itr.c.bitmap[hb] != 0 { + itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb]) return itr.peek(), false } } // If no bits found then move to the next container. - itr.i, itr.j = itr.i+1, -1 + if !itr.citer.Next() { + itr.c = nil + return 0, true + } + itr.key, itr.c = itr.citer.Value() + itr.j = -1 } } // peek returns the current value. func (itr *Iterator) peek() uint64 { - key := itr.bitmap.keys[itr.i] - c := itr.bitmap.containers[itr.i] - if c.isArray() { - return key<<16 | uint64(c.array[itr.j]) + if itr.c == nil { + return 0 } - if c.isRun() { - return key<<16 | uint64(c.runs[itr.j].start+uint16(itr.k)) + if itr.c.isArray() { + return itr.key<<16 | uint64(itr.c.array[itr.j]) } - return key<<16 | uint64(itr.j) + if itr.c.isRun() { + return itr.key<<16 | uint64(itr.c.runs[itr.j].start+uint16(itr.k)) + } + return itr.key<<16 | uint64(itr.j) } // ArrayMaxSize represents the maximum size of array containers. diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a2fbb7d64..268e02de4 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1714,7 +1714,8 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} - ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}} + ba := &Bitmap{} + ba.conts.Put(0, ca) ba2 := &Bitmap{} var buf bytes.Buffer _, err := ba.WriteTo(&buf) @@ -1725,8 +1726,8 @@ func TestWriteReadArray(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(ba2.containers[0].array, ca.array) { - t.Fatalf("array test expected %x, but got %x", ca.array, ba2.containers[0].array) + if !reflect.DeepEqual(ba2.conts.Get(0).array, ca.array) { + t.Fatalf("array test expected %x, but got %x", ca.array, ba2.conts.Get(0).array) } } @@ -1736,7 +1737,8 @@ func TestWriteReadBitmap(t *testing.T) { for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } - bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}} + bb := &Bitmap{} + bb.conts.Put(0, cb) bb2 := &Bitmap{} var buf bytes.Buffer _, err := bb.WriteTo(&buf) @@ -1747,8 +1749,8 @@ func TestWriteReadBitmap(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap) + if !reflect.DeepEqual(bb2.conts.Get(0).bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.conts.Get(0).bitmap) } } @@ -1758,7 +1760,8 @@ func TestWriteReadFullBitmap(t *testing.T) { for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } - bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}} + bb := &Bitmap{} + bb.conts.Put(0, cb) bb2 := &Bitmap{} var buf bytes.Buffer _, err := bb.WriteTo(&buf) @@ -1769,21 +1772,22 @@ func TestWriteReadFullBitmap(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap) + if !reflect.DeepEqual(bb2.conts.Get(0).bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.conts.Get(0).bitmap) } - if bb2.containers[0].n != cb.n { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n) + if bb2.conts.Get(0).n != cb.n { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.conts.Get(0).n) } - if bb2.containers[0].count() != cb.count() { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n) + if bb2.conts.Get(0).count() != cb.count() { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.conts.Get(0).n) } } func TestWriteReadRun(t *testing.T) { cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} - br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}} + br := &Bitmap{} + br.conts.Put(0, cr) br2 := &Bitmap{} var buf bytes.Buffer _, err := br.WriteTo(&buf) @@ -1794,8 +1798,8 @@ func TestWriteReadRun(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(br2.containers[0].runs, cr.runs) { - t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs) + if !reflect.DeepEqual(br2.conts.Get(0).runs, cr.runs) { + t.Fatalf("run test expected %x, but got %x", cr.runs, br2.conts.Get(0).runs) } } @@ -2086,34 +2090,34 @@ func TestXorBitmapRun(t *testing.T) { func TestIteratorArray(t *testing.T) { // use values that span two containers b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) - if !b.containers[0].isArray() { + if !b.conts.Get(0).isArray() { t.Fatalf("wrong container type") } itr := b.Iterator() - if !(itr.i == 0 && itr.j == -1) { + if !(itr.key == 0 && itr.j == -1) { t.Fatalf("iterator did not zero correctly: %v\n", itr) } itr.Seek(1000) - if !(itr.i == 0 && itr.j == 3) { + if !(itr.key == 0 && itr.j == 3) { t.Fatalf("iterator did not seek correctly: %v\n", itr) } itr.Seek(10000) itr.Next() val, eof := itr.Next() - if !(itr.i == 1 && itr.j == 0 && val == 90000 && !eof) { + if !(itr.key == 1 && itr.j == 0 && val == 90000 && !eof) { t.Fatalf("iterator did not next correctly across containers: %v\n", itr) } itr.Seek(80000) - if !(itr.i == 1 && itr.j == -1) { + if !(itr.key == 1 && itr.j == -1) { t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) } itr.Seek(100000) - if !(itr.i == 1 && itr.j == 0) { + if !(itr.key == 1 && itr.j == 0) { t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) } @@ -2139,34 +2143,34 @@ func TestIteratorBitmap(t *testing.T) { for i := uint64(75000); i < 75100; i++ { b.Add(i) } - if !b.containers[0].isBitmap() { + if !b.conts.Get(0).isBitmap() { t.Fatalf("wrong container type") } itr := b.Iterator() - if !(itr.i == 0 && itr.j == -1) { + if !(itr.key == 0 && itr.j == -1) { t.Fatalf("iterator did not zero correctly: %v\n", itr) } itr.Seek(65000) - if !(itr.i == 0 && itr.j == 64999) { + if !(itr.key == 0 && itr.j == 64999) { t.Fatalf("iterator did not seek correctly: %v\n", itr) } itr.Seek(65535) itr.Next() val, eof := itr.Next() - if !(itr.i == 1 && itr.j == 0 && val == 65536 && !eof) { + if !(itr.key == 1 && itr.j == 0 && val == 65536 && !eof) { t.Fatalf("iterator did not next correctly across containers: %v\n", itr) } itr.Seek(74000) - if !(itr.i == 1 && itr.j == 8463) { + if !(itr.key == 1 && itr.j == 8463) { t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) } itr.Seek(70999) - if !(itr.i == 1 && itr.j == 5462) { + if !(itr.key == 1 && itr.j == 5462) { t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) } @@ -2185,17 +2189,17 @@ func TestIteratorBitmap(t *testing.T) { func TestIteratorRuns(t *testing.T) { b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() - if !b.containers[0].isRun() { + if !b.conts.Get(0).isRun() { t.Fatalf("wrong container type") } itr := b.Iterator() - if !(itr.i == 0 && itr.j == 0 && itr.k == -1) { + if !(itr.key == 0 && itr.j == 0 && itr.k == -1) { t.Fatalf("iterator did not zero correctly: %v\n", itr) } itr.Seek(4) - if !(itr.i == 0 && itr.j == 0 && itr.k == 3) { + if !(itr.key == 0 && itr.j == 0 && itr.k == 3) { t.Fatalf("iterator did not seek correctly: %v\n", itr) } itr.Next() @@ -2218,22 +2222,22 @@ func TestIteratorRuns(t *testing.T) { } itr.Seek(500) - if !(itr.i == 0 && itr.j == 1 && itr.k == -1) { + if !(itr.key == 0 && itr.j == 1 && itr.k == -1) { t.Fatalf("iterator did not seek missing value correctly: %v\n", itr) } itr.Seek(1004) - if !(itr.i == 0 && itr.j == 1 && itr.k == 3) { + if !(itr.key == 0 && itr.j == 1 && itr.k == 3) { t.Fatalf("iterator did not seek correctly in multiple runs: %v\n", itr) } itr.Seek(1005) - if !(itr.i == 0 && itr.j == 1 && itr.k == 4) { + if !(itr.key == 0 && itr.j == 1 && itr.k == 4) { t.Fatalf("iterator did not seek correctly to end of run: %v\n", itr) } itr.Seek(100005) - if !(itr.i == 1 && itr.j == 0 && itr.k == 4) { + if !(itr.key == 1 && itr.j == 0 && itr.k == 4) { t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr) } From bc6fb2627ffe11ad991ccf5e5aac03fe3f40d7bb Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 13 Dec 2017 17:04:04 -0600 Subject: [PATCH 02/48] add initial skip list Containers impl --- roaring/containers.go | 85 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 roaring/containers.go diff --git a/roaring/containers.go b/roaring/containers.go new file mode 100644 index 000000000..467430c74 --- /dev/null +++ b/roaring/containers.go @@ -0,0 +1,85 @@ +package roaring + +import ( + "github.com/pilosa/fast-skiplist" +) + +func NewSkipListContainers() *SkipListContainers { + return &SkipListContainers{ + list: skiplist.New(), + } +} + +type SkipListContainers struct { + list *skiplist.SkipList +} + +func (slc *SkipListContainers) Get(key uint64) *container { + return slc.list.Get(key).Value().(*container) +} + +func (slc *SkipListContainers) Put(key uint64, c *container) { + slc.list.Set(key, c) +} + +func (slc *SkipListContainers) Remove(key uint64) { + slc.list.Remove(key) +} + +func (slc *SkipListContainers) GetOrCreate(key uint64) *container { + el := slc.list.Get(key) + if el == nil { + return slc.list.Set(key, newContainer()).Value().(*container) + } + return el.Value().(*container) +} + +func (slc *SkipListContainers) Clone() Containers { + nslc := NewSkipListContainers() + for c := slc.list.Front(); c != nil; c = c.Next() { + nslc.list.Set(c.Key(), c.Value().(*container).clone()) + } + return nslc +} + +func (slc *SkipListContainers) Last() (key uint64, c *container) { + el := slc.list.Last() + return el.Key(), el.Value().(*container) +} + +func (slc *SkipListContainers) Size() int { + return slc.list.Length() +} + +func (slc *SkipListContainers) Iterator(key uint64) (citer Contiterator, found bool) { + el := slc.list.GetNext(key) + if el.Key() == key { + found = true + } + + return &SLCIterator{ + el: el, + }, found + +} + +type SLCIterator struct { + started bool + el *skiplist.Element +} + +func (i *SLCIterator) Next() bool { + if i.el == nil { + return false + } + if !i.started { + i.started = true + return true + } + i.el = i.el.Next() + return i.el != nil +} + +func (i *SLCIterator) Value() (uint64, *container) { + return i.el.Key(), i.el.Value().(*container) +} From d7989637983d05f3a1602077aafc10a796f945ad Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 18 Dec 2017 14:40:09 -0600 Subject: [PATCH 03/48] fix a bunch of bugs and add some tests --- roaring/containers.go | 20 +++++--- roaring/containers_test.go | 84 ++++++++++++++++++++++++++++++++ roaring/roaring.go | 75 ++++++++++++++++++++-------- roaring/roaring_internal_test.go | 18 +++---- roaring/roaring_test.go | 6 +-- 5 files changed, 166 insertions(+), 37 deletions(-) create mode 100644 roaring/containers_test.go diff --git a/roaring/containers.go b/roaring/containers.go index 467430c74..c80dbeb06 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -1,8 +1,6 @@ package roaring -import ( - "github.com/pilosa/fast-skiplist" -) +import "github.com/pilosa/fast-skiplist" func NewSkipListContainers() *SkipListContainers { return &SkipListContainers{ @@ -15,7 +13,12 @@ type SkipListContainers struct { } func (slc *SkipListContainers) Get(key uint64) *container { - return slc.list.Get(key).Value().(*container) + var c *container + el := slc.list.Get(key) + if el != nil { + c = el.Value().(*container) + } + return c } func (slc *SkipListContainers) Put(key uint64, c *container) { @@ -43,6 +46,9 @@ func (slc *SkipListContainers) Clone() Containers { } func (slc *SkipListContainers) Last() (key uint64, c *container) { + if slc.list.Length() == 0 { + return 0, nil + } el := slc.list.Last() return el.Key(), el.Value().(*container) } @@ -53,14 +59,13 @@ func (slc *SkipListContainers) Size() int { func (slc *SkipListContainers) Iterator(key uint64) (citer Contiterator, found bool) { el := slc.list.GetNext(key) - if el.Key() == key { + if el != nil && el.Key() == key { found = true } return &SLCIterator{ el: el, }, found - } type SLCIterator struct { @@ -81,5 +86,8 @@ func (i *SLCIterator) Next() bool { } func (i *SLCIterator) Value() (uint64, *container) { + if !i.started || i.el == nil { + return 0, nil + } return i.el.Key(), i.el.Value().(*container) } diff --git a/roaring/containers_test.go b/roaring/containers_test.go new file mode 100644 index 000000000..464a92fd6 --- /dev/null +++ b/roaring/containers_test.go @@ -0,0 +1,84 @@ +package roaring + +import ( + "testing" +) + +func TestContainersIterator(t *testing.T) { + slc := NewSkipListContainers() + itr, found := slc.Iterator(0) + if found { + t.Fatalf("shouldn't have found 0 in empty slc") + } + if itr.Next() { + t.Fatal("Next() should be false for empty slc") + } + + slc.Put(1, &container{n: 1}) + slc.Put(2, &container{n: 2}) + + itr, found = slc.Iterator(0) + if found { + t.Fatalf("shouldn't have found 0") + } + + if !itr.Next() { + t.Fatalf("one should be next, but got false") + } + if key, val := itr.Value(); key != 1 || val.n != 1 { + t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.n) + } + if !itr.Next() { + t.Fatalf("two should be next, but got false") + } + if key, val := itr.Value(); key != 2 || val.n != 2 { + t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.n) + } + + if itr.Next() { + t.Fatalf("itr should be done, but got true") + } + + slc.Put(3, &container{n: 3}) + slc.Put(5, &container{n: 5}) + slc.Put(6, &container{n: 6}) + + itr, found = slc.Iterator(3) + if !itr.Next() { + t.Fatalf("3 should be next, but got false") + } + if !found { + t.Fatalf("should have found 3") + } + if key, val := itr.Value(); key != 3 || val.n != 3 { + t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.n) + } + if !itr.Next() { + t.Fatalf("5 should be next, but got false") + } + if key, val := itr.Value(); key != 5 || val.n != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + } + + itr, found = slc.Iterator(4) + if found { + t.Fatalf("shouldn't have found 4") + } + if !itr.Next() { + t.Fatalf("5 should be next, but got false") + } + if key, val := itr.Value(); key != 5 || val.n != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + } + if !itr.Next() { + t.Fatalf("6 should be next, but got false") + } + if key, val := itr.Value(); key != 6 || val.n != 6 { + t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.n) + } + + if itr.Next() { + t.Fatalf("itr should be done, but got true") + } + +} diff --git a/roaring/roaring.go b/roaring/roaring.go index 6fa352a12..2de14caf3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -21,6 +21,7 @@ import ( "fmt" "hash/fnv" "io" + "reflect" "sort" "unsafe" ) @@ -108,7 +109,9 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { - b := &Bitmap{} + b := &Bitmap{ + conts: NewSkipListContainers(), + } b.Add(a...) return b } @@ -263,6 +266,7 @@ func (b *Bitmap) Slice() []uint64 { var a []uint64 itr := b.Iterator() itr.Seek(0) + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { a = append(a, v) } @@ -313,7 +317,7 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) citer, _ := b.conts.Iterator(hi0) - var other Bitmap + other := NewBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { @@ -321,7 +325,7 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { } other.conts.Put(off+(k-hi0), c) } - return &other + return other } // container returns the container with the given key. @@ -335,7 +339,7 @@ func (b *Bitmap) container(key uint64) *container { func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { var n uint64 iiter, _ := b.conts.Iterator(0) - jiter, _ := b.conts.Iterator(0) + jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() @@ -358,9 +362,14 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { +<<<<<<< 49aec2dc4c2137c459c5ff4902609b93cfc72933 output := &Bitmap{} +======= + output := NewBitmap() + +>>>>>>> fix a bunch of bugs and add some tests iiter, _ := b.conts.Iterator(0) - jiter, _ := b.conts.Iterator(0) + jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() @@ -383,19 +392,19 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { - output := &Bitmap{} + output := NewBitmap() iiter, _ := b.conts.Iterator(0) - jiter, _ := b.conts.Iterator(0) + jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { - if !j || ki < kj { + if i && (!j || ki < kj) { output.conts.Put(ki, ci.clone()) i = iiter.Next() ki, ci = iiter.Value() - } else if !i || ki > kj { + } else if j && (!i || ki > kj) { output.conts.Put(kj, cj.clone()) j = jiter.Next() kj, cj = jiter.Value() @@ -411,19 +420,19 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { // Difference returns the difference of b and other. func (b *Bitmap) Difference(other *Bitmap) *Bitmap { - output := &Bitmap{} + output := NewBitmap() iiter, _ := b.conts.Iterator(0) - jiter, _ := b.conts.Iterator(0) + jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { - if !j || ki < kj { + if i && (!j || ki < kj) { output.conts.Put(ki, ci.clone()) i = iiter.Next() ki, ci = iiter.Value() - } else if !i || ki > kj { + } else if j && (!i || ki > kj) { j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -438,19 +447,19 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { // Xor returns the bitwise exclusive or of b and other. func (b *Bitmap) Xor(other *Bitmap) *Bitmap { - output := &Bitmap{} + output := NewBitmap() iiter, _ := b.conts.Iterator(0) - jiter, _ := b.conts.Iterator(0) + jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { - if !j || ki < kj { + if i && (!j || ki < kj) { output.conts.Put(ki, ci.clone()) i = iiter.Next() ki, ci = iiter.Value() - } else if !i || ki > kj { + } else if j && (!i || ki > kj) { output.conts.Put(kj, cj.clone()) j = jiter.Next() kj, cj = jiter.Value() @@ -800,7 +809,7 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.citer, _ = itr.bitmap.conts.Iterator(seek) + itr.citer, _ = itr.bitmap.conts.Iterator(highbits(seek)) if !itr.citer.Next() { itr.c = nil return // eof @@ -856,7 +865,7 @@ func (itr *Iterator) Seek(seek uint64) { // Returns eof as true if there are no values left in the iterator. func (itr *Iterator) Next() (v uint64, eof bool) { if itr.c == nil { - return + return 0, true } // Iterate over containers until we find the next value or EOF. for { @@ -3223,3 +3232,31 @@ func xorBitmapRun(a, b *container) *container { } return output } + +func BitmapsEqual(b, c *Bitmap) error { + if b.OpWriter != c.OpWriter { + return errors.New("opWriters not equal") + } + if b.opN != c.opN { + return errors.New("opNs not equal") + } + + biter, _ := b.conts.Iterator(0) + citer, _ := c.conts.Iterator(0) + bn, cn := biter.Next(), citer.Next() + for ; bn && cn; bn, cn = biter.Next(), citer.Next() { + bk, bc := biter.Value() + ck, cc := citer.Value() + if bk != ck { + return errors.New("keys not equal") + } + if !reflect.DeepEqual(bc, cc) { + return errors.New("containers not equal") + } + } + if bn && !cn || cn && !bn { + return errors.New("different numbers of containers") + } + + return nil +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 268e02de4..7503ab247 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1714,9 +1714,9 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} - ba := &Bitmap{} + ba := NewBitmap() ba.conts.Put(0, ca) - ba2 := &Bitmap{} + ba2 := NewBitmap() var buf bytes.Buffer _, err := ba.WriteTo(&buf) if err != nil { @@ -1737,9 +1737,9 @@ func TestWriteReadBitmap(t *testing.T) { for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } - bb := &Bitmap{} + bb := NewBitmap() bb.conts.Put(0, cb) - bb2 := &Bitmap{} + bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1760,9 +1760,9 @@ func TestWriteReadFullBitmap(t *testing.T) { for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } - bb := &Bitmap{} + bb := NewBitmap() bb.conts.Put(0, cb) - bb2 := &Bitmap{} + bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1786,9 +1786,9 @@ func TestWriteReadFullBitmap(t *testing.T) { func TestWriteReadRun(t *testing.T) { cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} - br := &Bitmap{} + br := NewBitmap() br.conts.Put(0, cr) - br2 := &Bitmap{} + br2 := NewBitmap() var buf bytes.Buffer _, err := br.WriteTo(&buf) if err != nil { @@ -2101,7 +2101,7 @@ func TestIteratorArray(t *testing.T) { itr.Seek(1000) if !(itr.key == 0 && itr.j == 3) { - t.Fatalf("iterator did not seek correctly: %v\n", itr) + t.Fatalf("iterator did not seek correctly: %#v\n", itr) } itr.Seek(10000) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 6378ee67f..f972d8573 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -35,8 +35,8 @@ func TestBitmapClone(t *testing.T) { b.Add(i) } c := b.Clone() - if !reflect.DeepEqual(b, c) { - t.Fatalf("Clone Objects not equal\n") + if err := roaring.BitmapsEqual(b, c); err != nil { + t.Fatalf("Clone Objects not equal: %v\n", err) } d := func() *roaring.Bitmap { //anybody know how to declare a nil value? return nil @@ -363,7 +363,7 @@ func TestBitmap_RunCountRange(t *testing.T) { } } -func TestBitmap_Intersection(t *testing.T) { +func TestBitmap_Intersectionz(t *testing.T) { bm0 := roaring.NewBitmap(0, 2683177) bm1 := roaring.NewBitmap() for i := uint64(628); i < 2683301; i++ { From a42a1a2c37ea5fc4a33849c28fc58996bab5f3d0 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 18 Dec 2017 15:07:28 -0600 Subject: [PATCH 04/48] make sure roaring.Bitmaps are created appropriately --- bitmap.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bitmap.go b/bitmap.go index 72296efab..bc2f260d4 100644 --- a/bitmap.go +++ b/bitmap.go @@ -189,11 +189,12 @@ func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment { } // Insert new segment. - b.segments = append(b.segments, BitmapSegment{}) + b.segments = append(b.segments, BitmapSegment{data: *roaring.NewBitmap()}) if i < len(b.segments) { copy(b.segments[i+1:], b.segments[i:]) } b.segments[i] = BitmapSegment{ + data: *roaring.NewBitmap(), slice: slice, writable: true, } From be4696ebb9f7d6621a78829bc956422f4f4c12e5 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 20 Dec 2017 09:02:03 -0600 Subject: [PATCH 05/48] remove interface --- roaring/containers.go | 2 +- roaring/roaring.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index c80dbeb06..faa531dd9 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -37,7 +37,7 @@ func (slc *SkipListContainers) GetOrCreate(key uint64) *container { return el.Value().(*container) } -func (slc *SkipListContainers) Clone() Containers { +func (slc *SkipListContainers) Clone() *SkipListContainers { nslc := NewSkipListContainers() for c := slc.list.Front(); c != nil; c = c.Next() { nslc.list.Set(c.Key(), c.Value().(*container).clone()) diff --git a/roaring/roaring.go b/roaring/roaring.go index 2de14caf3..62e4b7935 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,7 +98,7 @@ type Contiterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { - conts Containers + conts *SkipListContainers // Number of operations written to the writer. opN int From 5314d61086e70884d0c123cf163a8fe53fc8811f Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 20 Dec 2017 09:27:44 -0600 Subject: [PATCH 06/48] Revert "remove interface" This reverts commit 343e810bc65562d10d9408c941e65414c6945d9f. --- roaring/containers.go | 2 +- roaring/roaring.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index faa531dd9..c80dbeb06 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -37,7 +37,7 @@ func (slc *SkipListContainers) GetOrCreate(key uint64) *container { return el.Value().(*container) } -func (slc *SkipListContainers) Clone() *SkipListContainers { +func (slc *SkipListContainers) Clone() Containers { nslc := NewSkipListContainers() for c := slc.list.Front(); c != nil; c = c.Next() { nslc.list.Set(c.Key(), c.Value().(*container).clone()) diff --git a/roaring/roaring.go b/roaring/roaring.go index 62e4b7935..2de14caf3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,7 +98,7 @@ type Contiterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { - conts *SkipListContainers + conts Containers // Number of operations written to the writer. opN int From fab493c42b30873d55d8afce7289a0f92d861f2d Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 22 Dec 2017 11:14:12 -0600 Subject: [PATCH 07/48] fix up missed conflict --- roaring/roaring.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 2de14caf3..367983322 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -362,12 +362,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { -<<<<<<< 49aec2dc4c2137c459c5ff4902609b93cfc72933 - output := &Bitmap{} -======= output := NewBitmap() - ->>>>>>> fix a bunch of bugs and add some tests iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() From ce052c133c1c39e76a53b15a24da5c6ebc8daf7d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 12 Jan 2018 13:27:16 -0600 Subject: [PATCH 08/48] b+tree for Containers interface --- roaring/containers_btree.go | 125 ++++++++++++++++++ .../{containers.go => containers_skiplist.go} | 0 roaring/containers_test.go | 24 ++-- roaring/roaring.go | 2 +- 4 files changed, 138 insertions(+), 13 deletions(-) create mode 100644 roaring/containers_btree.go rename roaring/{containers.go => containers_skiplist.go} (100%) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go new file mode 100644 index 000000000..ce78c278a --- /dev/null +++ b/roaring/containers_btree.go @@ -0,0 +1,125 @@ +package roaring + +import ( + "io" + + btree "github.com/pilosa/b" +) + +func cmp(a, b uint64) int { + return int(a - b) +} + +func NewBTreeContainers() *BTreeContainers { + return &BTreeContainers{ + tree: btree.TreeNew(cmp), + } +} + +type BTreeContainers struct { + tree *btree.Tree + + lastKey uint64 + lastContainer *container +} + +func (btc *BTreeContainers) Get(key uint64) *container { + var c *container + el, ok := btc.tree.Get(key) + if ok { + c = el.(*container) + } + return c +} + +func (btc *BTreeContainers) Put(key uint64, c *container) { + btc.tree.Set(key, c) +} + +func (btc *BTreeContainers) Remove(key uint64) { + btc.tree.Delete(key) +} + +func (btc *BTreeContainers) GetOrCreate(key uint64) *container { + // Check the last* cache for same container. + if key == btc.lastKey && btc.lastContainer != nil { + return btc.lastContainer + } + + btc.lastKey = key + v, ok := btc.tree.Get(key) + if !ok { + cont := newContainer() + btc.tree.Set(key, cont) + btc.lastContainer = cont + return cont + } + + btc.lastContainer = v.(*container) + return btc.lastContainer +} + +func (btc *BTreeContainers) Clone() Containers { + nbtc := NewBTreeContainers() + + itr, err := btc.tree.SeekFirst() + if err == io.EOF { + return nbtc + } + for { + k, v, err := itr.Next() + if err == io.EOF { + break + } + nbtc.tree.Set(k, v.(*container).clone()) + } + + return nbtc +} + +func (btc *BTreeContainers) Last() (key uint64, c *container) { + if btc.tree.Len() == 0 { + return 0, nil + } + k, v := btc.tree.Last() + return k, v.(*container) +} + +func (btc *BTreeContainers) Size() int { + return btc.tree.Len() +} + +func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool) { + e, ok := btc.tree.Seek(key) + if ok { + found = true + } + + return &BTCIterator{ + e: e, + }, found +} + +type BTCIterator struct { + e *btree.Enumerator + key interface{} + val interface{} +} + +func (i *BTCIterator) Next() bool { + + k, v, err := i.e.Next() + if err == io.EOF { + return false + } + i.key = k + i.val = v + return true +} + +func (i *BTCIterator) Value() (uint64, *container) { + if i.val == nil { + return 0, nil + } + return i.key.(uint64), i.val.(*container) +} diff --git a/roaring/containers.go b/roaring/containers_skiplist.go similarity index 100% rename from roaring/containers.go rename to roaring/containers_skiplist.go diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 464a92fd6..6d155f397 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -5,19 +5,19 @@ import ( ) func TestContainersIterator(t *testing.T) { - slc := NewSkipListContainers() - itr, found := slc.Iterator(0) + btc := NewBTreeContainers() + itr, found := btc.Iterator(0) if found { - t.Fatalf("shouldn't have found 0 in empty slc") + t.Fatalf("shouldn't have found 0 in empty btc") } if itr.Next() { - t.Fatal("Next() should be false for empty slc") + t.Fatal("Next() should be false for empty btc") } - slc.Put(1, &container{n: 1}) - slc.Put(2, &container{n: 2}) + btc.Put(1, &container{n: 1}) + btc.Put(2, &container{n: 2}) - itr, found = slc.Iterator(0) + itr, found = btc.Iterator(0) if found { t.Fatalf("shouldn't have found 0") } @@ -39,11 +39,11 @@ func TestContainersIterator(t *testing.T) { t.Fatalf("itr should be done, but got true") } - slc.Put(3, &container{n: 3}) - slc.Put(5, &container{n: 5}) - slc.Put(6, &container{n: 6}) + btc.Put(3, &container{n: 3}) + btc.Put(5, &container{n: 5}) + btc.Put(6, &container{n: 6}) - itr, found = slc.Iterator(3) + itr, found = btc.Iterator(3) if !itr.Next() { t.Fatalf("3 should be next, but got false") } @@ -60,7 +60,7 @@ func TestContainersIterator(t *testing.T) { t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) } - itr, found = slc.Iterator(4) + itr, found = btc.Iterator(4) if found { t.Fatalf("shouldn't have found 4") } diff --git a/roaring/roaring.go b/roaring/roaring.go index 367983322..fb1994bfd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -110,7 +110,7 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ - conts: NewSkipListContainers(), + conts: NewBTreeContainers(), } b.Add(a...) return b From e0fc49b5123fa89a5c12d7ce06687aaf4096d0c5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 15 Jan 2018 19:59:14 -0600 Subject: [PATCH 09/48] reset lastContainer cache on Put to a mapped container. check lastContainer cache on Get --- roaring/containers_btree.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index ce78c278a..a97eaed57 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -24,15 +24,28 @@ type BTreeContainers struct { } func (btc *BTreeContainers) Get(key uint64) *container { + // Check the last* cache for same container. + if key == btc.lastKey && btc.lastContainer != nil { + return btc.lastContainer + } + var c *container el, ok := btc.tree.Get(key) if ok { c = el.(*container) + btc.lastKey = key + btc.lastContainer = c } return c } func (btc *BTreeContainers) Put(key uint64, c *container) { + // If a mapped container is added to the tree, reset the + // lastContainer cache so that the cache is not pointing + // at a read-only mmap. + if c.mapped { + btc.lastContainer = nil + } btc.tree.Set(key, c) } @@ -73,7 +86,6 @@ func (btc *BTreeContainers) Clone() Containers { } nbtc.tree.Set(k, v.(*container).clone()) } - return nbtc } From 3ae10fbd958717c36f41e6828484ea68c53c9e98 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 17 Jan 2018 14:27:27 -0600 Subject: [PATCH 10/48] snapshot benchmarking --- fragment_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fragment_test.go b/fragment_test.go index 2ccf538f7..14d36a56c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1075,3 +1075,25 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } + +func BenchmarkFragment_Snapshot(b *testing.B) { + if *FragmentPath == "" { + b.Skip("no fragment specified") + } + + // Open the fragment specified by the path. + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + if err := f.Open(); err != nil { + b.Fatal(err) + } + defer f.Close() + + // Reset timer and execute benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := f.Snapshot() + if err != nil { + b.Fatalf("unexpected count (reopen): %s", err) + } + } +} From 0bed8de647141e6f1fb1788ecfa055ec02dad5e9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 17 Jan 2018 15:31:56 -0600 Subject: [PATCH 11/48] vendor btree in roaring package to test non-interface on key/value --- roaring/btree.go | 934 ++++++++++++++++++++++++++++++++++++ roaring/containers_btree.go | 22 +- 2 files changed, 944 insertions(+), 12 deletions(-) create mode 100644 roaring/btree.go diff --git a/roaring/btree.go b/roaring/btree.go new file mode 100644 index 000000000..8cb3d0bd2 --- /dev/null +++ b/roaring/btree.go @@ -0,0 +1,934 @@ +// Copyright 2014 The b Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package roaring + +import ( + "fmt" + "io" + "sync" +) + +const ( + kx = 128 //TODO benchmark tune this number if using custom key/value type(s). + kd = 128 //TODO benchmark tune this number if using custom key/value type(s). +) + +func init() { + if kd < 1 { + panic(fmt.Errorf("kd %d: out of range", kd)) + } + + if kx < 2 { + panic(fmt.Errorf("kx %d: out of range", kx)) + } +} + +var ( + btDPool = sync.Pool{New: func() interface{} { return &d{} }} + btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} + btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} + btXPool = sync.Pool{New: func() interface{} { return &x{} }} +) + +type btTpool struct{ sync.Pool } + +func (p *btTpool) get(cmp Cmp) *Tree { + x := p.Get().(*Tree) + x.cmp = cmp + return x +} + +type btEpool struct{ sync.Pool } + +func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator { + x := p.Get().(*Enumerator) + x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver + return x +} + +type ( + // Cmp compares a and b. Return value is: + // + // < 0 if a < b + // 0 if a == b + // > 0 if a > b + // + Cmp func(a, b uint64) int + + d struct { // data page + c int + d [2*kd + 1]de + n *d + p *d + } + + de struct { // d element + k uint64 + v *container + } + + // Enumerator captures the state of enumerating a tree. It is returned + // from the Seek* methods. The enumerator is aware of any mutations + // made to the tree in the process of enumerating it and automatically + // resumes the enumeration at the proper key, if possible. + // + // However, once an Enumerator returns io.EOF to signal "no more + // items", it does no more attempt to "resync" on tree mutation(s). In + // other words, io.EOF from an Enumerator is "sticky" (idempotent). + Enumerator struct { + err error + hit bool + i int + k uint64 + q *d + t *Tree + ver int64 + } + + // Tree is a B+tree. + Tree struct { + c int + cmp Cmp + first *d + last *d + r interface{} + ver int64 + } + + xe struct { // x element + ch interface{} + k uint64 + } + + x struct { // index page + c int + x [2*kx + 2]xe + } +) + +var ( // R/O zero values + zd d + zde de + ze Enumerator + zk uint64 + zt Tree + zx x + zxe xe +) + +func clr(q interface{}) { + switch x := q.(type) { + case *x: + for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn + clr(x.x[i].ch) + } + *x = zx + btXPool.Put(x) + case *d: + *x = zd + btDPool.Put(x) + } +} + +// -------------------------------------------------------------------------- x + +func newX(ch0 interface{}) *x { + r := btXPool.Get().(*x) + r.x[0].ch = ch0 + return r +} + +func (q *x) extract(i int) { + q.c-- + if i < q.c { + copy(q.x[i:], q.x[i+1:q.c+1]) + q.x[q.c].ch = q.x[q.c+1].ch + q.x[q.c].k = zk // GC + q.x[q.c+1] = zxe // GC + } +} + +func (q *x) insert(i int, k uint64, ch interface{}) *x { + c := q.c + if i < c { + q.x[c+1].ch = q.x[c].ch + copy(q.x[i+2:], q.x[i+1:c]) + q.x[i+1].k = q.x[i].k + } + c++ + q.c = c + q.x[i].k = k + q.x[i+1].ch = ch + return q +} + +func (q *x) siblings(i int) (l, r *d) { + if i >= 0 { + if i > 0 { + l = q.x[i-1].ch.(*d) + } + if i < q.c { + r = q.x[i+1].ch.(*d) + } + } + return +} + +// -------------------------------------------------------------------------- d + +func (l *d) mvL(r *d, c int) { + copy(l.d[l.c:], r.d[:c]) + copy(r.d[:], r.d[c:r.c]) + // Zero out the de's here to prevent reading bad data + // and to avoid creating non-collectible (GC) references. + for i := 1; i < c; i++ { + r.d[r.c-i] = zde + } + l.c += c + r.c -= c +} + +func (l *d) mvR(r *d, c int) { + copy(r.d[c:], r.d[:r.c]) + copy(r.d[:c], l.d[l.c-c:]) + // Zero out the de's here to prevent reading bad data + // and to avoid creating non-collectible (GC) references. + for i := 1; i < c; i++ { + l.d[l.c-c+i] = zde + } + r.c += c + l.c -= c +} + +// ----------------------------------------------------------------------- Tree + +// TreeNew returns a newly created, empty Tree. The compare function is used +// for key collation. +func TreeNew(cmp Cmp) *Tree { + return btTPool.get(cmp) +} + +// Clear removes all K/V pairs from the tree. +func (t *Tree) Clear() { + if t.r == nil { + return + } + + clr(t.r) + t.c, t.first, t.last, t.r = 0, nil, nil, nil + t.ver++ +} + +// Close performs Clear and recycles t to a pool for possible later reuse. No +// references to t should exist or such references must not be used afterwards. +func (t *Tree) Close() { + t.Clear() + *t = zt + btTPool.Put(t) +} + +func (t *Tree) cat(p *x, q, r *d, pi int) { + t.ver++ + q.mvL(r, r.c) + if r.n != nil { + r.n.p = q + } else { + t.last = q + } + q.n = r.n + *r = zd + btDPool.Put(r) + if p.c > 1 { + p.extract(pi) + p.x[pi].ch = q + return + } + + switch x := t.r.(type) { + case *x: + *x = zx + btXPool.Put(x) + case *d: + *x = zd + btDPool.Put(x) + } + t.r = q +} + +func (t *Tree) catX(p, q, r *x, pi int) { + t.ver++ + q.x[q.c].k = p.x[pi].k + copy(q.x[q.c+1:], r.x[:r.c]) + q.c += r.c + 1 + q.x[q.c].ch = r.x[r.c].ch + *r = zx + btXPool.Put(r) + if p.c > 1 { + p.c-- + pc := p.c + if pi < pc { + p.x[pi].k = p.x[pi+1].k + copy(p.x[pi+1:], p.x[pi+2:pc+1]) + p.x[pc].ch = p.x[pc+1].ch + p.x[pc].k = zk // GC + p.x[pc+1].ch = nil // GC + } + return + } + + switch x := t.r.(type) { + case *x: + *x = zx + btXPool.Put(x) + case *d: + *x = zd + btDPool.Put(x) + } + t.r = q +} + +// Delete removes the k's KV pair, if it exists, in which case Delete returns +// true. +func (t *Tree) Delete(k uint64) (ok bool) { + pi := -1 + var p *x + q := t.r + if q == nil { + return false + } + + for { + var i int + i, ok = t.find(q, k) + if ok { + switch x := q.(type) { + case *x: + if x.c < kx && q != t.r { + x, i = t.underflowX(p, x, pi, i) + } + pi = i + 1 + p = x + q = x.x[pi].ch + continue + case *d: + t.extract(x, i) + if x.c >= kd { + return true + } + + if q != t.r { + t.underflow(p, x, pi) + } else if t.c == 0 { + t.Clear() + } + return true + } + } + + switch x := q.(type) { + case *x: + if x.c < kx && q != t.r { + x, i = t.underflowX(p, x, pi, i) + } + pi = i + p = x + q = x.x[i].ch + case *d: + return false + } + } +} + +func (t *Tree) extract(q *d, i int) { // (r *container) { + t.ver++ + //r = q.d[i].v // prepared for Extract + q.c-- + if i < q.c { + copy(q.d[i:], q.d[i+1:q.c+1]) + } + q.d[q.c] = zde // GC + t.c-- + return +} + +func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { + var mk uint64 + l := 0 + switch x := q.(type) { + case *x: + h := x.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = x.x[m].k + switch cmp := t.cmp(k, mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + case *d: + h := x.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = x.d[m].k + switch cmp := t.cmp(k, mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + } + return l, false +} + +// First returns the first item of the tree in the key collating order, or +// (zero-value, zero-value) if the tree is empty. +func (t *Tree) First() (k uint64, v *container) { + if q := t.first; q != nil { + q := &q.d[0] + k, v = q.k, q.v + } + return +} + +// Get returns the value associated with k and true if it exists. Otherwise Get +// returns (zero-value, false). +func (t *Tree) Get(k uint64) (v *container, ok bool) { + q := t.r + if q == nil { + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch x := q.(type) { + case *x: + q = x.x[i+1].ch + continue + case *d: + return x.d[i].v, true + } + } + switch x := q.(type) { + case *x: + q = x.x[i].ch + default: + return + } + } +} + +func (t *Tree) insert(q *d, i int, k uint64, v *container) *d { + t.ver++ + c := q.c + if i < c { + copy(q.d[i+1:], q.d[i:c]) + } + c++ + q.c = c + q.d[i].k, q.d[i].v = k, v + t.c++ + return q +} + +// Last returns the last item of the tree in the key collating order, or +// (zero-value, zero-value) if the tree is empty. +func (t *Tree) Last() (k uint64, v *container) { + if q := t.last; q != nil { + q := &q.d[q.c-1] + k, v = q.k, q.v + } + return +} + +// Len returns the number of items in the tree. +func (t *Tree) Len() int { + return t.c +} + +func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *container) { + t.ver++ + l, r := p.siblings(pi) + + // s is the number of items to shift out of the full data container to + // allow for the new data item. This logic shifts by half the available + // space plus one. In the case where the new item is to be inserted within + // the calculated shift space, then s is reduced to include only the + // data items up to the index of the new data item. + if l != nil && l.c < 2*kd && i != 0 { + s := (2*kd-l.c)/2 + 1 // half plus one + //s := 2*kd - l.c // all avaiable + if i < s { + s = i + } + l.mvL(q, s) + t.insert(q, i-s, k, v) + p.x[pi-1].k = q.d[0].k + return + } + + if r != nil && r.c < 2*kd { + if i < 2*kd { + s := (2*kd-r.c)/2 + 1 // half plus one + //s := 2*kd - r.c // all available + if 2*kd-i < s { + s = 2*kd - i + } + q.mvR(r, s) + t.insert(q, i, k, v) + p.x[pi].k = r.d[0].k + return + } + + t.insert(r, 0, k, v) + p.x[pi].k = k + return + } + + t.split(p, q, pi, i, k, v) +} + +// Seek returns an Enumerator positioned on an item such that k >= item's key. +// ok reports if k == item.key The Enumerator's position is possibly after the +// last item in the tree. +func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) { + q := t.r + if q == nil { + e = btEPool.get(nil, false, 0, k, nil, t, t.ver) + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch x := q.(type) { + case *x: + q = x.x[i+1].ch + continue + case *d: + return btEPool.get(nil, ok, i, k, x, t, t.ver), true + } + } + + switch x := q.(type) { + case *x: + q = x.x[i].ch + case *d: + return btEPool.get(nil, ok, i, k, x, t, t.ver), false + } + } +} + +// SeekFirst returns an enumerator positioned on the first KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *Tree) SeekFirst() (e *Enumerator, err error) { + q := t.first + if q == nil { + return nil, io.EOF + } + + return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil +} + +// SeekLast returns an enumerator positioned on the last KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *Tree) SeekLast() (e *Enumerator, err error) { + q := t.last + if q == nil { + return nil, io.EOF + } + + return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil +} + +// Set sets the value associated with k. +func (t *Tree) Set(k uint64, v *container) { + //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) + //defer func() { + // dbg("--- POST\n%s\n====\n", t.dump()) + //}() + + pi := -1 + var p *x + q := t.r + if q == nil { + z := t.insert(btDPool.Get().(*d), 0, k, v) + t.r, t.first, t.last = z, z, z + return + } + + for { + i, ok := t.find(q, k) + if ok { + switch x := q.(type) { + case *x: + i++ + if x.c > 2*kx { + x, i = t.splitX(p, x, pi, i) + } + pi = i + p = x + q = x.x[i].ch + continue + case *d: + x.d[i].v = v + } + return + } + + switch x := q.(type) { + case *x: + if x.c > 2*kx { + x, i = t.splitX(p, x, pi, i) + } + pi = i + p = x + q = x.x[i].ch + case *d: + switch { + case x.c < 2*kd: + t.insert(x, i, k, v) + default: + t.overflow(p, x, pi, i, k, v) + } + return + } + } +} + +// Put combines Get and Set in a more efficient way where the tree is walked +// only once. The upd(ater) receives (old-value, true) if a KV pair for k +// exists or (zero-value, false) otherwise. It can then return a (new-value, +// true) to create or overwrite the existing value in the KV pair, or +// (whatever, false) if it decides not to create or not to update the value of +// the KV pair. +// +// tree.Set(k, v) call conceptually equals calling +// +// tree.Put(k, func(uint64, bool){ return v, true }) +// +// modulo the differing return values. +func (t *Tree) Put(k uint64, upd func(oldV *container, exists bool) (newV *container, write bool)) (oldV *container, written bool) { + pi := -1 + var p *x + q := t.r + var newV *container + if q == nil { + // new KV pair in empty tree + newV, written = upd(newV, false) + if !written { + return + } + + z := t.insert(btDPool.Get().(*d), 0, k, newV) + t.r, t.first, t.last = z, z, z + return + } + + for { + i, ok := t.find(q, k) + if ok { + switch x := q.(type) { + case *x: + i++ + if x.c > 2*kx { + x, i = t.splitX(p, x, pi, i) + } + pi = i + p = x + q = x.x[i].ch + continue + case *d: + oldV = x.d[i].v + newV, written = upd(oldV, true) + if !written { + return + } + + x.d[i].v = newV + } + return + } + + switch x := q.(type) { + case *x: + if x.c > 2*kx { + x, i = t.splitX(p, x, pi, i) + } + pi = i + p = x + q = x.x[i].ch + case *d: // new KV pair + newV, written = upd(newV, false) + if !written { + return + } + + switch { + case x.c < 2*kd: + t.insert(x, i, k, newV) + default: + t.overflow(p, x, pi, i, k, newV) + } + return + } + } +} + +func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *container) { + t.ver++ + r := btDPool.Get().(*d) + if q.n != nil { + r.n = q.n + r.n.p = r + } else { + t.last = r + } + q.n = r + r.p = q + + copy(r.d[:], q.d[kd:2*kd]) + for i := range q.d[kd:] { + q.d[kd+i] = zde + } + q.c = kd + r.c = kd + var done bool + if i > kd { + done = true + t.insert(r, i-kd, k, v) + } + if pi >= 0 { + p.insert(pi, r.d[0].k, r) + } else { + t.r = newX(q).insert(0, r.d[0].k, r) + } + if done { + return + } + + t.insert(q, i, k, v) +} + +func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { + t.ver++ + r := btXPool.Get().(*x) + copy(r.x[:], q.x[kx+1:]) + q.c = kx + r.c = kx + if pi >= 0 { + p.insert(pi, q.x[kx].k, r) + } else { + t.r = newX(q).insert(0, q.x[kx].k, r) + } + + q.x[kx].k = zk + for i := range q.x[kx+1:] { + q.x[kx+i+1] = zxe + } + if i > kx { + q = r + i -= kx + 1 + } + + return q, i +} + +func (t *Tree) underflow(p *x, q *d, pi int) { + t.ver++ + l, r := p.siblings(pi) + + if l != nil && l.c+q.c >= 2*kd { + l.mvR(q, 1) + p.x[pi-1].k = q.d[0].k + return + } + + if r != nil && q.c+r.c >= 2*kd { + q.mvL(r, 1) + p.x[pi].k = r.d[0].k + r.d[r.c] = zde // GC + return + } + + if l != nil { + t.cat(p, l, q, pi-1) + return + } + + t.cat(p, q, r, pi) +} + +func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { + t.ver++ + var l, r *x + + if pi >= 0 { + if pi > 0 { + l = p.x[pi-1].ch.(*x) + } + if pi < p.c { + r = p.x[pi+1].ch.(*x) + } + } + + if l != nil && l.c > kx { + q.x[q.c+1].ch = q.x[q.c].ch + copy(q.x[1:], q.x[:q.c]) + q.x[0].ch = l.x[l.c].ch + q.x[0].k = p.x[pi-1].k + q.c++ + i++ + l.c-- + p.x[pi-1].k = l.x[l.c].k + return q, i + } + + if r != nil && r.c > kx { + q.x[q.c].k = p.x[pi].k + q.c++ + q.x[q.c].ch = r.x[0].ch + p.x[pi].k = r.x[0].k + copy(r.x[:], r.x[1:r.c]) + r.c-- + rc := r.c + r.x[rc].ch = r.x[rc+1].ch + r.x[rc].k = zk + r.x[rc+1].ch = nil + return q, i + } + + if l != nil { + i += l.c + 1 + t.catX(p, l, q, pi-1) + q = l + return q, i + } + + t.catX(p, q, r, pi) + return q, i +} + +// ----------------------------------------------------------------- Enumerator + +// Close recycles e to a pool for possible later reuse. No references to e +// should exist or such references must not be used afterwards. +func (e *Enumerator) Close() { + *e = ze + btEPool.Put(e) +} + +// Next returns the currently enumerated item, if it exists and moves to the +// next item in the key collation order. If there is no item to return, err == +// io.EOF is returned. +func (e *Enumerator) Next() (k uint64, v *container, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if e.i >= e.q.c { + if err = e.next(); err != nil { + return + } + } + + i := e.q.d[e.i] + k, v = i.k, i.v + e.k, e.hit = k, true + e.next() + return +} + +func (e *Enumerator) next() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i < e.q.c-1: + e.i++ + default: + if e.q, e.i = e.q.n, 0; e.q == nil { + e.err = io.EOF + } + } + return e.err +} + +// Prev returns the currently enumerated item, if it exists and moves to the +// previous item in the key collation order. If there is no item to return, err +// == io.EOF is returned. +func (e *Enumerator) Prev() (k uint64, v *container, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if !e.hit { + // move to previous because Seek overshoots if there's no hit + if err = e.prev(); err != nil { + return + } + } + + if e.i >= e.q.c { + if err = e.prev(); err != nil { + return + } + } + + i := e.q.d[e.i] + k, v = i.k, i.v + e.k, e.hit = k, true + e.prev() + return +} + +func (e *Enumerator) prev() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i > 0: + e.i-- + default: + if e.q = e.q.p; e.q == nil { + e.err = io.EOF + break + } + + e.i = e.q.c - 1 + } + return e.err +} diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index a97eaed57..f2878fdf9 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -2,8 +2,6 @@ package roaring import ( "io" - - btree "github.com/pilosa/b" ) func cmp(a, b uint64) int { @@ -12,12 +10,12 @@ func cmp(a, b uint64) int { func NewBTreeContainers() *BTreeContainers { return &BTreeContainers{ - tree: btree.TreeNew(cmp), + tree: TreeNew(cmp), } } type BTreeContainers struct { - tree *btree.Tree + tree *Tree lastKey uint64 lastContainer *container @@ -32,7 +30,7 @@ func (btc *BTreeContainers) Get(key uint64) *container { var c *container el, ok := btc.tree.Get(key) if ok { - c = el.(*container) + c = el btc.lastKey = key btc.lastContainer = c } @@ -68,7 +66,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *container { return cont } - btc.lastContainer = v.(*container) + btc.lastContainer = v return btc.lastContainer } @@ -84,7 +82,7 @@ func (btc *BTreeContainers) Clone() Containers { if err == io.EOF { break } - nbtc.tree.Set(k, v.(*container).clone()) + nbtc.tree.Set(k, v.clone()) } return nbtc } @@ -94,7 +92,7 @@ func (btc *BTreeContainers) Last() (key uint64, c *container) { return 0, nil } k, v := btc.tree.Last() - return k, v.(*container) + return k, v } func (btc *BTreeContainers) Size() int { @@ -113,9 +111,9 @@ func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool } type BTCIterator struct { - e *btree.Enumerator - key interface{} - val interface{} + e *Enumerator + key uint64 + val *container } func (i *BTCIterator) Next() bool { @@ -133,5 +131,5 @@ func (i *BTCIterator) Value() (uint64, *container) { if i.val == nil { return 0, nil } - return i.key.(uint64), i.val.(*container) + return i.key, i.val } From 0757a84f69945f177273af184a7ff5a9ec31cfaf Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 17 Jan 2018 17:05:13 -0600 Subject: [PATCH 12/48] add PutContainerValues to Containers interface to prevent re-allocation of containers --- roaring/containers_btree.go | 19 +++++++++++++++++++ roaring/containers_skiplist.go | 3 +++ roaring/roaring.go | 14 +++++++++----- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index f2878fdf9..39e479513 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -47,6 +47,25 @@ func (btc *BTreeContainers) Put(key uint64, c *container) { btc.tree.Set(key, c) } +func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + f := func(oldV *container, exists bool) (*container, bool) { + // update the existing container + if exists { + oldV.containerType = containerType + oldV.n = n + oldV.mapped = mapped + return oldV, true + } + return &container{ + containerType: containerType, + n: n, + mapped: mapped, + }, true + } + + btc.tree.Put(key, f) +} + func (btc *BTreeContainers) Remove(key uint64) { btc.tree.Delete(key) } diff --git a/roaring/containers_skiplist.go b/roaring/containers_skiplist.go index c80dbeb06..07d6dc870 100644 --- a/roaring/containers_skiplist.go +++ b/roaring/containers_skiplist.go @@ -25,6 +25,9 @@ func (slc *SkipListContainers) Put(key uint64, c *container) { slc.list.Set(key, c) } +func (slc *SkipListContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +} + func (slc *SkipListContainers) Remove(key uint64) { slc.list.Remove(key) } diff --git a/roaring/roaring.go b/roaring/roaring.go index fb1994bfd..51f1efe9a 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -70,6 +70,10 @@ type Containers interface { // Put adds the container at key. Put(key uint64, c *container) + // PutContainerValues updates an existing container at key. + // If a container does not exist for key, a new one is allocated. + PutContainerValues(key uint64, containerType byte, n int, mapped bool) + // Remove takes the container at key out. Remove(key uint64) @@ -631,11 +635,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { - b.conts.Put(binary.LittleEndian.Uint64(buf[0:8]), &container{ - containerType: byte(binary.LittleEndian.Uint16(buf[8:10])), - n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, - mapped: true, - }) + b.conts.PutContainerValues( + binary.LittleEndian.Uint64(buf[0:8]), + byte(binary.LittleEndian.Uint16(buf[8:10])), + int(binary.LittleEndian.Uint16(buf[10:12]))+1, + true) } opsOffset := headerSize + int(keyN)*12 From f825bc3b807d61194573dc219a79c2541d104a56 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 12:07:08 -0600 Subject: [PATCH 13/48] merge --- fragment_test.go | 4 ++-- roaring/assembly_test.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index 14d36a56c..754bd8dd1 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1081,15 +1081,15 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.Skip("no fragment specified") } + b.ReportAllocs() // Open the fragment specified by the path. f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } defer f.Close() - - // Reset timer and execute benchmark. b.ResetTimer() + // Reset timer and execute benchmark. for i := 0; i < b.N; i++ { err := f.Snapshot() if err != nil { diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go index f18aa65ec..a35ae58c5 100644 --- a/roaring/assembly_test.go +++ b/roaring/assembly_test.go @@ -42,6 +42,7 @@ func TestBSFQ_CompareGo(t *testing.T) { } */ } +var Result uint64 func BenchmarkBSF(b *testing.B) { for i := 0; i < b.N; i++ { BSFQ(uint64(i)) @@ -62,7 +63,7 @@ func BenchmarkPOPCNTQ(b *testing.B) { func BenchmarkPopcount(b *testing.B) { for i := 0; i < b.N; i++ { - popcount(uint64(i)) + Result=popcount(uint64(i)) } } @@ -76,7 +77,7 @@ func BenchmarkPopcntAsm(b *testing.B) { func BenchmarkPopcntGo(b *testing.B) { // run the Fib function b.N times for n := 0; n < b.N; n++ { - popcntGo(0xdeadbeef) + Result=popcntGo(uint64(n)) } } From f5d5cbb0b9db8abaa29f1e3828e7f0f33ce37966 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 16:03:46 -0600 Subject: [PATCH 14/48] limited scope of closure in btree PutContainerValues --- roaring/containers_btree.go | 40 ++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 39e479513..cc0f37a81 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -47,23 +47,31 @@ func (btc *BTreeContainers) Put(key uint64, c *container) { btc.tree.Set(key, c) } -func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - f := func(oldV *container, exists bool) (*container, bool) { - // update the existing container - if exists { - oldV.containerType = containerType - oldV.n = n - oldV.mapped = mapped - return oldV, true - } - return &container{ - containerType: containerType, - n: n, - mapped: mapped, - }, true - } +type updater struct { + key uint64 + containerType byte + n int + mapped bool +} - btc.tree.Put(key, f) +func (u updater) update(oldV *container, exists bool) (*container, bool) { + // update the existing container + if exists { + oldV.containerType = u.containerType + oldV.n = u.n + oldV.mapped = u.mapped + return oldV, true + } + return &container{ + containerType: u.containerType, + n: u.n, + mapped: u.mapped, + }, true +} + +func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + a := updater{key, containerType, n, mapped} + btc.tree.Put(key, a.update) } func (btc *BTreeContainers) Remove(key uint64) { From a159c74498cd338ff51a8d58da05a928fb5f0f13 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 23 Jan 2018 11:19:07 -0600 Subject: [PATCH 15/48] corrected return value in updater to prevent allocation --- roaring/containers_btree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index cc0f37a81..9bc2d118e 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -60,7 +60,7 @@ func (u updater) update(oldV *container, exists bool) (*container, bool) { oldV.containerType = u.containerType oldV.n = u.n oldV.mapped = u.mapped - return oldV, true + return oldV, false } return &container{ containerType: u.containerType, From 8a70c4e5a34fb59c8233b0abe30107ce1a589632 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 23 Jan 2018 16:27:12 -0600 Subject: [PATCH 16/48] added slice containers type for in memory bitmaps and btree for file based --- fragment.go | 2 +- roaring/containers_test.go | 3 ++- roaring/roaring.go | 8 ++++++++ roaring/roaring_test.go | 10 +++++----- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/fragment.go b/fragment.go index 930c8e02b..a18c4de61 100644 --- a/fragment.go +++ b/fragment.go @@ -189,7 +189,7 @@ func (f *Fragment) Open() error { func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. if f.storage == nil { - f.storage = roaring.NewBitmap() + f.storage = roaring.NewBitmapBtree() } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 6d155f397..02e998151 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -5,7 +5,8 @@ import ( ) func TestContainersIterator(t *testing.T) { - btc := NewBTreeContainers() + //btc := NewBTreeContainers() + btc := NewSliceContainers() itr, found := btc.Iterator(0) if found { t.Fatalf("shouldn't have found 0 in empty btc") diff --git a/roaring/roaring.go b/roaring/roaring.go index 51f1efe9a..bcef1762c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -113,6 +113,14 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { + b := &Bitmap{ + conts: NewSliceContainers(), + } + b.Add(a...) + return b +} + +func NewBitmapBtree(a ...uint64) *Bitmap { b := &Bitmap{ conts: NewBTreeContainers(), } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f972d8573..48d29c058 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1176,7 +1176,7 @@ const ( func BenchmarkContainerLinear(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewBitmapBtree() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1187,7 +1187,7 @@ func BenchmarkContainerLinear(b *testing.B) { func BenchmarkContainerReverse(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewBitmapBtree() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1198,7 +1198,7 @@ func BenchmarkContainerReverse(b *testing.B) { func BenchmarkContainerColumn(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewBitmapBtree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1210,7 +1210,7 @@ func BenchmarkContainerColumn(b *testing.B) { func BenchmarkContainerOutsideIn(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewBitmapBtree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { @@ -1224,7 +1224,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) { func BenchmarkContainerInsideOut(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewBitmapBtree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) From 36f59a03261cf7999e7760e9bc4d5f45e69debaa Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 23 Jan 2018 16:34:00 -0600 Subject: [PATCH 17/48] slice containers --- roaring/containers_slice.go | 149 ++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 roaring/containers_slice.go diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go new file mode 100644 index 000000000..a27d9bd06 --- /dev/null +++ b/roaring/containers_slice.go @@ -0,0 +1,149 @@ +package roaring + +func NewSliceContainers() *SliceContainers { + return &SliceContainers{} +} + +type SliceContainers struct { + keys []uint64 + containers []*container + lastKey uint64 + lastContainer *container +} + +func (sc *SliceContainers) Get(key uint64) *container { + i := search64(sc.keys, key) + if i < 0 { + return nil + } + return sc.containers[i] +} + +func (sc *SliceContainers) Put(key uint64, c *container) { + i := search64(sc.keys, key) + + // If index is negative then there's not an exact match + // and a container needs to be added. + if i < 0 { + sc.insertAt(key, c, -i-1) + } else { + //should this happen? + sc.containers[i] = c + } + +} + +func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + i := search64(sc.keys, key) + if i < 0 { + c := newContainer() + c.containerType = containerType + c.n = n + c.mapped = mapped + sc.insertAt(key, c, -i-1) + } else { + //should this happen? + c := sc.containers[i] + c.containerType = containerType + c.n = n + c.mapped = mapped + } + +} + +func (sc *SliceContainers) Remove(key uint64) { + i := search64(sc.keys, key) + if i < 0 { + return + } + sc.keys = append(sc.keys[:i], sc.keys[i+1:]...) + sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) + +} +func (sc *SliceContainers) insertAt(key uint64, c *container, i int) { + sc.keys = append(sc.keys, 0) + copy(sc.keys[i+1:], sc.keys[i:]) + sc.keys[i] = key + + sc.containers = append(sc.containers, nil) + copy(sc.containers[i+1:], sc.containers[i:]) + sc.containers[i] = c +} + +func (sc *SliceContainers) GetOrCreate(key uint64) *container { + // Check the last* cache for same container. + if key == sc.lastKey && sc.lastContainer != nil { + return sc.lastContainer + } + + sc.lastKey = key + i := search64(sc.keys, key) + if i < 0 { + c := newContainer() + sc.insertAt(key, c, -i-1) + sc.lastContainer = c + return c + } + + sc.lastContainer = sc.containers[i] + return sc.lastContainer +} + +func (sc *SliceContainers) Clone() Containers { + other := NewSliceContainers() + other.keys = make([]uint64, len(sc.keys)) + other.containers = make([]*container, len(sc.containers)) + copy(other.keys, sc.keys) + for i, c := range sc.containers { + other.containers[i] = c.clone() + } + return other +} + +func (sc *SliceContainers) Last() (key uint64, c *container) { + if len(sc.keys) == 0 { + return 0, nil + } + return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1] +} + +func (sc *SliceContainers) Size() int { + return len(sc.keys) + +} + +func (sc *SliceContainers) seek(key uint64) (int, bool) { + i := search64(sc.keys, key) + found := true + if i < 0 { + found = false + i = -i - 1 + } + return i, found +} + +func (sc *SliceContainers) Iterator(key uint64) (citer Contiterator, found bool) { + i, found := sc.seek(key) + return &SliceIterator{e: sc, i: i}, found +} + +type SliceIterator struct { + e *SliceContainers + i int + key uint64 + value *container +} + +func (si *SliceIterator) Next() bool { + if si.e == nil || si.i > len(si.e.keys)-1 { + return false + } + si.key = si.e.keys[si.i] + si.value = si.e.containers[si.i] + si.i++ + return true +} + +func (si *SliceIterator) Value() (uint64, *container) { + return si.key, si.value +} From 440980384c4c854ae9f4f67dd90689d207e9eb6a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 5 Feb 2018 14:04:10 -0600 Subject: [PATCH 18/48] applied travis suggestions --- fragment.go | 2 +- roaring/assembly_test.go | 8 +++++--- roaring/containers_btree.go | 15 ++++++++------- roaring/containers_slice.go | 2 -- roaring/containers_test.go | 31 +++++++++++++++++++------------ roaring/roaring.go | 2 +- roaring/roaring_test.go | 10 +++++----- 7 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fragment.go b/fragment.go index a18c4de61..d1cd41e27 100644 --- a/fragment.go +++ b/fragment.go @@ -189,7 +189,7 @@ func (f *Fragment) Open() error { func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. if f.storage == nil { - f.storage = roaring.NewBitmapBtree() + f.storage = roaring.NewBitmapBTree() } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go index a35ae58c5..b3e75c32c 100644 --- a/roaring/assembly_test.go +++ b/roaring/assembly_test.go @@ -42,7 +42,6 @@ func TestBSFQ_CompareGo(t *testing.T) { } */ } -var Result uint64 func BenchmarkBSF(b *testing.B) { for i := 0; i < b.N; i++ { BSFQ(uint64(i)) @@ -61,9 +60,12 @@ func BenchmarkPOPCNTQ(b *testing.B) { } } +// This value prevents the benchmarks from being optimized out +var Result uint64 + func BenchmarkPopcount(b *testing.B) { for i := 0; i < b.N; i++ { - Result=popcount(uint64(i)) + Result = popcount(uint64(i)) } } @@ -77,7 +79,7 @@ func BenchmarkPopcntAsm(b *testing.B) { func BenchmarkPopcntGo(b *testing.B) { // run the Fib function b.N times for n := 0; n < b.N; n++ { - Result=popcntGo(uint64(n)) + Result = popcntGo(uint64(n)) } } diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 9bc2d118e..134fb12a2 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -47,13 +47,6 @@ func (btc *BTreeContainers) Put(key uint64, c *container) { btc.tree.Set(key, c) } -type updater struct { - key uint64 - containerType byte - n int - mapped bool -} - func (u updater) update(oldV *container, exists bool) (*container, bool) { // update the existing container if exists { @@ -69,6 +62,14 @@ func (u updater) update(oldV *container, exists bool) (*container, bool) { }, true } +// this struct is added to prevent the closure locals from being escaped out to the heap +type updater struct { + key uint64 + containerType byte + n int + mapped bool +} + func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { a := updater{key, containerType, n, mapped} btc.tree.Put(key, a.update) diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go index a27d9bd06..391ed74e4 100644 --- a/roaring/containers_slice.go +++ b/roaring/containers_slice.go @@ -27,7 +27,6 @@ func (sc *SliceContainers) Put(key uint64, c *container) { if i < 0 { sc.insertAt(key, c, -i-1) } else { - //should this happen? sc.containers[i] = c } @@ -42,7 +41,6 @@ func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n c.mapped = mapped sc.insertAt(key, c, -i-1) } else { - //should this happen? c := sc.containers[i] c.containerType = containerType c.n = n diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 02e998151..4199fcbe5 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -4,10 +4,17 @@ import ( "testing" ) -func TestContainersIterator(t *testing.T) { - //btc := NewBTreeContainers() - btc := NewSliceContainers() - itr, found := btc.Iterator(0) +func TestContainersSliceIterator(t *testing.T) { + btc := NewBTreeContainers() + testContainersIterator(btc, t) +} +func TestContainersBTreeIterator(t *testing.T) { + slc := NewSliceContainers() + testContainersIterator(slc, t) + +} +func testContainersIterator(cs Containers, t *testing.T) { + itr, found := cs.Iterator(0) if found { t.Fatalf("shouldn't have found 0 in empty btc") } @@ -15,10 +22,10 @@ func TestContainersIterator(t *testing.T) { t.Fatal("Next() should be false for empty btc") } - btc.Put(1, &container{n: 1}) - btc.Put(2, &container{n: 2}) + cs.Put(1, &container{n: 1}) + cs.Put(2, &container{n: 2}) - itr, found = btc.Iterator(0) + itr, found = cs.Iterator(0) if found { t.Fatalf("shouldn't have found 0") } @@ -40,11 +47,11 @@ func TestContainersIterator(t *testing.T) { t.Fatalf("itr should be done, but got true") } - btc.Put(3, &container{n: 3}) - btc.Put(5, &container{n: 5}) - btc.Put(6, &container{n: 6}) + cs.Put(3, &container{n: 3}) + cs.Put(5, &container{n: 5}) + cs.Put(6, &container{n: 6}) - itr, found = btc.Iterator(3) + itr, found = cs.Iterator(3) if !itr.Next() { t.Fatalf("3 should be next, but got false") } @@ -61,7 +68,7 @@ func TestContainersIterator(t *testing.T) { t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) } - itr, found = btc.Iterator(4) + itr, found = cs.Iterator(4) if found { t.Fatalf("shouldn't have found 4") } diff --git a/roaring/roaring.go b/roaring/roaring.go index bcef1762c..3cc5c1638 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -120,7 +120,7 @@ func NewBitmap(a ...uint64) *Bitmap { return b } -func NewBitmapBtree(a ...uint64) *Bitmap { +func NewBitmapBTree(a ...uint64) *Bitmap { b := &Bitmap{ conts: NewBTreeContainers(), } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 48d29c058..bea8c140b 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1176,7 +1176,7 @@ const ( func BenchmarkContainerLinear(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBtree() + bm := roaring.NewBitmapBTree() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1187,7 +1187,7 @@ func BenchmarkContainerLinear(b *testing.B) { func BenchmarkContainerReverse(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBtree() + bm := roaring.NewBitmapBTree() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1198,7 +1198,7 @@ func BenchmarkContainerReverse(b *testing.B) { func BenchmarkContainerColumn(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBtree() + bm := roaring.NewBitmapBTree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1210,7 +1210,7 @@ func BenchmarkContainerColumn(b *testing.B) { func BenchmarkContainerOutsideIn(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBtree() + bm := roaring.NewBitmapBTree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { @@ -1224,7 +1224,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) { func BenchmarkContainerInsideOut(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBtree() + bm := roaring.NewBitmapBTree() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) From 23970b98dc48855d9d2f14d1b8a7052005f0a420 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 5 Feb 2018 14:24:03 -0600 Subject: [PATCH 19/48] changed roaring.NewBitmapBTree to roaring.NewBTreeBitmap --- fragment.go | 2 +- roaring/roaring.go | 2 +- roaring/roaring_test.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fragment.go b/fragment.go index d1cd41e27..0da6043cf 100644 --- a/fragment.go +++ b/fragment.go @@ -189,7 +189,7 @@ func (f *Fragment) Open() error { func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. if f.storage == nil { - f.storage = roaring.NewBitmapBTree() + f.storage = roaring.NewBTreeBitmap() } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) diff --git a/roaring/roaring.go b/roaring/roaring.go index 3cc5c1638..235905b88 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -120,7 +120,7 @@ func NewBitmap(a ...uint64) *Bitmap { return b } -func NewBitmapBTree(a ...uint64) *Bitmap { +func NewBTreeBitmap(a ...uint64) *Bitmap { b := &Bitmap{ conts: NewBTreeContainers(), } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index bea8c140b..97b562e08 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1176,7 +1176,7 @@ const ( func BenchmarkContainerLinear(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBTree() + bm := roaring.NewBTreeBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1187,7 +1187,7 @@ func BenchmarkContainerLinear(b *testing.B) { func BenchmarkContainerReverse(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBTree() + bm := roaring.NewBTreeBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1198,7 +1198,7 @@ func BenchmarkContainerReverse(b *testing.B) { func BenchmarkContainerColumn(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBTree() + bm := roaring.NewBTreeBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1210,7 +1210,7 @@ func BenchmarkContainerColumn(b *testing.B) { func BenchmarkContainerOutsideIn(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBTree() + bm := roaring.NewBTreeBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { @@ -1224,7 +1224,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) { func BenchmarkContainerInsideOut(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmapBTree() + bm := roaring.NewBTreeBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) From 6c0acf4dd3d67531175d0a32475701bf60bbeedc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 5 Feb 2018 14:14:04 -0600 Subject: [PATCH 20/48] squash with vendor btree commit --- roaring/btree.go | 1 - 1 file changed, 1 deletion(-) diff --git a/roaring/btree.go b/roaring/btree.go index 8cb3d0bd2..1c6630a04 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -350,7 +350,6 @@ func (t *Tree) extract(q *d, i int) { // (r *container) { } q.d[q.c] = zde // GC t.c-- - return } func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { From be8b79406098319b5dbd1b40274bcf044118bc78 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 5 Feb 2018 14:33:09 -0600 Subject: [PATCH 21/48] add NOTICE for third-party software licenses; in this case: btree --- NOTICE | 25 +++++++++++++++++++++++++ roaring/btree.go | 3 ++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..365e87c82 --- /dev/null +++ b/NOTICE @@ -0,0 +1,25 @@ +Software license +================ + +Copyright 2017 Pilosa Corp. + +Licensed under the Apache License, Version 2.0 (the "License"). +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Third-party software licenses +============================= + +The file /pilosa/roaring/btree.go is licensed under the BSD 3-Clause license, +which can be found at: + + https://github.com/cznic/b/blob/master/LICENSE + diff --git a/roaring/btree.go b/roaring/btree.go index 1c6630a04..24fd76507 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -1,6 +1,7 @@ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// license that can be found at: +// https://github.com/cznic/b/blob/master/LICENSE package roaring From 65dadd2b351175a3e6c1dc2650d1d602df146874 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 5 Feb 2018 15:10:51 -0600 Subject: [PATCH 22/48] Copy verbatim vendored legal notice and modify binary distribution process to include legal notices --- Dockerfile | 3 +++ Makefile | 4 ++-- NOTICE | 31 ++++++++++++++++++++++++++++--- roaring/btree.go | 34 ++++++++++++++++++++++++++++++---- 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6d86de974..fb3d5cc94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,9 @@ LABEL maintainer "dev@pilosa.com" COPY --from=builder /go/bin/pilosa /pilosa +COPY LICENSE /LICENSE +COPY NOTICE /NOTICE + EXPOSE 10101 VOLUME /data diff --git a/Makefile b/Makefile index 48c0075a8..a76225bd8 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ ifdef DOCKER_BUILD else make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" endif - cp LICENSE README.md build/pilosa-$(IDENTIFIER) + cp NOTICE LICENSE README.md build/pilosa-$(IDENTIFIER) tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" @@ -76,7 +76,7 @@ endif prerelease-build: vendor make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" - cp LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH) + cp NOTICE LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH) tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/ @echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz" diff --git a/NOTICE b/NOTICE index 365e87c82..b93e9a17d 100644 --- a/NOTICE +++ b/NOTICE @@ -18,8 +18,33 @@ limitations under the License. Third-party software licenses ============================= -The file /pilosa/roaring/btree.go is licensed under the BSD 3-Clause license, -which can be found at: +The file /pilosa/roaring/btree.go contains a modified redistribution of b +(https://github.com/cznic/b); the license follows: - https://github.com/cznic/b/blob/master/LICENSE + Copyright (c) 2014 The b Authors. All rights reserved. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the names of the authors nor the names of the + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/roaring/btree.go b/roaring/btree.go index 24fd76507..e3646113b 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -1,7 +1,33 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found at: -// https://github.com/cznic/b/blob/master/LICENSE +// This file is a modified redistribution of b (https://github.com/cznic/b), +// which is governed by the following license notice: +// +// Copyright (c) 2014 The b Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the names of the authors nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package roaring From 26b6f6b1192028df6754ae0c86bd51fc9dbf461d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 6 Feb 2018 13:12:45 -0600 Subject: [PATCH 23/48] rename NewBitmap to NewSliceBitmap --- bitmap.go | 4 +- ctl/check.go | 2 +- ctl/inspect.go | 2 +- roaring/roaring.go | 16 +-- roaring/roaring_internal_test.go | 30 +++--- roaring/roaring_test.go | 176 +++++++++++++++---------------- 6 files changed, 115 insertions(+), 115 deletions(-) diff --git a/bitmap.go b/bitmap.go index bc2f260d4..0728ec218 100644 --- a/bitmap.go +++ b/bitmap.go @@ -189,12 +189,12 @@ func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment { } // Insert new segment. - b.segments = append(b.segments, BitmapSegment{data: *roaring.NewBitmap()}) + b.segments = append(b.segments, BitmapSegment{data: *roaring.NewSliceBitmap()}) if i < len(b.segments) { copy(b.segments[i+1:], b.segments[i:]) } b.segments[i] = BitmapSegment{ - data: *roaring.NewBitmap(), + data: *roaring.NewSliceBitmap(), slice: slice, writable: true, } diff --git a/ctl/check.go b/ctl/check.go index 34b7033fc..12591a0ac 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -88,7 +88,7 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { defer syscall.Munmap(data) // Attach the mmap file to the bitmap. - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() if err := bm.UnmarshalBinary(data); err != nil { return err } diff --git a/ctl/inspect.go b/ctl/inspect.go index f5825c35a..dd59bf429 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -68,7 +68,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { // Attach the mmap file to the bitmap. t := time.Now() fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() if err := bm.UnmarshalBinary(data); err != nil { return err } diff --git a/roaring/roaring.go b/roaring/roaring.go index 235905b88..6feb90166 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -111,8 +111,8 @@ type Bitmap struct { OpWriter io.Writer } -// NewBitmap returns a Bitmap with an initial set of values. -func NewBitmap(a ...uint64) *Bitmap { +// NewSliceBitmap returns a Bitmap with an initial set of values. +func NewSliceBitmap(a ...uint64) *Bitmap { b := &Bitmap{ conts: NewSliceContainers(), } @@ -329,7 +329,7 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) citer, _ := b.conts.Iterator(hi0) - other := NewBitmap() + other := NewSliceBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { @@ -374,7 +374,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { - output := NewBitmap() + output := NewSliceBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -399,7 +399,7 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { - output := NewBitmap() + output := NewSliceBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -427,7 +427,7 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { // Difference returns the difference of b and other. func (b *Bitmap) Difference(other *Bitmap) *Bitmap { - output := NewBitmap() + output := NewSliceBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -454,7 +454,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { // Xor returns the bitwise exclusive or of b and other. func (b *Bitmap) Xor(other *Bitmap) *Bitmap { - output := NewBitmap() + output := NewSliceBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -768,7 +768,7 @@ func (b *Bitmap) Check() error { // Flip performs a logical negate of the bits in the range [start,end]. func (b *Bitmap) Flip(start, end uint64) *Bitmap { - result := NewBitmap() + result := NewSliceBitmap() itr := b.Iterator() v, eof := itr.Next() //copy over previous bits. diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 7503ab247..7a80e0ec5 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1501,7 +1501,7 @@ func MakeBitmap(start []uint64) []uint64 { return b } func MakeLastBitSet() []uint64 { - obj := NewBitmap(65535) + obj := NewSliceBitmap(65535) c := obj.container(0) c.arrayToBitmap() return c.bitmap @@ -1714,9 +1714,9 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} - ba := NewBitmap() + ba := NewSliceBitmap() ba.conts.Put(0, ca) - ba2 := NewBitmap() + ba2 := NewSliceBitmap() var buf bytes.Buffer _, err := ba.WriteTo(&buf) if err != nil { @@ -1737,9 +1737,9 @@ func TestWriteReadBitmap(t *testing.T) { for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } - bb := NewBitmap() + bb := NewSliceBitmap() bb.conts.Put(0, cb) - bb2 := NewBitmap() + bb2 := NewSliceBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1760,9 +1760,9 @@ func TestWriteReadFullBitmap(t *testing.T) { for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } - bb := NewBitmap() + bb := NewSliceBitmap() bb.conts.Put(0, cb) - bb2 := NewBitmap() + bb2 := NewSliceBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1786,9 +1786,9 @@ func TestWriteReadFullBitmap(t *testing.T) { func TestWriteReadRun(t *testing.T) { cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} - br := NewBitmap() + br := NewSliceBitmap() br.conts.Put(0, cr) - br2 := NewBitmap() + br2 := NewSliceBitmap() var buf bytes.Buffer _, err := br.WriteTo(&buf) if err != nil { @@ -2089,7 +2089,7 @@ func TestXorBitmapRun(t *testing.T) { func TestIteratorArray(t *testing.T) { // use values that span two containers - b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := NewSliceBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) if !b.conts.Get(0).isArray() { t.Fatalf("wrong container type") } @@ -2136,7 +2136,7 @@ func TestIteratorBitmap(t *testing.T) { // use values that span two containers // this dataset will update to bitmap after enough Adds, // but won't update to RLE until Optimize() is called - b := NewBitmap() + b := NewSliceBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -2187,7 +2187,7 @@ func TestIteratorBitmap(t *testing.T) { } func TestIteratorRuns(t *testing.T) { - b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() if !b.conts.Get(0).isRun() { t.Fatalf("wrong container type") @@ -2403,7 +2403,7 @@ func TestRunBinSearch(t *testing.T) { } } func TestBitmap_RemoveEmptyContainers(t *testing.T) { - bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewSliceBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) if bm1.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") @@ -2416,13 +2416,13 @@ func TestBitmap_RemoveEmptyContainers(t *testing.T) { } func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { - bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewSliceBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) var buf bytes.Buffer if _, err := bm1.WriteTo(&buf); err != nil { t.Fatalf("Failure to write to bitmap buffer. ") } - bm0 := NewBitmap() + bm0 := NewSliceBitmap() bm0.UnmarshalBinary(buf.Bytes()) if bm0.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 97b562e08..ec31f3226 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -30,7 +30,7 @@ import ( ) func TestBitmapClone(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewSliceBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -48,7 +48,7 @@ func TestBitmapClone(t *testing.T) { } func TestContainerCount(t *testing.T) { - b := roaring.NewBitmap(65535) + b := roaring.NewSliceBitmap(65535) if b.Count() != b.CountRange(0, 65546) { t.Fatalf("Count != CountRange\n") @@ -144,7 +144,7 @@ func TestCountRange(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) { - b := roaring.NewBitmap(test.bitmap...) + b := roaring.NewSliceBitmap(test.bitmap...) actual := b.CountRange(test.start, test.end) if actual != test.exp { t.Errorf("got: %d, exp: %d", actual, test.exp) @@ -154,7 +154,7 @@ func TestCountRange(t *testing.T) { } func TestCheckBitmap(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewSliceBitmap() x := 0 for i := uint64(61000); i < 71000; i++ { x++ @@ -171,7 +171,7 @@ func TestCheckBitmap(t *testing.T) { } func TestCheckArray(t *testing.T) { - b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := roaring.NewSliceBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) err := b.Check() if err != nil { t.Fatalf("%v\n", err) @@ -179,7 +179,7 @@ func TestCheckArray(t *testing.T) { } func TestCheckRun(t *testing.T) { - b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() // convert to runs err := b.Check() if err != nil { @@ -187,7 +187,7 @@ func TestCheckRun(t *testing.T) { } } func TestCheckFullRun(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewSliceBitmap() for i := uint64(0); i < 2097152; i++ { if i%16384 == 0 { b.Optimize() // convert to runs @@ -208,7 +208,7 @@ func TestCheckFullRun(t *testing.T) { // Ensure that we can transition between runs and arrays when materializing the bitmap. func TestContainerTransitions(t *testing.T) { // [run, run][array][run] - b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) + b := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) b.Optimize() // convert to runs if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) { t.Fatalf("unexpected slice: %+v", b.Slice()) @@ -216,7 +216,7 @@ func TestContainerTransitions(t *testing.T) { // Test the case where last and first bits of adjoining containers are set. // [run][array][run] - b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) + b2 := roaring.NewSliceBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) b2.Optimize() // convert to runs if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) { t.Fatalf("unexpected slice: %+v", b2.Slice()) @@ -225,26 +225,26 @@ func TestContainerTransitions(t *testing.T) { // Ensure an empty bitmap returns false if checking for existence. func TestBitmap_Contains_Empty(t *testing.T) { - if roaring.NewBitmap().Contains(1000) { + if roaring.NewSliceBitmap().Contains(1000) { t.Fatal("expected false") } } // Ensure an empty bitmap does nothing when removing an element. func TestBitmap_Remove_Empty(t *testing.T) { - roaring.NewBitmap().Remove(1000) + roaring.NewSliceBitmap().Remove(1000) } // Ensure a bitmap can return a slice of values. func TestBitmap_Slice(t *testing.T) { - if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := roaring.NewSliceBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected slice: %+v", a) } } // Ensure an empty bitmap returns an empty slice of values. func TestBitmap_Slice_Empty(t *testing.T) { - if a := roaring.NewBitmap().Slice(); len(a) != 0 { + if a := roaring.NewSliceBitmap().Slice(); len(a) != 0 { t.Fatalf("unexpected slice: %+v", a) } } @@ -252,7 +252,7 @@ func TestBitmap_Slice_Empty(t *testing.T) { // Ensure a bitmap can return a slice of values within a range. // TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { - if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { + if a := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { t.Fatalf("unexpected slice: %+v", a) } } @@ -260,7 +260,7 @@ func TestBitmap_SliceRange(t *testing.T) { // Ensure a bitmap can loop over a set of values. func TestBitmap_ForEach(t *testing.T) { var a []uint64 - roaring.NewBitmap(1, 2, 3).ForEach(func(v uint64) { + roaring.NewSliceBitmap(1, 2, 3).ForEach(func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{1, 2, 3}) { @@ -271,7 +271,7 @@ func TestBitmap_ForEach(t *testing.T) { // Ensure a bitmap can loop over a set of values in a range. func TestBitmap_ForEachRange(t *testing.T) { var a []uint64 - roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { + roaring.NewSliceBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{2, 3}) { @@ -281,7 +281,7 @@ func TestBitmap_ForEachRange(t *testing.T) { // Ensure bitmap can return the highest value. func TestBitmap_Max(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() for i := uint64(1000); i <= 100000; i++ { bm.Add(i) @@ -297,7 +297,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { e := uint64(2010 * 1048576) start := s + (39314024 % 1048576) - bm0 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { start += 16384 @@ -315,7 +315,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { } func TestBitmap_BitmapCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) + bm0 := roaring.NewSliceBitmap(0, 2683177) for i := uint64(628); i < 2683301; i++ { bm0.Add(i) } @@ -343,20 +343,20 @@ func TestBitmap_BitmapCountRange(t *testing.T) { } func TestBitmap_ArrayCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177, 2683313) + bm0 := roaring.NewSliceBitmap(0, 2683177, 2683313) if n := bm0.CountRange(1, 2683313); n != 1 { t.Fatalf("unexpected n: %d", n) } } func TestBitmap_RunCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) + bm0 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) bm0.Optimize() // convert to runs if n := bm0.CountRange(15, 1000003); n != 5 { t.Fatalf("unexpected n: %d", n) } - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) bm1.Optimize() // convert to runs if n := bm1.CountRange(5, 12); n != 7 { t.Fatalf("unexpected n: %d", n) @@ -364,8 +364,8 @@ func TestBitmap_RunCountRange(t *testing.T) { } func TestBitmap_Intersectionz(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(0, 2683177) + bm1 := roaring.NewSliceBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -378,8 +378,8 @@ func TestBitmap_Intersectionz(t *testing.T) { } func TestBitmap_Union1(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(0, 2683177) + bm1 := roaring.NewSliceBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -402,8 +402,8 @@ func TestBitmap_Union1(t *testing.T) { } func TestBitmap_Intersection_Empty(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(0, 2683177) + bm1 := roaring.NewSliceBitmap() result := bm0.Intersect(bm1) if n := result.Count(); n != 0 { @@ -413,8 +413,8 @@ func TestBitmap_Intersection_Empty(t *testing.T) { } func TestBitmap_IntersectArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2683, 5005) - bm1 := roaring.NewBitmap(0, 2683, 2684, 5000) + bm0 := roaring.NewSliceBitmap(0, 1, 2683, 5005) + bm1 := roaring.NewSliceBitmap(0, 2683, 2684, 5000) result := bm0.Intersect(bm1) if n := result.Count(); n != 2 { @@ -423,12 +423,12 @@ func TestBitmap_IntersectArrayArray(t *testing.T) { } func TestBitmap_IntersectBitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap() for i := uint64(0); i < 65536; i += 2 { bm0.Add(i) } - bm1 := roaring.NewBitmap() + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i < 65536; i += 3 { bm1.Add(i) } @@ -441,9 +441,9 @@ func TestBitmap_IntersectBitmapBitmap(t *testing.T) { func TestBitmap_IntersectRunRun(t *testing.T) { // Intersect two runs that result in an array. - bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) + bm0 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) bm0.Optimize() // convert to runs - bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11) + bm1 := roaring.NewSliceBitmap(5, 6, 7, 8, 9, 10, 11) bm1.Optimize() // convert to runs result := bm0.Intersect(bm1) if n := result.Count(); n != 3 { @@ -451,7 +451,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } // Intersect two runs that result in a bitmap. - bm2 := roaring.NewBitmap() + bm2 := roaring.NewSliceBitmap() runLen := uint64(25) spaceLen := uint64(8) offset := (runLen / 2) + spaceLen @@ -461,7 +461,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } } bm2.Optimize() // convert to runs - bm3 := roaring.NewBitmap() + bm3 := roaring.NewSliceBitmap() runLen = uint64(32) spaceLen = uint64(1) for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { @@ -477,8 +477,8 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } func TestBitmap_Difference(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(0, 2683177) + bm1 := roaring.NewSliceBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -489,8 +489,8 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewSliceBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) + bm1 := roaring.NewSliceBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) result := bm0.Difference(bm1) if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) @@ -498,8 +498,8 @@ func TestBitmap_Difference2(t *testing.T) { } func TestBitmap_Difference_Empty(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(0, 2683177) + bm1 := roaring.NewSliceBitmap() result := bm0.Difference(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -507,8 +507,8 @@ func TestBitmap_Difference_Empty(t *testing.T) { } func TestBitmap_DifferenceArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20) - bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18) + bm0 := roaring.NewSliceBitmap(0, 4, 8, 12, 16, 20) + bm1 := roaring.NewSliceBitmap(1, 3, 6, 9, 12, 15, 18) result := bm0.Difference(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -516,9 +516,9 @@ func TestBitmap_DifferenceArrayArray(t *testing.T) { } func TestBitmap_DifferenceArrayRun(t *testing.T) { - bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) + bm0 := roaring.NewSliceBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) - bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) + bm1 := roaring.NewSliceBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) bm1.Optimize() // convert to runs result := bm0.Difference(bm1) if n := result.Count(); n != 6 { @@ -527,8 +527,8 @@ func TestBitmap_DifferenceArrayRun(t *testing.T) { } func TestBitmap_Union(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) result := bm0.Union(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -537,7 +537,7 @@ func TestBitmap_Union(t *testing.T) { func TestBitmap_Xor(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewBitmap(0, 1, 2, 3) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3) result := bm1.Xor(bm0) if n := result.Count(); n != 75011 { t.Fatalf("unexpected n: %d", n) @@ -555,8 +555,8 @@ func TestBitmap_Xor(t *testing.T) { } func TestBitmap_Xor_ArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) result := bm0.Xor(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -572,8 +572,8 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) { //empty array test func TestBitmap_Xor_Empty(t *testing.T) { - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) - empty := roaring.NewBitmap() + bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) + empty := roaring.NewSliceBitmap() result := bm1.Xor(empty) if n := result.Count(); n != 4 { @@ -581,8 +581,8 @@ func TestBitmap_Xor_Empty(t *testing.T) { } } func TestBitmap_Xor_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) } @@ -603,7 +603,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { t.Fatalf("test 3 unexpected n: %d", n) } - empty := roaring.NewBitmap() + empty := roaring.NewSliceBitmap() result = bm1.Xor(empty) if n := result.Count(); n != 5000 { t.Fatalf("unexpected n: %d", n) @@ -611,8 +611,8 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { } func TestBitmap_Xor_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap() + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) @@ -630,7 +630,7 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) { // Ensure bitmap contents alternate. func TestBitmap_Flip_Empty(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() results := bm.Flip(0, 10) if n := results.Count(); n != 11 { t.Fatalf("unexpected n: %d", n) @@ -643,7 +643,7 @@ func TestBitmap_Flip_Empty(t *testing.T) { // Test Subrange Flip should not affect bits outside of Range func TestBitmap_Flip_Array(t *testing.T) { - bm := roaring.NewBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) + bm := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) results := bm.Flip(0, 4) if !reflect.DeepEqual(results.Slice(), []uint64{8, 16, 32, 64, 128, 256, 512, 1024}) { t.Fatalf("unexpected %v ", results.Slice()) @@ -657,7 +657,7 @@ func TestBitmap_Flip_Array(t *testing.T) { // Ensure Flip works with underlying Bitmap container. func TestBitmap_Flip_Bitmap(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() size := uint64(10000) for i := uint64(0); i < size; i += 2 { bm.Add(i) @@ -674,7 +674,7 @@ func TestBitmap_Flip_Bitmap(t *testing.T) { // Verify Flip works correctly with in different regions of bitmap, beginning, middle, and end. func TestBitmap_Flip_After(t *testing.T) { - bm := roaring.NewBitmap(0, 2, 4, 8) + bm := roaring.NewSliceBitmap(0, 2, 4, 8) results := bm.Flip(9, 10) if !reflect.DeepEqual(results.Slice(), []uint64{0, 2, 4, 8, 9, 10}) { @@ -693,8 +693,8 @@ func TestBitmap_Flip_After(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewSliceBitmap(0, 1, 1000001, 1000002, 1000003) + bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) if n := bm0.IntersectionCount(bm1); n != 3 { t.Fatalf("unexpected n: %d", n) @@ -705,8 +705,8 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 3 { @@ -718,9 +718,9 @@ func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_RunRun(t *testing.T) { - bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) + bm0 := roaring.NewSliceBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) bm0.Optimize() // convert to runs - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 6 { @@ -732,11 +732,11 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { - bm0 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap() for i := uint64(3); i <= 1000006; i += 2 { bm0.Add(i) } - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 4 { @@ -748,8 +748,8 @@ func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i <= 10000; i += 2 { bm1.Add(i) } @@ -763,8 +763,8 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() - bm1 := roaring.NewBitmap() + bm0 := roaring.NewSliceBitmap() + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i <= 10000; i += 2 { bm0.Add(i) bm1.Add(i + 1) @@ -784,8 +784,8 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { } func TestBitmap_IntersectionCount_Mixed(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) - bm3 := roaring.NewBitmap(131072) + bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) + bm3 := roaring.NewSliceBitmap(131072) if n := bm0.IntersectionCount(bm0); n != bm0.Count() { t.Fatalf("unexpected n: %d", n) @@ -807,7 +807,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { quick.Check(func(a []uint64) bool { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() m := make(map[uint64]struct{}) // Add values to the bitmap and set. @@ -894,7 +894,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { quick.Check(func(a0, a1 []uint64) bool { // Create bitmap with initial values set. - bm := roaring.NewBitmap(a0...) + bm := roaring.NewSliceBitmap(a0...) set := make(map[uint64]struct{}) for _, v := range a0 { @@ -923,7 +923,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { data := buf.Bytes() // Create new bitmap from ops log data. - bm2 := roaring.NewBitmap() + bm2 := roaring.NewSliceBitmap() if err := bm2.UnmarshalBinary(data); err != nil { t.Fatal(err) } @@ -952,7 +952,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { // TODO duplicate for all container types func TestIterator(t *testing.T) { t.Run("bitmap", func(t *testing.T) { - itr := roaring.NewBitmap(1, 2, 3).Iterator() + itr := roaring.NewSliceBitmap(1, 2, 3).Iterator() itr.Seek(0) var a []uint64 @@ -966,13 +966,13 @@ func TestIterator(t *testing.T) { }) t.Run("run", func(t *testing.T) { - bm1 := roaring.NewBitmap() + bm1 := roaring.NewSliceBitmap() for i := uint64(0); i < 11; i += 1 { bm1.Add(i) } bm1.Optimize() - bm2 := roaring.NewBitmap() + bm2 := roaring.NewSliceBitmap() for i := uint64(0); i < 12; i += 1 { bm2.Add(i) } @@ -1005,7 +1005,7 @@ func TestIterator(t *testing.T) { // testBM creates a bitmap with 3 containers: array, bitmap, and run. func testBM() *roaring.Bitmap { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() //the array for i := uint64(0); i < 1024; i += 4 { bm.Add((1 << 16) + i) @@ -1068,19 +1068,19 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } { const max = (1 << 24) / 64 // Build bitmap with array container. - data.a = roaring.NewBitmap() + data.a = roaring.NewSliceBitmap() for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ { data.a.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. - data.b = roaring.NewBitmap() + data.b = roaring.NewSliceBitmap() for i, n := 0, MaxContainerVal/3; i < n; i++ { data.b.Add(uint64(i * 3)) } // build bitmap with run container - data.r = roaring.NewBitmap() + data.r = roaring.NewSliceBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { data.r.Add(uint64(i)) } @@ -1236,7 +1236,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() for col := uint64(0); col < pilosa.SliceWidth; col++ { bm.Add(col) } @@ -1245,7 +1245,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewSliceBitmap() for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { bm.Add(col) } From b301399dcd8eee72343bc4e0773ce7a60f9483b1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 7 Feb 2018 13:37:58 -0600 Subject: [PATCH 24/48] remove skiplist container implementation --- roaring/containers_skiplist.go | 96 ---------------------------------- roaring/roaring.go | 3 +- 2 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 roaring/containers_skiplist.go diff --git a/roaring/containers_skiplist.go b/roaring/containers_skiplist.go deleted file mode 100644 index 07d6dc870..000000000 --- a/roaring/containers_skiplist.go +++ /dev/null @@ -1,96 +0,0 @@ -package roaring - -import "github.com/pilosa/fast-skiplist" - -func NewSkipListContainers() *SkipListContainers { - return &SkipListContainers{ - list: skiplist.New(), - } -} - -type SkipListContainers struct { - list *skiplist.SkipList -} - -func (slc *SkipListContainers) Get(key uint64) *container { - var c *container - el := slc.list.Get(key) - if el != nil { - c = el.Value().(*container) - } - return c -} - -func (slc *SkipListContainers) Put(key uint64, c *container) { - slc.list.Set(key, c) -} - -func (slc *SkipListContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { -} - -func (slc *SkipListContainers) Remove(key uint64) { - slc.list.Remove(key) -} - -func (slc *SkipListContainers) GetOrCreate(key uint64) *container { - el := slc.list.Get(key) - if el == nil { - return slc.list.Set(key, newContainer()).Value().(*container) - } - return el.Value().(*container) -} - -func (slc *SkipListContainers) Clone() Containers { - nslc := NewSkipListContainers() - for c := slc.list.Front(); c != nil; c = c.Next() { - nslc.list.Set(c.Key(), c.Value().(*container).clone()) - } - return nslc -} - -func (slc *SkipListContainers) Last() (key uint64, c *container) { - if slc.list.Length() == 0 { - return 0, nil - } - el := slc.list.Last() - return el.Key(), el.Value().(*container) -} - -func (slc *SkipListContainers) Size() int { - return slc.list.Length() -} - -func (slc *SkipListContainers) Iterator(key uint64) (citer Contiterator, found bool) { - el := slc.list.GetNext(key) - if el != nil && el.Key() == key { - found = true - } - - return &SLCIterator{ - el: el, - }, found -} - -type SLCIterator struct { - started bool - el *skiplist.Element -} - -func (i *SLCIterator) Next() bool { - if i.el == nil { - return false - } - if !i.started { - i.started = true - return true - } - i.el = i.el.Next() - return i.el != nil -} - -func (i *SLCIterator) Value() (uint64, *container) { - if !i.started || i.el == nil { - return 0, nil - } - return i.el.Key(), i.el.Value().(*container) -} diff --git a/roaring/roaring.go b/roaring/roaring.go index 6feb90166..8ac2b4d4e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -166,8 +166,7 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { } func (b *Bitmap) add(v uint64) bool { - hb := highbits(v) - cont := b.conts.GetOrCreate(hb) + cont := b.conts.GetOrCreate(highbits(v)) return cont.add(lowbits(v)) } From 34c3497e20c3f44aae85825d7d107e5ba5ec8bcb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 7 Feb 2018 14:00:13 -0600 Subject: [PATCH 25/48] rebase and fix tests --- roaring/roaring_internal_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 7a80e0ec5..27636838e 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2254,7 +2254,7 @@ func TestIteratorVarious(t *testing.T) { exp uint64 }{ { - bm: NewBitmap(3, 4, 5), + bm: NewSliceBitmap(3, 4, 5), exp: 3, }, { @@ -2262,7 +2262,7 @@ func TestIteratorVarious(t *testing.T) { exp: 61221, }, { - bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + bm: NewSliceBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), exp: 7, }, } @@ -2669,7 +2669,7 @@ func bitmapVariousContainers() *Bitmap { bits = append(bits, bitCont(7, true, true, true)...) bits = append(bits, arrCont(8, true, true, true)...) bits = append(bits, rleCont(9, true, true, true)...) - bm := NewBitmap(bits...) + bm := NewSliceBitmap(bits...) bm.Optimize() return bm } From 4014f22802c4c378392d5bbd10e88b22176b3ea3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 11 May 2018 10:57:06 -0500 Subject: [PATCH 26/48] Rename (export) container -> Container --- roaring/btree.go | 24 +-- roaring/containers_btree.go | 20 +-- roaring/containers_slice.go | 20 +-- roaring/containers_test.go | 10 +- roaring/roaring.go | 259 +++++++++++++++---------------- roaring/roaring_helpers_test.go | 14 +- roaring/roaring_internal_test.go | 204 ++++++++++++------------ 7 files changed, 275 insertions(+), 276 deletions(-) diff --git a/roaring/btree.go b/roaring/btree.go index e3646113b..371a73e18 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -93,7 +93,7 @@ type ( de struct { // d element k uint64 - v *container + v *Container } // Enumerator captures the state of enumerating a tree. It is returned @@ -417,7 +417,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { // First returns the first item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k uint64, v *container) { +func (t *Tree) First() (k uint64, v *Container) { if q := t.first; q != nil { q := &q.d[0] k, v = q.k, q.v @@ -427,7 +427,7 @@ func (t *Tree) First() (k uint64, v *container) { // Get returns the value associated with k and true if it exists. Otherwise Get // returns (zero-value, false). -func (t *Tree) Get(k uint64) (v *container, ok bool) { +func (t *Tree) Get(k uint64) (v *Container, ok bool) { q := t.r if q == nil { return @@ -453,7 +453,7 @@ func (t *Tree) Get(k uint64) (v *container, ok bool) { } } -func (t *Tree) insert(q *d, i int, k uint64, v *container) *d { +func (t *Tree) insert(q *d, i int, k uint64, v *Container) *d { t.ver++ c := q.c if i < c { @@ -468,7 +468,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *container) *d { // Last returns the last item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k uint64, v *container) { +func (t *Tree) Last() (k uint64, v *Container) { if q := t.last; q != nil { q := &q.d[q.c-1] k, v = q.k, q.v @@ -481,7 +481,7 @@ func (t *Tree) Len() int { return t.c } -func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *container) { +func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *Container) { t.ver++ l, r := p.siblings(pi) @@ -577,7 +577,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) { } // Set sets the value associated with k. -func (t *Tree) Set(k uint64, v *container) { +func (t *Tree) Set(k uint64, v *Container) { //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) @@ -643,11 +643,11 @@ func (t *Tree) Set(k uint64, v *container) { // tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. -func (t *Tree) Put(k uint64, upd func(oldV *container, exists bool) (newV *container, write bool)) (oldV *container, written bool) { +func (t *Tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Container, write bool)) (oldV *Container, written bool) { pi := -1 var p *x q := t.r - var newV *container + var newV *Container if q == nil { // new KV pair in empty tree newV, written = upd(newV, false) @@ -710,7 +710,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *container, exists bool) (newV *conta } } -func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *container) { +func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *Container) { t.ver++ r := btDPool.Get().(*d) if q.n != nil { @@ -856,7 +856,7 @@ func (e *Enumerator) Close() { // Next returns the currently enumerated item, if it exists and moves to the // next item in the key collation order. If there is no item to return, err == // io.EOF is returned. -func (e *Enumerator) Next() (k uint64, v *container, err error) { +func (e *Enumerator) Next() (k uint64, v *Container, err error) { if err = e.err; err != nil { return } @@ -904,7 +904,7 @@ func (e *Enumerator) next() error { // Prev returns the currently enumerated item, if it exists and moves to the // previous item in the key collation order. If there is no item to return, err // == io.EOF is returned. -func (e *Enumerator) Prev() (k uint64, v *container, err error) { +func (e *Enumerator) Prev() (k uint64, v *Container, err error) { if err = e.err; err != nil { return } diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 134fb12a2..d3e317880 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -18,16 +18,16 @@ type BTreeContainers struct { tree *Tree lastKey uint64 - lastContainer *container + lastContainer *Container } -func (btc *BTreeContainers) Get(key uint64) *container { +func (btc *BTreeContainers) Get(key uint64) *Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer } - var c *container + var c *Container el, ok := btc.tree.Get(key) if ok { c = el @@ -37,7 +37,7 @@ func (btc *BTreeContainers) Get(key uint64) *container { return c } -func (btc *BTreeContainers) Put(key uint64, c *container) { +func (btc *BTreeContainers) Put(key uint64, c *Container) { // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. @@ -47,7 +47,7 @@ func (btc *BTreeContainers) Put(key uint64, c *container) { btc.tree.Set(key, c) } -func (u updater) update(oldV *container, exists bool) (*container, bool) { +func (u updater) update(oldV *Container, exists bool) (*Container, bool) { // update the existing container if exists { oldV.containerType = u.containerType @@ -55,7 +55,7 @@ func (u updater) update(oldV *container, exists bool) (*container, bool) { oldV.mapped = u.mapped return oldV, false } - return &container{ + return &Container{ containerType: u.containerType, n: u.n, mapped: u.mapped, @@ -79,7 +79,7 @@ func (btc *BTreeContainers) Remove(key uint64) { btc.tree.Delete(key) } -func (btc *BTreeContainers) GetOrCreate(key uint64) *container { +func (btc *BTreeContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer @@ -115,7 +115,7 @@ func (btc *BTreeContainers) Clone() Containers { return nbtc } -func (btc *BTreeContainers) Last() (key uint64, c *container) { +func (btc *BTreeContainers) Last() (key uint64, c *Container) { if btc.tree.Len() == 0 { return 0, nil } @@ -141,7 +141,7 @@ func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool type BTCIterator struct { e *Enumerator key uint64 - val *container + val *Container } func (i *BTCIterator) Next() bool { @@ -155,7 +155,7 @@ func (i *BTCIterator) Next() bool { return true } -func (i *BTCIterator) Value() (uint64, *container) { +func (i *BTCIterator) Value() (uint64, *Container) { if i.val == nil { return 0, nil } diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go index 391ed74e4..84cec96c1 100644 --- a/roaring/containers_slice.go +++ b/roaring/containers_slice.go @@ -6,12 +6,12 @@ func NewSliceContainers() *SliceContainers { type SliceContainers struct { keys []uint64 - containers []*container + containers []*Container lastKey uint64 - lastContainer *container + lastContainer *Container } -func (sc *SliceContainers) Get(key uint64) *container { +func (sc *SliceContainers) Get(key uint64) *Container { i := search64(sc.keys, key) if i < 0 { return nil @@ -19,7 +19,7 @@ func (sc *SliceContainers) Get(key uint64) *container { return sc.containers[i] } -func (sc *SliceContainers) Put(key uint64, c *container) { +func (sc *SliceContainers) Put(key uint64, c *Container) { i := search64(sc.keys, key) // If index is negative then there's not an exact match @@ -58,7 +58,7 @@ func (sc *SliceContainers) Remove(key uint64) { sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) } -func (sc *SliceContainers) insertAt(key uint64, c *container, i int) { +func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { sc.keys = append(sc.keys, 0) copy(sc.keys[i+1:], sc.keys[i:]) sc.keys[i] = key @@ -68,7 +68,7 @@ func (sc *SliceContainers) insertAt(key uint64, c *container, i int) { sc.containers[i] = c } -func (sc *SliceContainers) GetOrCreate(key uint64) *container { +func (sc *SliceContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. if key == sc.lastKey && sc.lastContainer != nil { return sc.lastContainer @@ -90,7 +90,7 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *container { func (sc *SliceContainers) Clone() Containers { other := NewSliceContainers() other.keys = make([]uint64, len(sc.keys)) - other.containers = make([]*container, len(sc.containers)) + other.containers = make([]*Container, len(sc.containers)) copy(other.keys, sc.keys) for i, c := range sc.containers { other.containers[i] = c.clone() @@ -98,7 +98,7 @@ func (sc *SliceContainers) Clone() Containers { return other } -func (sc *SliceContainers) Last() (key uint64, c *container) { +func (sc *SliceContainers) Last() (key uint64, c *Container) { if len(sc.keys) == 0 { return 0, nil } @@ -129,7 +129,7 @@ type SliceIterator struct { e *SliceContainers i int key uint64 - value *container + value *Container } func (si *SliceIterator) Next() bool { @@ -142,6 +142,6 @@ func (si *SliceIterator) Next() bool { return true } -func (si *SliceIterator) Value() (uint64, *container) { +func (si *SliceIterator) Value() (uint64, *Container) { return si.key, si.value } diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 4199fcbe5..55fa24ff4 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -22,8 +22,8 @@ func testContainersIterator(cs Containers, t *testing.T) { t.Fatal("Next() should be false for empty btc") } - cs.Put(1, &container{n: 1}) - cs.Put(2, &container{n: 2}) + cs.Put(1, &Container{n: 1}) + cs.Put(2, &Container{n: 2}) itr, found = cs.Iterator(0) if found { @@ -47,9 +47,9 @@ func testContainersIterator(cs Containers, t *testing.T) { t.Fatalf("itr should be done, but got true") } - cs.Put(3, &container{n: 3}) - cs.Put(5, &container{n: 5}) - cs.Put(6, &container{n: 6}) + cs.Put(3, &Container{n: 3}) + cs.Put(5, &Container{n: 5}) + cs.Put(6, &Container{n: 6}) itr, found = cs.Iterator(3) if !itr.Next() { diff --git a/roaring/roaring.go b/roaring/roaring.go index f25b3faca..b1f25393e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -21,8 +21,8 @@ import ( "fmt" "hash/fnv" "io" - "reflect" "math/bits" + "reflect" "sort" "unsafe" ) @@ -66,10 +66,10 @@ const ( type Containers interface { // Get returns nil if the key does not exist. - Get(key uint64) *container + Get(key uint64) *Container // Put adds the container at key. - Put(key uint64, c *container) + Put(key uint64, c *Container) // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. @@ -79,13 +79,13 @@ type Containers interface { Remove(key uint64) // GetOrCreate returns the container at key, creating a new empty container if necessary. - GetOrCreate(key uint64) *container + GetOrCreate(key uint64) *Container // Clone does a deep copy of Containers, including cloning all containers contained. Clone() Containers // Last returns the highest key and associated container. - Last() (key uint64, c *container) + Last() (key uint64, c *Container) // Size returns the number of containers stored. Size() int @@ -98,7 +98,7 @@ type Containers interface { type Contiterator interface { Next() bool - Value() (uint64, *container) + Value() (uint64, *Container) } // Bitmap represents a roaring bitmap. @@ -341,7 +341,7 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { } // container returns the container with the given key. -func (b *Bitmap) container(key uint64) *container { +func (b *Bitmap) container(key uint64) *Container { return b.conts.Get(key) } @@ -805,7 +805,7 @@ type Iterator struct { bitmap *Bitmap citer Contiterator key uint64 - c *container + c *Container j, k int // i: container; j: array index, bit index, or run index; k: offset within the run } @@ -996,17 +996,17 @@ const ArrayMaxSize = 4096 // RunMaxSize represents the maximum size of run length encoded containers. const RunMaxSize = 2048 -// container represents a container for uint16 integers. +// Container represents a Container for uint16 integers. // // These are used for storing the low bits of numbers in larger sets of uint64. -// The high bits are stored in a container's key which is tracked by a separate -// data structure. Integers in a container can be encoded in one of three ways - -// the encoding used is usually whichever is most compact, though any container +// The high bits are stored in a Container's key which is tracked by a separate +// data structure. Integers in a Container can be encoded in one of three ways - +// the encoding used is usually whichever is most compact, though any Container // type should be able to encode any set of integers safely. For containers with // less than 4,096 values, an array is often used. Containers with long runs of // integers would use run length encoding, and more random data usually uses // bitmap encoding. -type container struct { +type Container struct { mapped bool // mapped directly to a byte slice when true containerType byte // array, bitmap, or run n int // number of integers in container @@ -1026,22 +1026,22 @@ func (iv interval16) runlen() int { } // newContainer returns a new instance of container. -func newContainer() *container { - return &container{containerType: ContainerArray} +func newContainer() *Container { + return &Container{containerType: ContainerArray} } // isArray returns true if the container is an array container. -func (c *container) isArray() bool { +func (c *Container) isArray() bool { return c.containerType == ContainerArray } // isBitmap returns true if the container is a bitmap container. -func (c *container) isBitmap() bool { +func (c *Container) isBitmap() bool { return c.containerType == ContainerBitmap } // isRun returns true if the container is a run-length-encoded container. -func (c *container) isRun() bool { +func (c *Container) isRun() bool { return c.containerType == ContainerRun } @@ -1049,7 +1049,7 @@ func (c *container) isRun() bool { // // This is performed when altering the container since its contents could be // pointing at a read-only mmap. -func (c *container) unmap() { +func (c *Container) unmap() { if !c.mapped { return } @@ -1072,12 +1072,12 @@ func (c *container) unmap() { } // count counts all bits in the container. -func (c *container) count() (n int) { +func (c *Container) count() (n int) { return c.countRange(0, maxContainerVal+1) } // countRange counts the number of bits set between [start, end). -func (c *container) countRange(start, end int) (n int) { +func (c *Container) countRange(start, end int) (n int) { if c.isArray() { return c.arrayCountRange(start, end) } else if c.isRun() { @@ -1086,7 +1086,7 @@ func (c *container) countRange(start, end int) (n int) { return c.bitmapCountRange(start, end) } -func (c *container) arrayCountRange(start, end int) (n int) { +func (c *Container) arrayCountRange(start, end int) (n int) { i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start }) for ; i < len(c.array); i++ { v := int(c.array[i]) @@ -1098,7 +1098,7 @@ func (c *container) arrayCountRange(start, end int) (n int) { return n } -func (c *container) bitmapCountRange(start, end int) int { +func (c *Container) bitmapCountRange(start, end int) int { var n uint64 i, j := start/64, end/64 // Special case when start and end fall in the same word. @@ -1128,7 +1128,7 @@ func (c *container) bitmapCountRange(start, end int) int { return int(n) } -func (c *container) runCountRange(start, end int) (n int) { +func (c *Container) runCountRange(start, end int) (n int) { for _, iv := range c.runs { // iv is before range if int(iv.last) < start { @@ -1159,7 +1159,7 @@ func (c *container) runCountRange(start, end int) (n int) { } // add adds a value to the container. -func (c *container) add(v uint16) (added bool) { +func (c *Container) add(v uint16) (added bool) { if c.isArray() { added = c.arrayAdd(v) @@ -1174,7 +1174,7 @@ func (c *container) add(v uint16) (added bool) { return added } -func (c *container) arrayAdd(v uint16) bool { +func (c *Container) arrayAdd(v uint16) bool { // Optimize appending to the end of an array container. if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v { c.unmap() @@ -1204,7 +1204,7 @@ func (c *container) arrayAdd(v uint16) bool { } -func (c *container) bitmapAdd(v uint16) bool { +func (c *Container) bitmapAdd(v uint16) bool { if c.bitmapContains(v) { return false } @@ -1213,7 +1213,7 @@ func (c *container) bitmapAdd(v uint16) bool { return true } -func (c *container) runAdd(v uint16) bool { +func (c *Container) runAdd(v uint16) bool { if len(c.runs) == 0 { c.unmap() c.runs = []interval16{{start: v, last: v}} @@ -1260,7 +1260,7 @@ func (c *container) runAdd(v uint16) bool { } // contains returns true if v is in the container. -func (c *container) contains(v uint16) bool { +func (c *Container) contains(v uint16) bool { if c.isArray() { return c.arrayContains(v) } else if c.isRun() { @@ -1270,7 +1270,7 @@ func (c *container) contains(v uint16) bool { } } -func (c *container) bitmapCountRuns() (r int) { +func (c *Container) bitmapCountRuns() (r int) { for i := 0; i < 1023; i++ { v, v1 := c.bitmap[i], c.bitmap[i+1] r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1)) @@ -1280,7 +1280,7 @@ func (c *container) bitmapCountRuns() (r int) { return r } -func (c *container) arrayCountRuns() (r int) { +func (c *Container) arrayCountRuns() (r int) { prev := -2 for _, v := range c.array { if prev+1 != int(v) { @@ -1291,7 +1291,7 @@ func (c *container) arrayCountRuns() (r int) { return r } -func (c *container) countRuns() (r int) { +func (c *Container) countRuns() (r int) { if c.isArray() { return c.arrayCountRuns() } else if c.isBitmap() { @@ -1306,7 +1306,7 @@ func (c *container) countRuns() (r int) { // Optimize converts the container to the type which will take up the least // amount of space. -func (c *container) Optimize() { +func (c *Container) Optimize() { if c.n == 0 { return } @@ -1343,11 +1343,11 @@ func (c *container) Optimize() { } } -func (c *container) arrayContains(v uint16) bool { +func (c *Container) arrayContains(v uint16) bool { return search32(c.array, v) >= 0 } -func (c *container) bitmapContains(v uint16) bool { +func (c *Container) bitmapContains(v uint16) bool { return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0 } @@ -1365,13 +1365,13 @@ func binSearchRuns(v uint16, a []interval16) (int, bool) { // runContains determines if v is in the container assuming c is a run // container. -func (c *container) runContains(v uint16) bool { +func (c *Container) runContains(v uint16) bool { _, found := binSearchRuns(v, c.runs) return found } // remove removes a value from the container. -func (c *container) remove(v uint16) (removed bool) { +func (c *Container) remove(v uint16) (removed bool) { if c.isArray() { removed = c.arrayRemove(v) } else if c.isRun() { @@ -1385,7 +1385,7 @@ func (c *container) remove(v uint16) (removed bool) { return removed } -func (c *container) arrayRemove(v uint16) bool { +func (c *Container) arrayRemove(v uint16) bool { i := search32(c.array, v) if i < 0 { return false @@ -1396,7 +1396,7 @@ func (c *container) arrayRemove(v uint16) bool { return true } -func (c *container) bitmapRemove(v uint16) bool { +func (c *Container) bitmapRemove(v uint16) bool { if !c.bitmapContains(v) { return false } @@ -1414,7 +1414,7 @@ func (c *container) bitmapRemove(v uint16) bool { } // runRemove removes v from a run container, and returns true if v was removed. -func (c *container) runRemove(v uint16) bool { +func (c *Container) runRemove(v uint16) bool { i, contains := binSearchRuns(v, c.runs) if !contains { return false @@ -1435,7 +1435,7 @@ func (c *container) runRemove(v uint16) bool { } // max returns the maximum value in the container. -func (c *container) max() uint16 { +func (c *Container) max() uint16 { if c.isArray() { return c.arrayMax() } else if c.isRun() { @@ -1445,14 +1445,14 @@ func (c *container) max() uint16 { } } -func (c *container) arrayMax() uint16 { +func (c *Container) arrayMax() uint16 { if len(c.array) == 0 { return 0 // probably hiding some ugly bug but it prevents a crash } return c.array[len(c.array)-1] } -func (c *container) bitmapMax() uint16 { +func (c *Container) bitmapMax() uint16 { // Search bitmap in reverse order. for i := len(c.bitmap) - 1; i >= 0; i-- { // If value is zero then skip. @@ -1474,7 +1474,7 @@ func (c *container) bitmapMax() uint16 { return 0 } -func (c *container) runMax() uint16 { +func (c *Container) runMax() uint16 { if len(c.runs) == 0 { return 0 } @@ -1482,7 +1482,7 @@ func (c *container) runMax() uint16 { } // bitmapToArray converts from bitmap format to array format. -func (c *container) bitmapToArray() { +func (c *Container) bitmapToArray() { c.array = make([]uint16, 0, c.n) c.containerType = ContainerArray @@ -1505,7 +1505,7 @@ func (c *container) bitmapToArray() { } // arrayToBitmap converts from array format to bitmap format. -func (c *container) arrayToBitmap() { +func (c *Container) arrayToBitmap() { c.bitmap = make([]uint64, bitmapN) c.containerType = ContainerBitmap @@ -1524,7 +1524,7 @@ func (c *container) arrayToBitmap() { } // runToBitmap converts from RLE format to bitmap format. -func (c *container) runToBitmap() { +func (c *Container) runToBitmap() { c.bitmap = make([]uint64, bitmapN) c.containerType = ContainerBitmap @@ -1547,7 +1547,7 @@ func (c *container) runToBitmap() { } // bitmapToRun converts from bitmap format to RLE format. -func (c *container) bitmapToRun() { +func (c *Container) bitmapToRun() { c.containerType = ContainerRun // return early if empty if c.n == 0 { @@ -1603,7 +1603,7 @@ func (c *container) bitmapToRun() { } // arrayToRun converts from array format to RLE format. -func (c *container) arrayToRun() { +func (c *Container) arrayToRun() { c.containerType = ContainerRun // return early if empty if c.n == 0 { @@ -1630,7 +1630,7 @@ func (c *container) arrayToRun() { } // runToArray converts from RLE format to array format. -func (c *container) runToArray() { +func (c *Container) runToArray() { c.containerType = ContainerArray c.array = make([]uint16, 0, c.n) @@ -1651,8 +1651,8 @@ func (c *container) runToArray() { } // clone returns a copy of c. -func (c *container) clone() *container { - other := &container{n: c.n, containerType: c.containerType} +func (c *Container) clone() *Container { + other := &Container{n: c.n, containerType: c.containerType} switch c.containerType { case ContainerArray: @@ -1669,7 +1669,7 @@ func (c *container) clone() *container { } // WriteTo writes c to w. -func (c *container) WriteTo(w io.Writer) (n int64, err error) { +func (c *Container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { return c.arrayWriteTo(w) } else if c.isRun() { @@ -1679,7 +1679,7 @@ func (c *container) WriteTo(w io.Writer) (n int64, err error) { } } -func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { +func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { if len(c.array) == 0 { return 0, nil } @@ -1695,13 +1695,13 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { return int64(nn), err } -func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) { +func (c *Container) bitmapWriteTo(w io.Writer) (n int64, err error) { // Write sizeof(uint64) * bitmapN bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)]) return int64(nn), err } -func (c *container) runWriteTo(w io.Writer) (n int64, err error) { +func (c *Container) runWriteTo(w io.Writer) (n int64, err error) { if len(c.runs) == 0 { return 0, nil } @@ -1716,7 +1716,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) { } // size returns the encoded size of the container, in bytes. -func (c *container) size() int { +func (c *Container) size() int { if c.isArray() { return len(c.array) * 2 // sizeof(uint16) } else if c.isRun() { @@ -1727,7 +1727,7 @@ func (c *container) size() int { } // info returns the current stats about the container. -func (c *container) info() ContainerInfo { +func (c *Container) info() ContainerInfo { info := ContainerInfo{N: c.n} if c.isArray() { @@ -1755,7 +1755,7 @@ func (c *container) info() ContainerInfo { } // check performs a consistency check on the container. -func (c *container) check() error { +func (c *Container) check() error { var a ErrorList if c.isArray() { @@ -1795,7 +1795,7 @@ type ContainerInfo struct { // flip returns a new container containing the inverse of all // bits in a. -func flip(a *container) *container { +func flip(a *Container) *Container { if a.isArray() { return flipArray(a) } else if a.isRun() { @@ -1805,15 +1805,15 @@ func flip(a *container) *container { } } -func flipArray(b *container) *container { +func flipArray(b *Container) *Container { // TODO: actually implement this x := b.clone() x.arrayToBitmap() return flipBitmap(x) } -func flipBitmap(b *container) *container { - other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} +func flipBitmap(b *Container) *Container { + other := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} for i, bitmap := range b.bitmap { other.bitmap[i] = ^bitmap @@ -1823,14 +1823,14 @@ func flipBitmap(b *container) *container { return other } -func flipRun(b *container) *container { +func flipRun(b *Container) *Container { // TODO: actually implement this x := b.clone() x.runToBitmap() return flipBitmap(x) } -func intersectionCount(a, b *container) int { +func intersectionCount(a, b *Container) int { if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) @@ -1858,7 +1858,7 @@ func intersectionCount(a, b *container) int { } } -func intersectionCountArrayArray(a, b *container) (n int) { +func intersectionCountArrayArray(a, b *Container) (n int) { na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1874,7 +1874,7 @@ func intersectionCountArrayArray(a, b *container) (n int) { return n } -func intersectionCountArrayRun(a, b *container) (n int) { +func intersectionCountArrayRun(a, b *Container) (n int) { na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -1890,7 +1890,7 @@ func intersectionCountArrayRun(a, b *container) (n int) { return n } -func intersectionCountRunRun(a, b *container) (n int) { +func intersectionCountRunRun(a, b *Container) (n int) { na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -1921,14 +1921,14 @@ func intersectionCountRunRun(a, b *container) (n int) { return } -func intersectionCountBitmapRun(a, b *container) (n int) { +func intersectionCountBitmapRun(a, b *Container) (n int) { for _, iv := range b.runs { n += a.bitmapCountRange(int(iv.start), int(iv.last)+1) } return n } -func intersectionCountArrayBitmap(a, b *container) (n int) { +func intersectionCountArrayBitmap(a, b *Container) (n int) { ln := len(b.bitmap) for _, val := range a.array { i := int(val >> 6) @@ -1941,11 +1941,11 @@ func intersectionCountArrayBitmap(a, b *container) (n int) { return n } -func intersectionCountBitmapBitmap(a, b *container) (n int) { +func intersectionCountBitmapBitmap(a, b *Container) (n int) { return int(popcountAndSlice(a.bitmap, b.bitmap)) } -func intersect(a, b *container) *container { +func intersect(a, b *Container) *Container { if a.isArray() { if b.isArray() { return intersectArrayArray(a, b) @@ -1973,8 +1973,8 @@ func intersect(a, b *container) *container { } } -func intersectArrayArray(a, b *container) *container { - output := &container{containerType: ContainerArray} +func intersectArrayArray(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1994,8 +1994,8 @@ func intersectArrayArray(a, b *container) *container { // intersectArrayRun computes the intersect of an array container and a run // container. The return is always an array container (since it's guaranteed to // be low-cardinality) -func intersectArrayRun(a, b *container) *container { - output := &container{containerType: ContainerArray} +func intersectArrayRun(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -2013,8 +2013,8 @@ func intersectArrayRun(a, b *container) *container { } // intersectRunRun computes the intersect of two run containers. -func intersectRunRun(a, b *container) *container { - output := &container{containerType: ContainerRun} +func intersectRunRun(a, b *Container) *Container { + output := &Container{containerType: ContainerRun} na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -2052,11 +2052,11 @@ func intersectRunRun(a, b *container) *container { // intersectBitmapRun returns an array container if the run container's // cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container. -func intersectBitmapRun(a, b *container) *container { - var output *container +func intersectBitmapRun(a, b *Container) *Container { + var output *Container if b.n < ArrayMaxSize { // output is array container - output = &container{containerType: ContainerArray} + output = &Container{containerType: ContainerArray} for _, iv := range b.runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { @@ -2073,7 +2073,7 @@ func intersectBitmapRun(a, b *container) *container { // right now this iterates through the runs and sets integers in the // bitmap that are in the runs. alternately, we could zero out ranges in // the bitmap which are between runs. - output = &container{ + output = &Container{ bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap, } @@ -2115,8 +2115,8 @@ func intersectBitmapRun(a, b *container) *container { return output } -func intersectArrayBitmap(a, b *container) *container { - output := &container{containerType: ContainerArray} +func intersectArrayBitmap(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2130,8 +2130,8 @@ func intersectArrayBitmap(a, b *container) *container { return output } -func intersectBitmapBitmap(a, b *container) *container { - output := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} +func intersectBitmapBitmap(a, b *Container) *Container { + output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} for i := range a.bitmap { v := a.bitmap[i] & b.bitmap[i] @@ -2143,7 +2143,7 @@ func intersectBitmapBitmap(a, b *container) *container { return output } -func union(a, b *container) *container { +func union(a, b *Container) *Container { if a.isArray() { if b.isArray() { return unionArrayArray(a, b) @@ -2171,8 +2171,8 @@ func union(a, b *container) *container { } } -func unionArrayArray(a, b *container) *container { - output := &container{containerType: ContainerArray} +func unionArrayArray(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; ; { if i >= na && j >= nb { @@ -2204,11 +2204,11 @@ func unionArrayArray(a, b *container) *container { // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. -func unionArrayRun(a, b *container) *container { +func unionArrayRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.clone() } - output := &container{containerType: ContainerRun} + output := &Container{containerType: ContainerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -2241,7 +2241,7 @@ func unionArrayRun(a, b *container) *container { // interval is earlier than the start of the last interval in the list of runs. // Its return value is the amount by which the cardinality of the container was // increased. -func (c *container) runAppendInterval(v interval16) int { +func (c *Container) runAppendInterval(v interval16) int { if len(c.runs) == 0 { c.runs = append(c.runs, v) return int(v.last-v.start) + 1 @@ -2261,7 +2261,7 @@ func (c *container) runAppendInterval(v interval16) int { return 0 } -func unionRunRun(a, b *container) *container { +func unionRunRun(a, b *Container) *Container { if a.n == maxContainerVal+1 { return a.clone() } @@ -2269,7 +2269,7 @@ func unionRunRun(a, b *container) *container { return b.clone() } na, nb := len(a.runs), len(b.runs) - output := &container{ + output := &Container{ runs: make([]interval16, 0, na+nb), containerType: ContainerRun, } @@ -2295,7 +2295,7 @@ func unionRunRun(a, b *container) *container { return output } -func unionBitmapRun(a, b *container) *container { +func unionBitmapRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.clone() } @@ -2312,7 +2312,7 @@ func unionBitmapRun(a, b *container) *container { const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j) (c must be a bitmap container) -func (c *container) bitmapSetRange(i, j uint64) { +func (c *Container) bitmapSetRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) @@ -2335,7 +2335,7 @@ func (c *container) bitmapSetRange(i, j uint64) { } // xor's all bits in [i, j) with all true (c must be a bitmap container). -func (c *container) bitmapXorRange(i, j uint64) { +func (c *Container) bitmapXorRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) @@ -2360,7 +2360,7 @@ func (c *container) bitmapXorRange(i, j uint64) { } // zeroes all bits in [i, j) (c must be a bitmap container) -func (c *container) bitmapZeroRange(i, j uint64) { +func (c *Container) bitmapZeroRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) @@ -2380,7 +2380,7 @@ func (c *container) bitmapZeroRange(i, j uint64) { } } -func unionArrayBitmap(a, b *container) *container { +func unionArrayBitmap(a, b *Container) *Container { output := b.clone() for _, v := range a.array { if !output.bitmapContains(v) { @@ -2391,8 +2391,8 @@ func unionArrayBitmap(a, b *container) *container { return output } -func unionBitmapBitmap(a, b *container) *container { - output := &container{ +func unionBitmapBitmap(a, b *Container) *Container { + output := &Container{ bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap, } @@ -2406,7 +2406,7 @@ func unionBitmapBitmap(a, b *container) *container { return output } -func difference(a, b *container) *container { +func difference(a, b *Container) *Container { if a.isArray() { if b.isArray() { return differenceArrayArray(a, b) @@ -2435,8 +2435,8 @@ func difference(a, b *container) *container { } // differenceArrayArray computes the difference bween two arrays. -func differenceArrayArray(a, b *container) *container { - output := &container{containerType: ContainerArray} +func differenceArrayArray(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na; { va := a.array[i] @@ -2460,14 +2460,14 @@ func differenceArrayArray(a, b *container) *container { } // differenceArrayRun computes the difference of an array from a run. -func differenceArrayRun(a, b *container) *container { +func differenceArrayRun(a, b *Container) *Container { // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { if a.n == 0 || b.n == 0 { return a.clone() } - output := &container{array: make([]uint16, 0, a.n), containerType: ContainerArray} + output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray} // cardinality upper bound: card(A) i := 0 // array index @@ -2517,7 +2517,7 @@ func differenceArrayRun(a, b *container) *container { } // differenceBitmapRun computes the difference of an bitmap from a run. -func differenceBitmapRun(a, b *container) *container { +func differenceBitmapRun(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.clone() } @@ -2531,11 +2531,11 @@ func differenceBitmapRun(a, b *container) *container { // differenceRunArray subtracts the bits in an array container from a run // container. -func differenceRunArray(a, b *container) *container { +func differenceRunArray(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.clone() } - output := &container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} + output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} bidx := 0 vb := b.array[bidx] @@ -2586,12 +2586,12 @@ RUNLOOP: } // differenceRunBitmap computes the difference of an run from a bitmap. -func differenceRunBitmap(a, b *container) *container { +func differenceRunBitmap(a, b *Container) *Container { // If a is full, difference is the flip of b. if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return flipBitmap(b) } - output := &container{containerType: ContainerRun} + output := &Container{containerType: ContainerRun} output.n = a.n if len(a.runs) == 0 { return output @@ -2643,7 +2643,7 @@ func differenceRunBitmap(a, b *container) *container { } // differenceRunRun computes the difference of two runs. -func differenceRunRun(a, b *container) *container { +func differenceRunRun(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.clone() } @@ -2657,7 +2657,7 @@ func differenceRunRun(a, b *container) *container { alen := len(a.runs) blen := len(b.runs) - output := &container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else + output := &Container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -2706,8 +2706,8 @@ func differenceRunRun(a, b *container) *container { return output } -func differenceArrayBitmap(a, b *container) *container { - output := &container{containerType: ContainerArray} +func differenceArrayBitmap(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2722,7 +2722,7 @@ func differenceArrayBitmap(a, b *container) *container { return output } -func differenceBitmapArray(a, b *container) *container { +func differenceBitmapArray(a, b *Container) *Container { output := a.clone() for _, v := range b.array { @@ -2737,8 +2737,8 @@ func differenceBitmapArray(a, b *container) *container { return output } -func differenceBitmapBitmap(a, b *container) *container { - output := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} +func differenceBitmapBitmap(a, b *Container) *Container { + output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} for i := range a.bitmap { v := a.bitmap[i] & (^b.bitmap[i]) @@ -2752,7 +2752,7 @@ func differenceBitmapBitmap(a, b *container) *container { return output } -func xor(a, b *container) *container { +func xor(a, b *Container) *Container { if a.isArray() { if b.isArray() { return xorArrayArray(a, b) @@ -2780,8 +2780,8 @@ func xor(a, b *container) *container { } } -func xorArrayArray(a, b *container) *container { - output := &container{containerType: ContainerArray} +func xorArrayArray(a, b *Container) *Container { + output := &Container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { @@ -2809,7 +2809,7 @@ func xorArrayArray(a, b *container) *container { return output } -func xorArrayBitmap(a, b *container) *container { +func xorArrayBitmap(a, b *Container) *Container { output := b.clone() for _, v := range a.array { if b.bitmapContains(v) { @@ -2828,8 +2828,8 @@ func xorArrayBitmap(a, b *container) *container { return output } -func xorBitmapBitmap(a, b *container) *container { - output := &container{ +func xorBitmapBitmap(a, b *Container) *Container { + output := &Container{ bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap, } @@ -3038,8 +3038,8 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) { } // xorArrayRun computes the exclusive or of an array and a run container. -func xorArrayRun(a, b *container) *container { - output := &container{containerType: ContainerRun} +func xorArrayRun(a, b *Container) *Container { + output := &Container{containerType: ContainerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -3200,7 +3200,7 @@ type xorstm struct { } // xorRunRun computes the exclusive or of two run containers. -func xorRunRun(a, b *container) *container { +func xorRunRun(a, b *Container) *Container { na, nb := len(a.runs), len(b.runs) if na == 0 { return b.clone() @@ -3208,7 +3208,7 @@ func xorRunRun(a, b *container) *container { if nb == 0 { return a.clone() } - output := &container{containerType: ContainerRun} + output := &Container{containerType: ContainerRun} lastI, lastJ := -1, -1 @@ -3248,7 +3248,7 @@ func xorRunRun(a, b *container) *container { } // xorRunRun computes the exclusive or of a bitmap and a run container. -func xorBitmapRun(a, b *container) *container { +func xorBitmapRun(a, b *Container) *Container { output := a.clone() for j := 0; j < len(b.runs); j++ { output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) @@ -3333,4 +3333,3 @@ func popcountXorSlice(s, m []uint64) uint64 { } return cnt } - diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 417ac7db6..24db8e1ca 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -230,8 +230,8 @@ type testOp struct { exp string } -func doContainer(containerType byte, data interface{}) *container { - c := &container{ +func doContainer(containerType byte, data interface{}) *Container { + c := &Container{ containerType: containerType, } @@ -248,12 +248,12 @@ func doContainer(containerType byte, data interface{}) *container { return c } -func setupContainerTests() map[byte]map[string]*container { +func setupContainerTests() map[byte]map[string]*Container { - cts := make(map[byte]map[string]*container) + cts := make(map[byte]map[string]*Container) // array containers - cts[ContainerArray] = map[string]*container{ + cts[ContainerArray] = map[string]*Container{ "empty": doContainer(ContainerArray, arrayEmpty()), "full": doContainer(ContainerArray, arrayFull()), "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), @@ -267,7 +267,7 @@ func setupContainerTests() map[byte]map[string]*container { } // bitmap containers - cts[ContainerBitmap] = map[string]*container{ + cts[ContainerBitmap] = map[string]*Container{ "empty": doContainer(ContainerBitmap, bitmapEmpty()), "full": doContainer(ContainerBitmap, bitmapFull()), "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), @@ -281,7 +281,7 @@ func setupContainerTests() map[byte]map[string]*container { } // run containers - cts[ContainerRun] = map[string]*container{ + cts[ContainerRun] = map[string]*Container{ "empty": doContainer(ContainerRun, runEmpty()), "full": doContainer(ContainerRun, runFull()), "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index c41eb1dc6..2d3fe10bf 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -28,12 +28,12 @@ func (iv interval16) String() string { return fmt.Sprintf("[%d, %d]", iv.start, iv.last) } -func (c *container) String() string { +func (c *Container) String() string { return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.containerType) } func TestRunAppendInterval(t *testing.T) { - a := container{containerType: ContainerRun} + a := Container{containerType: ContainerRun} tests := []struct { base []interval16 app interval16 @@ -82,7 +82,7 @@ func TestInterval16RunLen(t *testing.T) { } func TestContainerRunAdd(t *testing.T) { - c := container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: ContainerRun} tests := []struct { op uint16 exp []interval16 @@ -113,7 +113,7 @@ func TestContainerRunAdd(t *testing.T) { } func TestContainerRunAdd2(t *testing.T) { - c := container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: ContainerRun} ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs) @@ -128,7 +128,7 @@ func TestContainerRunAdd2(t *testing.T) { } func TestRunCountRange(t *testing.T) { - c := container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: ContainerRun} cnt := c.runCountRange(2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) @@ -181,7 +181,7 @@ func TestRunCountRange(t *testing.T) { } func TestRunContains(t *testing.T) { - c := container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: ContainerRun} if c.runContains(5) { t.Fatalf("empty run container should not contain 5") } @@ -203,7 +203,7 @@ func TestRunContains(t *testing.T) { } func TestBitmapCountRange(t *testing.T) { - c := container{containerType: ContainerBitmap} + c := Container{containerType: ContainerBitmap} tests := []struct { start int end int @@ -228,7 +228,7 @@ func TestBitmapCountRange(t *testing.T) { } func TestIntersectionCountArrayBitmap3(t *testing.T) { - a, b := &container{}, &container{} + a, b := &Container{}, &Container{} a.containerType = ContainerBitmap a.bitmap = getFullBitmap() a.n = maxContainerVal + 1 @@ -255,7 +255,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) { } func TestIntersectionCountArrayBitmap2(t *testing.T) { - a, b := &container{}, &container{} + a, b := &Container{}, &Container{} tests := []struct { array []uint16 bitmap []uint64 @@ -301,7 +301,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} tests := []struct { op uint16 exp []interval16 @@ -335,13 +335,13 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} max := c.max() if max != 16 { t.Fatalf("max for %v should be 16", c.runs) } - c = container{runs: []interval16{}} + c = Container{runs: []interval16{}} max = c.max() if max != 0 { t.Fatalf("max for %v should be 0", c.runs) @@ -349,8 +349,8 @@ func TestRunMax(t *testing.T) { } func TestIntersectionCountArrayRun(t *testing.T) { - a := &container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} - b := &container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + a := &Container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} + b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -359,16 +359,16 @@ func TestIntersectionCountArrayRun(t *testing.T) { } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} - b := &container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} + a := &Container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} + b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) if ret != 1 { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) } - a = &container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + a = &Container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -377,8 +377,8 @@ func TestIntersectionCountBitmapRun(t *testing.T) { } func TestIntersectionCountRunRun(t *testing.T) { - a := &container{} - b := &container{} + a := &Container{} + b := &Container{} tests := []struct { aruns []interval16 bruns []interval16 @@ -428,8 +428,8 @@ func TestIntersectionCountRunRun(t *testing.T) { } func TestIntersectArrayRun(t *testing.T) { - a := &container{} - b := &container{} + a := &Container{} + b := &Container{} tests := []struct { array []uint16 runs []interval16 @@ -470,8 +470,8 @@ func TestIntersectArrayRun(t *testing.T) { } func TestIntersectRunRun(t *testing.T) { - a := &container{} - b := &container{} + a := &Container{} + b := &Container{} tests := []struct { aruns []interval16 bruns []interval16 @@ -532,8 +532,8 @@ func TestIntersectRunRun(t *testing.T) { } func TestIntersectBitmapRunBitmap(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN)} - b := &container{} + a := &Container{bitmap: make([]uint64, bitmapN)} + b := &Container{} tests := []struct { bitmap []uint64 runs []interval16 @@ -598,8 +598,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { } func TestIntersectBitmapRunArray(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN)} - b := &container{} + a := &Container{bitmap: make([]uint64, bitmapN)} + b := &Container{} tests := []struct { bitmap []uint64 runs []interval16 @@ -658,19 +658,19 @@ func TestIntersectBitmapRunArray(t *testing.T) { func TestUnionMixed(t *testing.T) { // array container - a := &container{} + a := &Container{} a.array = []uint16{1, 4, 5, 7, 10, 11, 12} a.containerType = ContainerArray a.n = 7 // bitmap container - b := &container{bitmap: make([]uint64, bitmapN)} + b := &Container{bitmap: make([]uint64, bitmapN)} b.bitmap[0] = uint64(0x3) b.n = 2 b.containerType = ContainerBitmap // run container - r := &container{} + r := &Container{} r.runs = []interval16{{start: 5, last: 10}} r.containerType = ContainerRun r.n = 6 @@ -678,8 +678,8 @@ func TestUnionMixed(t *testing.T) { t.Run("various container Unions", func(t *testing.T) { tests := []struct { name string - c1 *container - c2 *container + c1 *Container + c2 *Container exp []uint16 }{ {name: "run-array", c1: r, c2: a, exp: []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}}, @@ -707,9 +707,9 @@ func TestUnionMixed(t *testing.T) { } func TestIntersectMixed(t *testing.T) { - a := &container{} - b := &container{} - c := &container{} + a := &Container{} + b := &Container{} + c := &Container{} a.runs = []interval16{{start: 5, last: 10}} a.n = 6 @@ -755,10 +755,10 @@ func TestIntersectMixed(t *testing.T) { } func TestDifferenceMixed(t *testing.T) { - a := &container{} - b := &container{} - c := &container{} - d := &container{} + a := &Container{} + b := &Container{} + c := &Container{} + d := &Container{} a.runs = []interval16{{start: 5, last: 10}} a.n = a.runCountRange(0, 100) @@ -834,8 +834,8 @@ func TestDifferenceMixed(t *testing.T) { } func TestUnionRunRun(t *testing.T) { - a := &container{} - b := &container{} + a := &Container{} + b := &Container{} tests := []struct { aruns []interval16 bruns []interval16 @@ -895,8 +895,8 @@ func TestUnionRunRun(t *testing.T) { } func TestUnionArrayRun(t *testing.T) { - a := &container{} - b := &container{} + a := &Container{} + b := &Container{} tests := []struct { array []uint16 runs []interval16 @@ -937,7 +937,7 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -977,7 +977,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &container{containerType: ContainerArray} + a := &Container{containerType: ContainerArray} tests := []struct { array []uint16 exp []uint64 @@ -1008,7 +1008,7 @@ func TestArrayToBitmap(t *testing.T) { } func TestBitmapToArray(t *testing.T) { - a := &container{containerType: ContainerBitmap} + a := &Container{containerType: ContainerBitmap} tests := []struct { bitmap []uint64 exp []uint16 @@ -1039,7 +1039,7 @@ func TestBitmapToArray(t *testing.T) { } func TestRunToBitmap(t *testing.T) { - a := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerRun} tests := []struct { runs []interval16 exp []uint64 @@ -1093,7 +1093,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &container{containerType: ContainerBitmap} + a := &Container{containerType: ContainerBitmap} tests := []struct { bitmap []uint64 exp []interval16 @@ -1171,7 +1171,7 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := &container{containerType: ContainerArray} + a := &Container{containerType: ContainerArray} tests := []struct { array []uint16 exp []interval16 @@ -1205,7 +1205,7 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerRun} tests := []struct { runs []interval16 exp []uint16 @@ -1239,7 +1239,7 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1283,8 +1283,8 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1322,7 +1322,7 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 exp int @@ -1372,7 +1372,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &container{containerType: ContainerArray} + c := &Container{containerType: ContainerArray} tests := []struct { array []uint16 exp int @@ -1413,8 +1413,8 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &container{containerType: ContainerArray} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerArray} + b := &Container{containerType: ContainerRun} tests := []struct { array []uint16 runs []interval16 @@ -1439,8 +1439,8 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := &container{containerType: ContainerRun} - b := &container{containerType: ContainerArray} + a := &Container{containerType: ContainerRun} + b := &Container{containerType: ContainerArray} tests := []struct { runs []interval16 array []uint16 @@ -1520,8 +1520,8 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := &container{containerType: ContainerRun} - b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: ContainerRun} + b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 bitmap []uint64 @@ -1583,8 +1583,8 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1666,8 +1666,8 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - a := &container{containerType: ContainerArray} + b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: ContainerArray} tests := []struct { bitmap []uint64 array []uint16 @@ -1716,8 +1716,8 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - b := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + a := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + b := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1746,8 +1746,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := &container{containerType: ContainerRun} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerRun} + b := &Container{containerType: ContainerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -1780,7 +1780,7 @@ func TestDifferenceRunRun(t *testing.T) { } func TestWriteReadArray(t *testing.T) { - ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} + ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} ba := NewSliceBitmap() ba.conts.Put(0, ca) ba2 := NewSliceBitmap() @@ -1800,7 +1800,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap} for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } @@ -1823,7 +1823,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap} for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } @@ -1852,7 +1852,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} + cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} br := NewSliceBitmap() br.conts.Put(0, cr) br2 := NewSliceBitmap() @@ -1872,26 +1872,26 @@ func TestWriteReadRun(t *testing.T) { func TestXorArrayRun(t *testing.T) { tests := []struct { - a *container - b *container - exp *container + a *Container + b *Container + exp *Container }{ { - a: &container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray}, - b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12}, + a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12}, }, { - a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray}, - b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12}, + a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12}, }, { - a: &container{array: []uint16{65535}, containerType: ContainerArray}, - b: &container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, - exp: &container{array: []uint16{65534}, containerType: ContainerArray, n: 1}, + a: &Container{array: []uint16{65535}, containerType: ContainerArray}, + b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, + exp: &Container{array: []uint16{65534}, containerType: ContainerArray, n: 1}, }, { - a: &container{array: []uint16{65535}, containerType: ContainerArray}, - b: &container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, - exp: &container{array: []uint16{}, containerType: ContainerArray, n: 0}, + a: &Container{array: []uint16{65535}, containerType: ContainerArray}, + b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, + exp: &Container{array: []uint16{}, containerType: ContainerArray, n: 0}, }, } @@ -1912,8 +1912,8 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := &container{containerType: ContainerRun} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerRun} + b := &Container{containerType: ContainerRun} a.runs = []interval16{{start: 4, last: 10}} b.runs = []interval16{{start: 5, last: 10}} ret := xorRunRun(a, b) @@ -1927,8 +1927,8 @@ func TestXorRunRun1(t *testing.T) { } func TestXorRunRun(t *testing.T) { - a := &container{containerType: ContainerRun} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerRun} + b := &Container{containerType: ContainerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -2025,7 +2025,7 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + c := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { bitmap []uint64 start uint64 @@ -2093,8 +2093,8 @@ func TestBitmapXorRange(t *testing.T) { } func TestXorBitmapRun(t *testing.T) { - a := &container{containerType: ContainerBitmap} - b := &container{containerType: ContainerRun} + a := &Container{containerType: ContainerBitmap} + b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -2546,7 +2546,7 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &container{containerType: ContainerArray}, &container{ + a, b := &Container{containerType: ContainerArray}, &Container{ containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN), } @@ -3226,12 +3226,12 @@ func TestContainerCombinations(t *testing.T) { } //func getFunc(func(a, b *container) *container, m, n *container) *container { -func runContainerFunc(f interface{}, c ...*container) *container { +func runContainerFunc(f interface{}, c ...*Container) *Container { switch f.(type) { - case func(*container) *container: - return f.(func(*container) *container)(c[0]) - case func(*container, *container) *container: - return f.(func(a, b *container) *container)(c[0], c[1]) + case func(*Container) *Container: + return f.(func(*Container) *Container)(c[0]) + case func(*Container, *Container) *Container: + return f.(func(a, b *Container) *Container)(c[0], c[1]) } return nil } From faa79a385cac4f2b2d4e1d516a3bfffc0ddc41d9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 11 May 2018 20:20:24 -0500 Subject: [PATCH 27/48] Add B+tree to enterprise subpackage --- Makefile | 4 + bitmap.go | 4 +- ctl/check.go | 2 +- ctl/inspect.go | 2 +- {roaring => enterprise/b}/btree.go | 28 +++-- enterprise/b/containers_btree.go | 165 +++++++++++++++++++++++++ fragment.go | 2 +- roaring/containers.go | 143 ++++++++++++++++++++++ roaring/containers_btree.go | 164 +------------------------ roaring/containers_slice.go | 146 +--------------------- roaring/containers_test.go | 12 +- roaring/roaring.go | 38 +++--- roaring/roaring_internal_test.go | 36 +++--- roaring/roaring_test.go | 186 ++++++++++++++--------------- 14 files changed, 476 insertions(+), 456 deletions(-) rename {roaring => enterprise/b}/btree.go (95%) create mode 100644 enterprise/b/containers_btree.go create mode 100644 roaring/containers.go diff --git a/Makefile b/Makefile index e4d120659..94081cb52 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,10 @@ cover-viz: cover build: vendor go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Compile Pilosa EE +build-ee: vendor + go build -tags release -tags enterprise -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" diff --git a/bitmap.go b/bitmap.go index 975b4fb03..0b805aa4a 100644 --- a/bitmap.go +++ b/bitmap.go @@ -187,12 +187,12 @@ func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment { } // Insert new segment. - b.segments = append(b.segments, BitmapSegment{data: *roaring.NewSliceBitmap()}) + b.segments = append(b.segments, BitmapSegment{data: *roaring.NewBitmap()}) if i < len(b.segments) { copy(b.segments[i+1:], b.segments[i:]) } b.segments[i] = BitmapSegment{ - data: *roaring.NewSliceBitmap(), + data: *roaring.NewBitmap(), slice: slice, writable: true, } diff --git a/ctl/check.go b/ctl/check.go index e9589ddf4..04f947eb4 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -89,7 +89,7 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { defer syscall.Munmap(data) // Attach the mmap file to the bitmap. - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { return errors.Wrap(err, "unmarshalling") } diff --git a/ctl/inspect.go b/ctl/inspect.go index 943ab6e57..0c38c3fb5 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -69,7 +69,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { // Attach the mmap file to the bitmap. t := time.Now() fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { return errors.Wrap(err, "unmarshalling") } diff --git a/roaring/btree.go b/enterprise/b/btree.go similarity index 95% rename from roaring/btree.go rename to enterprise/b/btree.go index 371a73e18..3d4c09888 100644 --- a/roaring/btree.go +++ b/enterprise/b/btree.go @@ -29,12 +29,14 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -package roaring +package b import ( "fmt" "io" "sync" + + "github.com/pilosa/pilosa/roaring" ) const ( @@ -93,7 +95,7 @@ type ( de struct { // d element k uint64 - v *Container + v *roaring.Container } // Enumerator captures the state of enumerating a tree. It is returned @@ -417,7 +419,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { // First returns the first item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k uint64, v *Container) { +func (t *Tree) First() (k uint64, v *roaring.Container) { if q := t.first; q != nil { q := &q.d[0] k, v = q.k, q.v @@ -427,7 +429,7 @@ func (t *Tree) First() (k uint64, v *Container) { // Get returns the value associated with k and true if it exists. Otherwise Get // returns (zero-value, false). -func (t *Tree) Get(k uint64) (v *Container, ok bool) { +func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) { q := t.r if q == nil { return @@ -453,7 +455,7 @@ func (t *Tree) Get(k uint64) (v *Container, ok bool) { } } -func (t *Tree) insert(q *d, i int, k uint64, v *Container) *d { +func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { t.ver++ c := q.c if i < c { @@ -468,7 +470,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *Container) *d { // Last returns the last item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k uint64, v *Container) { +func (t *Tree) Last() (k uint64, v *roaring.Container) { if q := t.last; q != nil { q := &q.d[q.c-1] k, v = q.k, q.v @@ -481,7 +483,7 @@ func (t *Tree) Len() int { return t.c } -func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *Container) { +func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ l, r := p.siblings(pi) @@ -577,7 +579,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) { } // Set sets the value associated with k. -func (t *Tree) Set(k uint64, v *Container) { +func (t *Tree) Set(k uint64, v *roaring.Container) { //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) @@ -643,11 +645,11 @@ func (t *Tree) Set(k uint64, v *Container) { // tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. -func (t *Tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Container, write bool)) (oldV *Container, written bool) { +func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { pi := -1 var p *x q := t.r - var newV *Container + var newV *roaring.Container if q == nil { // new KV pair in empty tree newV, written = upd(newV, false) @@ -710,7 +712,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta } } -func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *Container) { +func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ r := btDPool.Get().(*d) if q.n != nil { @@ -856,7 +858,7 @@ func (e *Enumerator) Close() { // Next returns the currently enumerated item, if it exists and moves to the // next item in the key collation order. If there is no item to return, err == // io.EOF is returned. -func (e *Enumerator) Next() (k uint64, v *Container, err error) { +func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } @@ -904,7 +906,7 @@ func (e *Enumerator) next() error { // Prev returns the currently enumerated item, if it exists and moves to the // previous item in the key collation order. If there is no item to return, err // == io.EOF is returned. -func (e *Enumerator) Prev() (k uint64, v *Container, err error) { +func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go new file mode 100644 index 000000000..48975a4ef --- /dev/null +++ b/enterprise/b/containers_btree.go @@ -0,0 +1,165 @@ +package b + +import ( + "io" + + "github.com/pilosa/pilosa/roaring" +) + +func cmp(a, b uint64) int { + return int(a - b) +} + +func NewBTreeContainers() *BTreeContainers { + return &BTreeContainers{ + tree: TreeNew(cmp), + } +} + +type BTreeContainers struct { + tree *Tree + + lastKey uint64 + lastContainer *roaring.Container +} + +func (btc *BTreeContainers) Get(key uint64) *roaring.Container { + // Check the last* cache for same container. + if key == btc.lastKey && btc.lastContainer != nil { + return btc.lastContainer + } + + var c *roaring.Container + el, ok := btc.tree.Get(key) + if ok { + c = el + btc.lastKey = key + btc.lastContainer = c + } + return c +} + +func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) { + // If a mapped container is added to the tree, reset the + // lastContainer cache so that the cache is not pointing + // at a read-only mmap. + if c.mapped { + btc.lastContainer = nil + } + btc.tree.Set(key, c) +} + +func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) { + // update the existing container + if exists { + oldV.containerType = u.containerType + oldV.n = u.n + oldV.mapped = u.mapped + return oldV, false + } + return &roaring.Container{ + containerType: u.containerType, + n: u.n, + mapped: u.mapped, + }, true +} + +// this struct is added to prevent the closure locals from being escaped out to the heap +type updater struct { + key uint64 + containerType byte + n int + mapped bool +} + +func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + a := updater{key, containerType, n, mapped} + btc.tree.Put(key, a.update) +} + +func (btc *BTreeContainers) Remove(key uint64) { + btc.tree.Delete(key) +} + +func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { + // Check the last* cache for same container. + if key == btc.lastKey && btc.lastContainer != nil { + return btc.lastContainer + } + + btc.lastKey = key + v, ok := btc.tree.Get(key) + if !ok { + cont := newContainer() + btc.tree.Set(key, cont) + btc.lastContainer = cont + return cont + } + + btc.lastContainer = v + return btc.lastContainer +} + +func (btc *BTreeContainers) Clone() Containers { + nbtc := NewBTreeContainers() + + itr, err := btc.tree.SeekFirst() + if err == io.EOF { + return nbtc + } + for { + k, v, err := itr.Next() + if err == io.EOF { + break + } + nbtc.tree.Set(k, v.clone()) + } + return nbtc +} + +func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) { + if btc.tree.Len() == 0 { + return 0, nil + } + k, v := btc.tree.Last() + return k, v +} + +func (btc *BTreeContainers) Size() int { + return btc.tree.Len() +} + +func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool) { + e, ok := btc.tree.Seek(key) + if ok { + found = true + } + + return &BTCIterator{ + e: e, + }, found +} + +type BTCIterator struct { + e *Enumerator + key uint64 + val *roaring.Container +} + +func (i *BTCIterator) Next() bool { + + k, v, err := i.e.Next() + if err == io.EOF { + return false + } + i.key = k + i.val = v + return true +} + +func (i *BTCIterator) Value() (uint64, *roaring.Container) { + if i.val == nil { + return 0, nil + } + return i.key, i.val +} diff --git a/fragment.go b/fragment.go index 97b756213..7168f4443 100644 --- a/fragment.go +++ b/fragment.go @@ -189,7 +189,7 @@ func (f *Fragment) Open() error { func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. if f.storage == nil { - f.storage = roaring.NewBTreeBitmap() + f.storage = roaring.NewBitmap() } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) diff --git a/roaring/containers.go b/roaring/containers.go new file mode 100644 index 000000000..6e1bfe2e3 --- /dev/null +++ b/roaring/containers.go @@ -0,0 +1,143 @@ +package roaring + +type SliceContainers struct { + keys []uint64 + containers []*Container + lastKey uint64 + lastContainer *Container +} + +func (sc *SliceContainers) Get(key uint64) *Container { + i := search64(sc.keys, key) + if i < 0 { + return nil + } + return sc.containers[i] +} + +func (sc *SliceContainers) Put(key uint64, c *Container) { + i := search64(sc.keys, key) + + // If index is negative then there's not an exact match + // and a container needs to be added. + if i < 0 { + sc.insertAt(key, c, -i-1) + } else { + sc.containers[i] = c + } + +} + +func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + i := search64(sc.keys, key) + if i < 0 { + c := newContainer() + c.containerType = containerType + c.n = n + c.mapped = mapped + sc.insertAt(key, c, -i-1) + } else { + c := sc.containers[i] + c.containerType = containerType + c.n = n + c.mapped = mapped + } + +} + +func (sc *SliceContainers) Remove(key uint64) { + i := search64(sc.keys, key) + if i < 0 { + return + } + sc.keys = append(sc.keys[:i], sc.keys[i+1:]...) + sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) + +} +func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { + sc.keys = append(sc.keys, 0) + copy(sc.keys[i+1:], sc.keys[i:]) + sc.keys[i] = key + + sc.containers = append(sc.containers, nil) + copy(sc.containers[i+1:], sc.containers[i:]) + sc.containers[i] = c +} + +func (sc *SliceContainers) GetOrCreate(key uint64) *Container { + // Check the last* cache for same container. + if key == sc.lastKey && sc.lastContainer != nil { + return sc.lastContainer + } + + sc.lastKey = key + i := search64(sc.keys, key) + if i < 0 { + c := newContainer() + sc.insertAt(key, c, -i-1) + sc.lastContainer = c + return c + } + + sc.lastContainer = sc.containers[i] + return sc.lastContainer +} + +func (sc *SliceContainers) Clone() Containerser { + other := NewContainers() + other.keys = make([]uint64, len(sc.keys)) + other.containers = make([]*Container, len(sc.containers)) + copy(other.keys, sc.keys) + for i, c := range sc.containers { + other.containers[i] = c.clone() + } + return other +} + +func (sc *SliceContainers) Last() (key uint64, c *Container) { + if len(sc.keys) == 0 { + return 0, nil + } + return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1] +} + +func (sc *SliceContainers) Size() int { + return len(sc.keys) + +} + +func (sc *SliceContainers) seek(key uint64) (int, bool) { + i := search64(sc.keys, key) + found := true + if i < 0 { + found = false + i = -i - 1 + } + return i, found +} + +func (sc *SliceContainers) Iterator(key uint64) (citer Contiterator, found bool) { + i, found := sc.seek(key) + return &SliceIterator{e: sc, i: i}, found +} + +type SliceIterator struct { + e *SliceContainers + i int + key uint64 + value *Container +} + +func (si *SliceIterator) Next() bool { + if si.e == nil || si.i > len(si.e.keys)-1 { + return false + } + si.key = si.e.keys[si.i] + si.value = si.e.containers[si.i] + si.i++ + return true +} + +func (si *SliceIterator) Value() (uint64, *Container) { + return si.key, si.value +} diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index d3e317880..008007a48 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -1,163 +1,9 @@ +// +build enterprise + package roaring -import ( - "io" -) +import "github.com/pilosa/pilosa/enterprise/b" -func cmp(a, b uint64) int { - return int(a - b) -} - -func NewBTreeContainers() *BTreeContainers { - return &BTreeContainers{ - tree: TreeNew(cmp), - } -} - -type BTreeContainers struct { - tree *Tree - - lastKey uint64 - lastContainer *Container -} - -func (btc *BTreeContainers) Get(key uint64) *Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - var c *Container - el, ok := btc.tree.Get(key) - if ok { - c = el - btc.lastKey = key - btc.lastContainer = c - } - return c -} - -func (btc *BTreeContainers) Put(key uint64, c *Container) { - // If a mapped container is added to the tree, reset the - // lastContainer cache so that the cache is not pointing - // at a read-only mmap. - if c.mapped { - btc.lastContainer = nil - } - btc.tree.Set(key, c) -} - -func (u updater) update(oldV *Container, exists bool) (*Container, bool) { - // update the existing container - if exists { - oldV.containerType = u.containerType - oldV.n = u.n - oldV.mapped = u.mapped - return oldV, false - } - return &Container{ - containerType: u.containerType, - n: u.n, - mapped: u.mapped, - }, true -} - -// this struct is added to prevent the closure locals from being escaped out to the heap -type updater struct { - key uint64 - containerType byte - n int - mapped bool -} - -func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - a := updater{key, containerType, n, mapped} - btc.tree.Put(key, a.update) -} - -func (btc *BTreeContainers) Remove(key uint64) { - btc.tree.Delete(key) -} - -func (btc *BTreeContainers) GetOrCreate(key uint64) *Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - btc.lastKey = key - v, ok := btc.tree.Get(key) - if !ok { - cont := newContainer() - btc.tree.Set(key, cont) - btc.lastContainer = cont - return cont - } - - btc.lastContainer = v - return btc.lastContainer -} - -func (btc *BTreeContainers) Clone() Containers { - nbtc := NewBTreeContainers() - - itr, err := btc.tree.SeekFirst() - if err == io.EOF { - return nbtc - } - for { - k, v, err := itr.Next() - if err == io.EOF { - break - } - nbtc.tree.Set(k, v.clone()) - } - return nbtc -} - -func (btc *BTreeContainers) Last() (key uint64, c *Container) { - if btc.tree.Len() == 0 { - return 0, nil - } - k, v := btc.tree.Last() - return k, v -} - -func (btc *BTreeContainers) Size() int { - return btc.tree.Len() -} - -func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool) { - e, ok := btc.tree.Seek(key) - if ok { - found = true - } - - return &BTCIterator{ - e: e, - }, found -} - -type BTCIterator struct { - e *Enumerator - key uint64 - val *Container -} - -func (i *BTCIterator) Next() bool { - - k, v, err := i.e.Next() - if err == io.EOF { - return false - } - i.key = k - i.val = v - return true -} - -func (i *BTCIterator) Value() (uint64, *Container) { - if i.val == nil { - return 0, nil - } - return i.key, i.val +func NewContainers() *b.BTreeContainers { + return &b.BTreeContainers{} } diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go index 84cec96c1..8642dbabc 100644 --- a/roaring/containers_slice.go +++ b/roaring/containers_slice.go @@ -1,147 +1,7 @@ +// +build !enterprise + package roaring -func NewSliceContainers() *SliceContainers { +func NewContainers() *SliceContainers { return &SliceContainers{} } - -type SliceContainers struct { - keys []uint64 - containers []*Container - lastKey uint64 - lastContainer *Container -} - -func (sc *SliceContainers) Get(key uint64) *Container { - i := search64(sc.keys, key) - if i < 0 { - return nil - } - return sc.containers[i] -} - -func (sc *SliceContainers) Put(key uint64, c *Container) { - i := search64(sc.keys, key) - - // If index is negative then there's not an exact match - // and a container needs to be added. - if i < 0 { - sc.insertAt(key, c, -i-1) - } else { - sc.containers[i] = c - } - -} - -func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - i := search64(sc.keys, key) - if i < 0 { - c := newContainer() - c.containerType = containerType - c.n = n - c.mapped = mapped - sc.insertAt(key, c, -i-1) - } else { - c := sc.containers[i] - c.containerType = containerType - c.n = n - c.mapped = mapped - } - -} - -func (sc *SliceContainers) Remove(key uint64) { - i := search64(sc.keys, key) - if i < 0 { - return - } - sc.keys = append(sc.keys[:i], sc.keys[i+1:]...) - sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) - -} -func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { - sc.keys = append(sc.keys, 0) - copy(sc.keys[i+1:], sc.keys[i:]) - sc.keys[i] = key - - sc.containers = append(sc.containers, nil) - copy(sc.containers[i+1:], sc.containers[i:]) - sc.containers[i] = c -} - -func (sc *SliceContainers) GetOrCreate(key uint64) *Container { - // Check the last* cache for same container. - if key == sc.lastKey && sc.lastContainer != nil { - return sc.lastContainer - } - - sc.lastKey = key - i := search64(sc.keys, key) - if i < 0 { - c := newContainer() - sc.insertAt(key, c, -i-1) - sc.lastContainer = c - return c - } - - sc.lastContainer = sc.containers[i] - return sc.lastContainer -} - -func (sc *SliceContainers) Clone() Containers { - other := NewSliceContainers() - other.keys = make([]uint64, len(sc.keys)) - other.containers = make([]*Container, len(sc.containers)) - copy(other.keys, sc.keys) - for i, c := range sc.containers { - other.containers[i] = c.clone() - } - return other -} - -func (sc *SliceContainers) Last() (key uint64, c *Container) { - if len(sc.keys) == 0 { - return 0, nil - } - return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1] -} - -func (sc *SliceContainers) Size() int { - return len(sc.keys) - -} - -func (sc *SliceContainers) seek(key uint64) (int, bool) { - i := search64(sc.keys, key) - found := true - if i < 0 { - found = false - i = -i - 1 - } - return i, found -} - -func (sc *SliceContainers) Iterator(key uint64) (citer Contiterator, found bool) { - i, found := sc.seek(key) - return &SliceIterator{e: sc, i: i}, found -} - -type SliceIterator struct { - e *SliceContainers - i int - key uint64 - value *Container -} - -func (si *SliceIterator) Next() bool { - if si.e == nil || si.i > len(si.e.keys)-1 { - return false - } - si.key = si.e.keys[si.i] - si.value = si.e.containers[si.i] - si.i++ - return true -} - -func (si *SliceIterator) Value() (uint64, *Container) { - return si.key, si.value -} diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 55fa24ff4..9acf9cc08 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -4,16 +4,16 @@ import ( "testing" ) -func TestContainersSliceIterator(t *testing.T) { - btc := NewBTreeContainers() - testContainersIterator(btc, t) -} +//func TestContainersSliceIterator(t *testing.T) { +// btc := NewBTreeContainers() +// testContainersIterator(btc, t) +//} func TestContainersBTreeIterator(t *testing.T) { - slc := NewSliceContainers() + slc := NewContainers() testContainersIterator(slc, t) } -func testContainersIterator(cs Containers, t *testing.T) { +func testContainersIterator(cs Containerser, t *testing.T) { itr, found := cs.Iterator(0) if found { t.Fatalf("shouldn't have found 0 in empty btc") diff --git a/roaring/roaring.go b/roaring/roaring.go index b1f25393e..5ad67c3cd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -64,7 +64,7 @@ const ( maxContainerVal = 0xffff ) -type Containers interface { +type Containerser interface { // Get returns nil if the key does not exist. Get(key uint64) *Container @@ -82,7 +82,7 @@ type Containers interface { GetOrCreate(key uint64) *Container // Clone does a deep copy of Containers, including cloning all containers contained. - Clone() Containers + Clone() Containerser // Last returns the highest key and associated container. Last() (key uint64, c *Container) @@ -103,7 +103,7 @@ type Contiterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { - conts Containers + conts Containerser // Number of operations written to the writer. opN int @@ -112,22 +112,22 @@ type Bitmap struct { OpWriter io.Writer } -// NewSliceBitmap returns a Bitmap with an initial set of values. -func NewSliceBitmap(a ...uint64) *Bitmap { +// NewBitmap returns a Bitmap with an initial set of values. +func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ - conts: NewSliceContainers(), + conts: NewContainers(), } b.Add(a...) return b } -func NewBTreeBitmap(a ...uint64) *Bitmap { - b := &Bitmap{ - conts: NewBTreeContainers(), - } - b.Add(a...) - return b -} +//func NewBTreeBitmap(a ...uint64) *Bitmap { +// b := &Bitmap{ +// conts: NewBTreeContainers(), +// } +// b.Add(a...) +// return b +//} // Clone returns a heap allocated copy of the bitmap. // Note: The OpWriter IS NOT copied to the new bitmap. @@ -329,7 +329,7 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) citer, _ := b.conts.Iterator(hi0) - other := NewSliceBitmap() + other := NewBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { @@ -374,7 +374,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { - output := NewSliceBitmap() + output := NewBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -399,7 +399,7 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { - output := NewSliceBitmap() + output := NewBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -427,7 +427,7 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { // Difference returns the difference of b and other. func (b *Bitmap) Difference(other *Bitmap) *Bitmap { - output := NewSliceBitmap() + output := NewBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -454,7 +454,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { // Xor returns the bitwise exclusive or of b and other. func (b *Bitmap) Xor(other *Bitmap) *Bitmap { - output := NewSliceBitmap() + output := NewBitmap() iiter, _ := b.conts.Iterator(0) jiter, _ := other.conts.Iterator(0) @@ -768,7 +768,7 @@ func (b *Bitmap) Check() error { // Flip performs a logical negate of the bits in the range [start,end]. func (b *Bitmap) Flip(start, end uint64) *Bitmap { - result := NewSliceBitmap() + result := NewBitmap() itr := b.Iterator() v, eof := itr.Next() //copy over previous bits. diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 2d3fe10bf..da542aa36 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1513,7 +1513,7 @@ func MakeBitmap(start []uint64) []uint64 { return b } func MakeLastBitSet() []uint64 { - obj := NewSliceBitmap(65535) + obj := NewBitmap(65535) c := obj.container(0) c.arrayToBitmap() return c.bitmap @@ -1781,9 +1781,9 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} - ba := NewSliceBitmap() + ba := NewBitmap() ba.conts.Put(0, ca) - ba2 := NewSliceBitmap() + ba2 := NewBitmap() var buf bytes.Buffer _, err := ba.WriteTo(&buf) if err != nil { @@ -1804,9 +1804,9 @@ func TestWriteReadBitmap(t *testing.T) { for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } - bb := NewSliceBitmap() + bb := NewBitmap() bb.conts.Put(0, cb) - bb2 := NewSliceBitmap() + bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1827,9 +1827,9 @@ func TestWriteReadFullBitmap(t *testing.T) { for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } - bb := NewSliceBitmap() + bb := NewBitmap() bb.conts.Put(0, cb) - bb2 := NewSliceBitmap() + bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1853,9 +1853,9 @@ func TestWriteReadFullBitmap(t *testing.T) { func TestWriteReadRun(t *testing.T) { cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} - br := NewSliceBitmap() + br := NewBitmap() br.conts.Put(0, cr) - br2 := NewSliceBitmap() + br2 := NewBitmap() var buf bytes.Buffer _, err := br.WriteTo(&buf) if err != nil { @@ -2124,7 +2124,7 @@ func TestXorBitmapRun(t *testing.T) { func TestIteratorArray(t *testing.T) { // use values that span two containers - b := NewSliceBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) if !b.conts.Get(0).isArray() { t.Fatalf("wrong container type") } @@ -2171,7 +2171,7 @@ func TestIteratorBitmap(t *testing.T) { // use values that span two containers // this dataset will update to bitmap after enough Adds, // but won't update to RLE until Optimize() is called - b := NewSliceBitmap() + b := NewBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -2222,7 +2222,7 @@ func TestIteratorBitmap(t *testing.T) { } func TestIteratorRuns(t *testing.T) { - b := NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() if !b.conts.Get(0).isRun() { t.Fatalf("wrong container type") @@ -2289,7 +2289,7 @@ func TestIteratorVarious(t *testing.T) { exp uint64 }{ { - bm: NewSliceBitmap(3, 4, 5), + bm: NewBitmap(3, 4, 5), exp: 3, }, { @@ -2297,7 +2297,7 @@ func TestIteratorVarious(t *testing.T) { exp: 61221, }, { - bm: NewSliceBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), exp: 7, }, } @@ -2438,7 +2438,7 @@ func TestRunBinSearch(t *testing.T) { } } func TestBitmap_RemoveEmptyContainers(t *testing.T) { - bm1 := NewSliceBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) if bm1.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") @@ -2451,13 +2451,13 @@ func TestBitmap_RemoveEmptyContainers(t *testing.T) { } func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { - bm1 := NewSliceBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) var buf bytes.Buffer if _, err := bm1.WriteTo(&buf); err != nil { t.Fatalf("Failure to write to bitmap buffer. ") } - bm0 := NewSliceBitmap() + bm0 := NewBitmap() bm0.UnmarshalBinary(buf.Bytes()) if bm0.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") @@ -2686,7 +2686,7 @@ func bitmapVariousContainers() *Bitmap { bits = append(bits, bitCont(7, true, true, true)...) bits = append(bits, arrCont(8, true, true, true)...) bits = append(bits, rleCont(9, true, true, true)...) - bm := NewSliceBitmap(bits...) + bm := NewBitmap(bits...) bm.Optimize() return bm } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 4cbf17e2e..2f2c4cda2 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -30,7 +30,7 @@ import ( ) func TestBitmapClone(t *testing.T) { - b := roaring.NewSliceBitmap() + b := roaring.NewBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -48,7 +48,7 @@ func TestBitmapClone(t *testing.T) { } func TestContainerCount(t *testing.T) { - b := roaring.NewSliceBitmap(65535) + b := roaring.NewBitmap(65535) if b.Count() != b.CountRange(0, 65546) { t.Fatalf("Count != CountRange\n") @@ -144,7 +144,7 @@ func TestCountRange(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) { - b := roaring.NewSliceBitmap(test.bitmap...) + b := roaring.NewBitmap(test.bitmap...) actual := b.CountRange(test.start, test.end) if actual != test.exp { t.Errorf("got: %d, exp: %d", actual, test.exp) @@ -154,7 +154,7 @@ func TestCountRange(t *testing.T) { } func TestCheckBitmap(t *testing.T) { - b := roaring.NewSliceBitmap() + b := roaring.NewBitmap() x := 0 for i := uint64(61000); i < 71000; i++ { x++ @@ -171,7 +171,7 @@ func TestCheckBitmap(t *testing.T) { } func TestCheckArray(t *testing.T) { - b := roaring.NewSliceBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) err := b.Check() if err != nil { t.Fatalf("%v\n", err) @@ -179,7 +179,7 @@ func TestCheckArray(t *testing.T) { } func TestCheckRun(t *testing.T) { - b := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() // convert to runs err := b.Check() if err != nil { @@ -187,7 +187,7 @@ func TestCheckRun(t *testing.T) { } } func TestCheckFullRun(t *testing.T) { - b := roaring.NewSliceBitmap() + b := roaring.NewBitmap() for i := uint64(0); i < 2097152; i++ { if i%16384 == 0 { b.Optimize() // convert to runs @@ -208,7 +208,7 @@ func TestCheckFullRun(t *testing.T) { // Ensure that we can transition between runs and arrays when materializing the bitmap. func TestContainerTransitions(t *testing.T) { // [run, run][array][run] - b := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) + b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) b.Optimize() // convert to runs if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) { t.Fatalf("unexpected slice: %+v", b.Slice()) @@ -216,7 +216,7 @@ func TestContainerTransitions(t *testing.T) { // Test the case where last and first bits of adjoining containers are set. // [run][array][run] - b2 := roaring.NewSliceBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) + b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) b2.Optimize() // convert to runs if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) { t.Fatalf("unexpected slice: %+v", b2.Slice()) @@ -225,26 +225,26 @@ func TestContainerTransitions(t *testing.T) { // Ensure an empty bitmap returns false if checking for existence. func TestBitmap_Contains_Empty(t *testing.T) { - if roaring.NewSliceBitmap().Contains(1000) { + if roaring.NewBitmap().Contains(1000) { t.Fatal("expected false") } } // Ensure an empty bitmap does nothing when removing an element. func TestBitmap_Remove_Empty(t *testing.T) { - roaring.NewSliceBitmap().Remove(1000) + roaring.NewBitmap().Remove(1000) } // Ensure a bitmap can return a slice of values. func TestBitmap_Slice(t *testing.T) { - if a := roaring.NewSliceBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected slice: %+v", a) } } // Ensure an empty bitmap returns an empty slice of values. func TestBitmap_Slice_Empty(t *testing.T) { - if a := roaring.NewSliceBitmap().Slice(); len(a) != 0 { + if a := roaring.NewBitmap().Slice(); len(a) != 0 { t.Fatalf("unexpected slice: %+v", a) } } @@ -252,7 +252,7 @@ func TestBitmap_Slice_Empty(t *testing.T) { // Ensure a bitmap can return a slice of values within a range. // TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { - if a := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { + if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { t.Fatalf("unexpected slice: %+v", a) } } @@ -260,7 +260,7 @@ func TestBitmap_SliceRange(t *testing.T) { // Ensure a bitmap can loop over a set of values. func TestBitmap_ForEach(t *testing.T) { var a []uint64 - roaring.NewSliceBitmap(1, 2, 3).ForEach(func(v uint64) { + roaring.NewBitmap(1, 2, 3).ForEach(func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{1, 2, 3}) { @@ -271,7 +271,7 @@ func TestBitmap_ForEach(t *testing.T) { // Ensure a bitmap can loop over a set of values in a range. func TestBitmap_ForEachRange(t *testing.T) { var a []uint64 - roaring.NewSliceBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { + roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{2, 3}) { @@ -281,7 +281,7 @@ func TestBitmap_ForEachRange(t *testing.T) { // Ensure bitmap can return the highest value. func TestBitmap_Max(t *testing.T) { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() for i := uint64(1000); i <= 100000; i++ { bm.Add(i) @@ -297,7 +297,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { e := uint64(2010 * 1048576) start := s + (39314024 % 1048576) - bm0 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { start += 16384 @@ -315,7 +315,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { } func TestBitmap_BitmapCountRange(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) + bm0 := roaring.NewBitmap(0, 2683177) for i := uint64(628); i < 2683301; i++ { bm0.Add(i) } @@ -343,20 +343,20 @@ func TestBitmap_BitmapCountRange(t *testing.T) { } func TestBitmap_ArrayCountRange(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177, 2683313) + bm0 := roaring.NewBitmap(0, 2683177, 2683313) if n := bm0.CountRange(1, 2683313); n != 1 { t.Fatalf("unexpected n: %d", n) } } func TestBitmap_RunCountRange(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) + bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) bm0.Optimize() // convert to runs if n := bm0.CountRange(15, 1000003); n != 5 { t.Fatalf("unexpected n: %d", n) } - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) bm1.Optimize() // convert to runs if n := bm1.CountRange(5, 12); n != 7 { t.Fatalf("unexpected n: %d", n) @@ -364,8 +364,8 @@ func TestBitmap_RunCountRange(t *testing.T) { } func TestBitmap_Intersectionz(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -378,8 +378,8 @@ func TestBitmap_Intersectionz(t *testing.T) { } func TestBitmap_Union1(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -402,8 +402,8 @@ func TestBitmap_Union1(t *testing.T) { } func TestBitmap_Intersection_Empty(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() result := bm0.Intersect(bm1) if n := result.Count(); n != 0 { @@ -413,8 +413,8 @@ func TestBitmap_Intersection_Empty(t *testing.T) { } func TestBitmap_IntersectArrayArray(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1, 2683, 5005) - bm1 := roaring.NewSliceBitmap(0, 2683, 2684, 5000) + bm0 := roaring.NewBitmap(0, 1, 2683, 5005) + bm1 := roaring.NewBitmap(0, 2683, 2684, 5000) result := bm0.Intersect(bm1) if n := result.Count(); n != 2 { @@ -423,12 +423,12 @@ func TestBitmap_IntersectArrayArray(t *testing.T) { } func TestBitmap_IntersectBitmapBitmap(t *testing.T) { - bm0 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap() for i := uint64(0); i < 65536; i += 2 { bm0.Add(i) } - bm1 := roaring.NewSliceBitmap() + bm1 := roaring.NewBitmap() for i := uint64(0); i < 65536; i += 3 { bm1.Add(i) } @@ -441,9 +441,9 @@ func TestBitmap_IntersectBitmapBitmap(t *testing.T) { func TestBitmap_IntersectRunRun(t *testing.T) { // Intersect two runs that result in an array. - bm0 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) + bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) bm0.Optimize() // convert to runs - bm1 := roaring.NewSliceBitmap(5, 6, 7, 8, 9, 10, 11) + bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11) bm1.Optimize() // convert to runs result := bm0.Intersect(bm1) if n := result.Count(); n != 3 { @@ -451,7 +451,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } // Intersect two runs that result in a bitmap. - bm2 := roaring.NewSliceBitmap() + bm2 := roaring.NewBitmap() runLen := uint64(25) spaceLen := uint64(8) offset := (runLen / 2) + spaceLen @@ -461,7 +461,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } } bm2.Optimize() // convert to runs - bm3 := roaring.NewSliceBitmap() + bm3 := roaring.NewBitmap() runLen = uint64(32) spaceLen = uint64(1) for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { @@ -477,8 +477,8 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } func TestBitmap_Difference(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -489,8 +489,8 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewSliceBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) + bm1 := roaring.NewBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) result := bm0.Difference(bm1) if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) @@ -498,8 +498,8 @@ func TestBitmap_Difference2(t *testing.T) { } func TestBitmap_Difference_Empty(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 2683177) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(0, 2683177) + bm1 := roaring.NewBitmap() result := bm0.Difference(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -507,8 +507,8 @@ func TestBitmap_Difference_Empty(t *testing.T) { } func TestBitmap_DifferenceArrayArray(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 4, 8, 12, 16, 20) - bm1 := roaring.NewSliceBitmap(1, 3, 6, 9, 12, 15, 18) + bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20) + bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18) result := bm0.Difference(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -516,9 +516,9 @@ func TestBitmap_DifferenceArrayArray(t *testing.T) { } func TestBitmap_DifferenceArrayRun(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) + bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) - bm1 := roaring.NewSliceBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) + bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) bm1.Optimize() // convert to runs result := bm0.Difference(bm1) if n := result.Count(); n != 6 { @@ -527,8 +527,8 @@ func TestBitmap_DifferenceArrayRun(t *testing.T) { } func TestBitmap_Union(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) result := bm0.Union(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -537,7 +537,7 @@ func TestBitmap_Union(t *testing.T) { func TestBitmap_Xor(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3) + bm1 := roaring.NewBitmap(0, 1, 2, 3) result := bm1.Xor(bm0) if n := result.Count(); n != 75011 { t.Fatalf("unexpected n: %d", n) @@ -555,8 +555,8 @@ func TestBitmap_Xor(t *testing.T) { } func TestBitmap_Xor_ArrayArray(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) result := bm0.Xor(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -572,8 +572,8 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) { //empty array test func TestBitmap_Xor_Empty(t *testing.T) { - bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) - empty := roaring.NewSliceBitmap() + bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + empty := roaring.NewBitmap() result := bm1.Xor(empty) if n := result.Count(); n != 4 { @@ -581,8 +581,8 @@ func TestBitmap_Xor_Empty(t *testing.T) { } } func TestBitmap_Xor_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewSliceBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) } @@ -603,7 +603,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { t.Fatalf("test 3 unexpected n: %d", n) } - empty := roaring.NewSliceBitmap() + empty := roaring.NewBitmap() result = bm1.Xor(empty) if n := result.Count(); n != 5000 { t.Fatalf("unexpected n: %d", n) @@ -611,8 +611,8 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { } func TestBitmap_Xor_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewSliceBitmap() - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap() + bm1 := roaring.NewBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) @@ -630,7 +630,7 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) { // Ensure bitmap contents alternate. func TestBitmap_Flip_Empty(t *testing.T) { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() results := bm.Flip(0, 10) if n := results.Count(); n != 11 { t.Fatalf("unexpected n: %d", n) @@ -643,7 +643,7 @@ func TestBitmap_Flip_Empty(t *testing.T) { // Test Subrange Flip should not affect bits outside of Range func TestBitmap_Flip_Array(t *testing.T) { - bm := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) + bm := roaring.NewBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) results := bm.Flip(0, 4) if !reflect.DeepEqual(results.Slice(), []uint64{8, 16, 32, 64, 128, 256, 512, 1024}) { t.Fatalf("unexpected %v ", results.Slice()) @@ -657,7 +657,7 @@ func TestBitmap_Flip_Array(t *testing.T) { // Ensure Flip works with underlying Bitmap container. func TestBitmap_Flip_Bitmap(t *testing.T) { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() size := uint64(10000) for i := uint64(0); i < size; i += 2 { bm.Add(i) @@ -674,7 +674,7 @@ func TestBitmap_Flip_Bitmap(t *testing.T) { // Verify Flip works correctly with in different regions of bitmap, beginning, middle, and end. func TestBitmap_Flip_After(t *testing.T) { - bm := roaring.NewSliceBitmap(0, 2, 4, 8) + bm := roaring.NewBitmap(0, 2, 4, 8) results := bm.Flip(9, 10) if !reflect.DeepEqual(results.Slice(), []uint64{0, 2, 4, 8, 9, 10}) { @@ -693,8 +693,8 @@ func TestBitmap_Flip_After(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1, 1000001, 1000002, 1000003) - bm1 := roaring.NewSliceBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) if n := bm0.IntersectionCount(bm1); n != 3 { t.Fatalf("unexpected n: %d", n) @@ -705,8 +705,8 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { - bm0 := roaring.NewSliceBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 3 { @@ -718,9 +718,9 @@ func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_RunRun(t *testing.T) { - bm0 := roaring.NewSliceBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) + bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) bm0.Optimize() // convert to runs - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 6 { @@ -732,11 +732,11 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { - bm0 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap() for i := uint64(3); i <= 1000006; i += 2 { bm0.Add(i) } - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 4 { @@ -748,8 +748,8 @@ func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewSliceBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewBitmap() for i := uint64(0); i <= 10000; i += 2 { bm1.Add(i) } @@ -763,8 +763,8 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewSliceBitmap() - bm1 := roaring.NewSliceBitmap() + bm0 := roaring.NewBitmap() + bm1 := roaring.NewBitmap() for i := uint64(0); i <= 10000; i += 2 { bm0.Add(i) bm1.Add(i + 1) @@ -784,8 +784,8 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { } func TestBitmap_IntersectionCount_Mixed(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewSliceBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) - bm3 := roaring.NewSliceBitmap(131072) + bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) + bm3 := roaring.NewBitmap(131072) if n := bm0.IntersectionCount(bm0); n != bm0.Count() { t.Fatalf("unexpected n: %d", n) @@ -807,7 +807,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { quick.Check(func(a []uint64) bool { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() m := make(map[uint64]struct{}) // Add values to the bitmap and set. @@ -894,7 +894,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { quick.Check(func(a0, a1 []uint64) bool { // Create bitmap with initial values set. - bm := roaring.NewSliceBitmap(a0...) + bm := roaring.NewBitmap(a0...) set := make(map[uint64]struct{}) for _, v := range a0 { @@ -923,7 +923,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { data := buf.Bytes() // Create new bitmap from ops log data. - bm2 := roaring.NewSliceBitmap() + bm2 := roaring.NewBitmap() if err := bm2.UnmarshalBinary(data); err != nil { t.Fatal(err) } @@ -952,7 +952,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { // TODO duplicate for all container types func TestIterator(t *testing.T) { t.Run("bitmap", func(t *testing.T) { - itr := roaring.NewSliceBitmap(1, 2, 3).Iterator() + itr := roaring.NewBitmap(1, 2, 3).Iterator() itr.Seek(0) var a []uint64 @@ -966,13 +966,13 @@ func TestIterator(t *testing.T) { }) t.Run("run", func(t *testing.T) { - bm1 := roaring.NewSliceBitmap() + bm1 := roaring.NewBitmap() for i := uint64(0); i < 11; i += 1 { bm1.Add(i) } bm1.Optimize() - bm2 := roaring.NewSliceBitmap() + bm2 := roaring.NewBitmap() for i := uint64(0); i < 12; i += 1 { bm2.Add(i) } @@ -1005,7 +1005,7 @@ func TestIterator(t *testing.T) { // testBM creates a bitmap with 3 containers: array, bitmap, and run. func testBM() *roaring.Bitmap { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() //the array for i := uint64(0); i < 1024; i += 4 { bm.Add((1 << 16) + i) @@ -1068,19 +1068,19 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } { const max = (1 << 24) / 64 // Build bitmap with array container. - data.a = roaring.NewSliceBitmap() + data.a = roaring.NewBitmap() for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ { data.a.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. - data.b = roaring.NewSliceBitmap() + data.b = roaring.NewBitmap() for i, n := 0, MaxContainerVal/3; i < n; i++ { data.b.Add(uint64(i * 3)) } // build bitmap with run container - data.r = roaring.NewSliceBitmap() + data.r = roaring.NewBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { data.r.Add(uint64(i)) } @@ -1176,7 +1176,7 @@ const ( func BenchmarkContainerLinear(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBTreeBitmap() + bm := roaring.NewBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1187,7 +1187,7 @@ func BenchmarkContainerLinear(b *testing.B) { func BenchmarkContainerReverse(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBTreeBitmap() + bm := roaring.NewBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1198,7 +1198,7 @@ func BenchmarkContainerReverse(b *testing.B) { func BenchmarkContainerColumn(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBTreeBitmap() + bm := roaring.NewBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1210,7 +1210,7 @@ func BenchmarkContainerColumn(b *testing.B) { func BenchmarkContainerOutsideIn(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBTreeBitmap() + bm := roaring.NewBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { @@ -1224,7 +1224,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) { func BenchmarkContainerInsideOut(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBTreeBitmap() + bm := roaring.NewBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1236,7 +1236,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() for col := uint64(0); col < pilosa.SliceWidth; col++ { bm.Add(col) } @@ -1245,7 +1245,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewSliceBitmap() + bm := roaring.NewBitmap() for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { bm.Add(col) } From 4282e90fe15e874b47cf570cfe764dc33e459b6c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 14 May 2018 14:03:46 -0500 Subject: [PATCH 28/48] Export a few things to enable enterprise/b/containers_btree.go to work --- enterprise/b/containers_btree.go | 22 +++++------ roaring/containers.go | 6 +-- roaring/roaring.go | 66 +++++++++++++++++++------------- roaring/roaring_internal_test.go | 2 +- 4 files changed, 52 insertions(+), 44 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 48975a4ef..69075e9e7 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -43,7 +43,7 @@ func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) { // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. - if c.mapped { + if c.Mapped() { btc.lastContainer = nil } btc.tree.Set(key, c) @@ -52,16 +52,12 @@ func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) { func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) { // update the existing container if exists { - oldV.containerType = u.containerType - oldV.n = u.n - oldV.mapped = u.mapped + oldV.Update(u.containerType, u.n, u.mapped) return oldV, false } - return &roaring.Container{ - containerType: u.containerType, - n: u.n, - mapped: u.mapped, - }, true + cont := roaring.NewContainer() + cont.Update(u.containerType, u.n, u.mapped) + return cont, true } // this struct is added to prevent the closure locals from being escaped out to the heap @@ -90,7 +86,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { btc.lastKey = key v, ok := btc.tree.Get(key) if !ok { - cont := newContainer() + cont := roaring.NewContainer() btc.tree.Set(key, cont) btc.lastContainer = cont return cont @@ -100,7 +96,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { return btc.lastContainer } -func (btc *BTreeContainers) Clone() Containers { +func (btc *BTreeContainers) Clone() roaring.Containerser { nbtc := NewBTreeContainers() itr, err := btc.tree.SeekFirst() @@ -112,7 +108,7 @@ func (btc *BTreeContainers) Clone() Containers { if err == io.EOF { break } - nbtc.tree.Set(k, v.clone()) + nbtc.tree.Set(k, v.Clone()) } return nbtc } @@ -129,7 +125,7 @@ func (btc *BTreeContainers) Size() int { return btc.tree.Len() } -func (btc *BTreeContainers) Iterator(key uint64) (citer Contiterator, found bool) { +func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.Contiterator, found bool) { e, ok := btc.tree.Seek(key) if ok { found = true diff --git a/roaring/containers.go b/roaring/containers.go index 6e1bfe2e3..ecc69b6fd 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -31,7 +31,7 @@ func (sc *SliceContainers) Put(key uint64, c *Container) { func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { i := search64(sc.keys, key) if i < 0 { - c := newContainer() + c := NewContainer() c.containerType = containerType c.n = n c.mapped = mapped @@ -73,7 +73,7 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container { sc.lastKey = key i := search64(sc.keys, key) if i < 0 { - c := newContainer() + c := NewContainer() sc.insertAt(key, c, -i-1) sc.lastContainer = c return c @@ -89,7 +89,7 @@ func (sc *SliceContainers) Clone() Containerser { other.containers = make([]*Container, len(sc.containers)) copy(other.keys, sc.keys) for i, c := range sc.containers { - other.containers[i] = c.clone() + other.containers[i] = c.Clone() } return other } diff --git a/roaring/roaring.go b/roaring/roaring.go index 5ad67c3cd..f279f20d5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -408,11 +408,11 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.clone()) + output.conts.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.conts.Put(kj, cj.clone()) + output.conts.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -436,7 +436,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.clone()) + output.conts.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { @@ -463,11 +463,11 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.clone()) + output.conts.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.conts.Put(kj, cj.clone()) + output.conts.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -1026,10 +1026,22 @@ func (iv interval16) runlen() int { } // newContainer returns a new instance of container. -func newContainer() *Container { +func NewContainer() *Container { return &Container{containerType: ContainerArray} } +// Mapped returns true if the container is mapped directly to a byte slice +func (c *Container) Mapped() bool { + return c.mapped +} + +// Update updates the container +func (c *Container) Update(containerType byte, n int, mapped bool) { + c.containerType = containerType + c.n = n + c.mapped = mapped +} + // isArray returns true if the container is an array container. func (c *Container) isArray() bool { return c.containerType == ContainerArray @@ -1650,8 +1662,8 @@ func (c *Container) runToArray() { c.mapped = false } -// clone returns a copy of c. -func (c *Container) clone() *Container { +// Clone returns a copy of c. +func (c *Container) Clone() *Container { other := &Container{n: c.n, containerType: c.containerType} switch c.containerType { @@ -1807,7 +1819,7 @@ func flip(a *Container) *Container { func flipArray(b *Container) *Container { // TODO: actually implement this - x := b.clone() + x := b.Clone() x.arrayToBitmap() return flipBitmap(x) } @@ -1825,7 +1837,7 @@ func flipBitmap(b *Container) *Container { func flipRun(b *Container) *Container { // TODO: actually implement this - x := b.clone() + x := b.Clone() x.runToBitmap() return flipBitmap(x) } @@ -2206,7 +2218,7 @@ func unionArrayArray(a, b *Container) *Container { // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { - return b.clone() + return b.Clone() } output := &Container{containerType: ContainerRun} na, nb := len(a.array), len(b.runs) @@ -2263,10 +2275,10 @@ func (c *Container) runAppendInterval(v interval16) int { func unionRunRun(a, b *Container) *Container { if a.n == maxContainerVal+1 { - return a.clone() + return a.Clone() } if b.n == maxContainerVal+1 { - return b.clone() + return b.Clone() } na, nb := len(a.runs), len(b.runs) output := &Container{ @@ -2297,12 +2309,12 @@ func unionRunRun(a, b *Container) *Container { func unionBitmapRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { - return b.clone() + return b.Clone() } if a.n == maxContainerVal+1 { - return a.clone() + return a.Clone() } - output := a.clone() + output := a.Clone() for j := 0; j < len(b.runs); j++ { output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } @@ -2381,7 +2393,7 @@ func (c *Container) bitmapZeroRange(i, j uint64) { } func unionArrayBitmap(a, b *Container) *Container { - output := b.clone() + output := b.Clone() for _, v := range a.array { if !output.bitmapContains(v) { output.bitmap[v/64] |= (1 << uint64(v%64)) @@ -2464,7 +2476,7 @@ func differenceArrayRun(a, b *Container) *Container { // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { if a.n == 0 || b.n == 0 { - return a.clone() + return a.Clone() } output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray} @@ -2519,10 +2531,10 @@ func differenceArrayRun(a, b *Container) *Container { // differenceBitmapRun computes the difference of an bitmap from a run. func differenceBitmapRun(a, b *Container) *Container { if a.n == 0 || b.n == 0 { - return a.clone() + return a.Clone() } - output := a.clone() + output := a.Clone() for j := 0; j < len(b.runs); j++ { output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } @@ -2533,7 +2545,7 @@ func differenceBitmapRun(a, b *Container) *Container { // container. func differenceRunArray(a, b *Container) *Container { if a.n == 0 || b.n == 0 { - return a.clone() + return a.Clone() } output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} @@ -2645,7 +2657,7 @@ func differenceRunBitmap(a, b *Container) *Container { // differenceRunRun computes the difference of two runs. func differenceRunRun(a, b *Container) *Container { if a.n == 0 || b.n == 0 { - return a.clone() + return a.Clone() } apos := 0 // current a-run index @@ -2723,7 +2735,7 @@ func differenceArrayBitmap(a, b *Container) *Container { } func differenceBitmapArray(a, b *Container) *Container { - output := a.clone() + output := a.Clone() for _, v := range b.array { if output.bitmapContains(v) { @@ -2810,7 +2822,7 @@ func xorArrayArray(a, b *Container) *Container { } func xorArrayBitmap(a, b *Container) *Container { - output := b.clone() + output := b.Clone() for _, v := range a.array { if b.bitmapContains(v) { output.remove(v) @@ -3203,10 +3215,10 @@ type xorstm struct { func xorRunRun(a, b *Container) *Container { na, nb := len(a.runs), len(b.runs) if na == 0 { - return b.clone() + return b.Clone() } if nb == 0 { - return a.clone() + return a.Clone() } output := &Container{containerType: ContainerRun} @@ -3249,7 +3261,7 @@ func xorRunRun(a, b *Container) *Container { // xorRunRun computes the exclusive or of a bitmap and a run container. func xorBitmapRun(a, b *Container) *Container { - output := a.clone() + output := a.Clone() for j := 0; j < len(b.runs); j++ { output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index da542aa36..8abee4e8e 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3179,7 +3179,7 @@ func TestContainerCombinations(t *testing.T) { // Convert to all container types and check result. for _, ct := range containerTypes { - clone := ret.clone() + clone := ret.Clone() if ct == ContainerArray { if clone.isBitmap() { clone.bitmapToArray() From 513c7fd705d3bc413d62af81a5556ff4bb18a90b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 14 May 2018 17:32:30 -0500 Subject: [PATCH 29/48] B+tree integration work. Export necessary vars from roaring to fix b+tree containers implementation. Clean up naming. Use constructor replacement for enterprise integration. --- enterprise/b/containers_btree.go | 22 ++++-- fragment.go | 2 +- roaring/containers.go | 8 ++- roaring/containers_btree.go | 9 --- roaring/containers_slice.go | 7 -- roaring/containers_test.go | 4 +- roaring/roaring.go | 111 +++++++++++++++---------------- roaring/roaring_internal_test.go | 38 +++++------ server/enterprise.go | 13 ++++ 9 files changed, 110 insertions(+), 104 deletions(-) delete mode 100644 roaring/containers_btree.go delete mode 100644 roaring/containers_slice.go create mode 100644 server/enterprise.go diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 69075e9e7..66d20b9ca 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -2,6 +2,7 @@ package b import ( "io" + "log" "github.com/pilosa/pilosa/roaring" ) @@ -10,17 +11,26 @@ func cmp(a, b uint64) int { return int(a - b) } +type BTreeContainers struct { + tree *Tree + + lastKey uint64 + lastContainer *roaring.Container +} + func NewBTreeContainers() *BTreeContainers { return &BTreeContainers{ tree: TreeNew(cmp), } } -type BTreeContainers struct { - tree *Tree - - lastKey uint64 - lastContainer *roaring.Container +func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { + log.Println("btree bitmap") + b := &roaring.Bitmap{ + Containers: NewBTreeContainers(), + } + b.Add(a...) + return b } func (btc *BTreeContainers) Get(key uint64) *roaring.Container { @@ -96,7 +106,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { return btc.lastContainer } -func (btc *BTreeContainers) Clone() roaring.Containerser { +func (btc *BTreeContainers) Clone() roaring.Containers { nbtc := NewBTreeContainers() itr, err := btc.tree.SeekFirst() diff --git a/fragment.go b/fragment.go index 7168f4443..3c40f1e19 100644 --- a/fragment.go +++ b/fragment.go @@ -189,7 +189,7 @@ func (f *Fragment) Open() error { func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. if f.storage == nil { - f.storage = roaring.NewBitmap() + f.storage = roaring.NewFileBitmap() } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) diff --git a/roaring/containers.go b/roaring/containers.go index ecc69b6fd..f91360ee3 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -7,6 +7,10 @@ type SliceContainers struct { lastContainer *Container } +func NewSliceContainers() *SliceContainers { + return &SliceContainers{} +} + func (sc *SliceContainers) Get(key uint64) *Container { i := search64(sc.keys, key) if i < 0 { @@ -83,8 +87,8 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container { return sc.lastContainer } -func (sc *SliceContainers) Clone() Containerser { - other := NewContainers() +func (sc *SliceContainers) Clone() Containers { + other := NewSliceContainers() other.keys = make([]uint64, len(sc.keys)) other.containers = make([]*Container, len(sc.containers)) copy(other.keys, sc.keys) diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go deleted file mode 100644 index 008007a48..000000000 --- a/roaring/containers_btree.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build enterprise - -package roaring - -import "github.com/pilosa/pilosa/enterprise/b" - -func NewContainers() *b.BTreeContainers { - return &b.BTreeContainers{} -} diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go deleted file mode 100644 index 8642dbabc..000000000 --- a/roaring/containers_slice.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !enterprise - -package roaring - -func NewContainers() *SliceContainers { - return &SliceContainers{} -} diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 9acf9cc08..cb92e5d50 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -9,11 +9,11 @@ import ( // testContainersIterator(btc, t) //} func TestContainersBTreeIterator(t *testing.T) { - slc := NewContainers() + slc := NewSliceContainers() testContainersIterator(slc, t) } -func testContainersIterator(cs Containerser, t *testing.T) { +func testContainersIterator(cs Containers, t *testing.T) { itr, found := cs.Iterator(0) if found { t.Fatalf("shouldn't have found 0 in empty btc") diff --git a/roaring/roaring.go b/roaring/roaring.go index f279f20d5..88dbf3c31 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -64,7 +64,7 @@ const ( maxContainerVal = 0xffff ) -type Containerser interface { +type Containers interface { // Get returns nil if the key does not exist. Get(key uint64) *Container @@ -82,7 +82,7 @@ type Containerser interface { GetOrCreate(key uint64) *Container // Clone does a deep copy of Containers, including cloning all containers contained. - Clone() Containerser + Clone() Containers // Last returns the highest key and associated container. Last() (key uint64, c *Container) @@ -103,7 +103,7 @@ type Contiterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { - conts Containerser + Containers Containers // Number of operations written to the writer. opN int @@ -115,19 +115,14 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ - conts: NewContainers(), + Containers: NewSliceContainers(), } b.Add(a...) return b } -//func NewBTreeBitmap(a ...uint64) *Bitmap { -// b := &Bitmap{ -// conts: NewBTreeContainers(), -// } -// b.Add(a...) -// return b -//} +// NewFileBitmap returns a Bitmap with an initial set of values. +var NewFileBitmap func(a ...uint64) *Bitmap = NewBitmap // Clone returns a heap allocated copy of the bitmap. // Note: The OpWriter IS NOT copied to the new bitmap. @@ -138,7 +133,7 @@ func (b *Bitmap) Clone() *Bitmap { // Create a copy of the bitmap structure. other := &Bitmap{ - conts: b.conts.Clone(), + Containers: b.Containers.Clone(), } return other @@ -167,13 +162,13 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { } func (b *Bitmap) add(v uint64) bool { - cont := b.conts.GetOrCreate(highbits(v)) + cont := b.Containers.GetOrCreate(highbits(v)) return cont.add(lowbits(v)) } // Contains returns true if v is in the bitmap. func (b *Bitmap) Contains(v uint64) bool { - c := b.conts.Get(highbits(v)) + c := b.Containers.Get(highbits(v)) if c == nil { return false } @@ -201,7 +196,7 @@ func (b *Bitmap) Remove(a ...uint64) (changed bool, err error) { } func (b *Bitmap) remove(v uint64) bool { - c := b.conts.Get(highbits(v)) + c := b.Containers.Get(highbits(v)) if c == nil { return false } @@ -212,18 +207,18 @@ func (b *Bitmap) remove(v uint64) bool { // Max returns the highest value in the bitmap. // Returns zero if the bitmap is empty. func (b *Bitmap) Max() uint64 { - if b.conts.Size() == 0 { + if b.Containers.Size() == 0 { return 0 } - hb, c := b.conts.Last() + hb, c := b.Containers.Last() lb := c.max() return hb<<16 | uint64(lb) } // Count returns the number of bits set in the bitmap. func (b *Bitmap) Count() (n uint64) { - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() n += uint64(c.n) @@ -233,14 +228,14 @@ func (b *Bitmap) Count() (n uint64) { // CountRange returns the number of bits set between [start, end). func (b *Bitmap) CountRange(start, end uint64) (n uint64) { - if b.conts.Size() == 0 { + if b.Containers.Size() == 0 { return } skey := highbits(start) ekey := highbits(end) - citer, found := b.conts.Iterator(highbits(start)) + citer, found := b.Containers.Iterator(highbits(start)) // If range is entirely in one container then just count that range. if found && skey == ekey { citer.Next() @@ -328,21 +323,21 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) - citer, _ := b.conts.Iterator(hi0) + citer, _ := b.Containers.Iterator(hi0) other := NewBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { break } - other.conts.Put(off+(k-hi0), c) + other.Containers.Put(off+(k-hi0), c) } return other } // container returns the container with the given key. func (b *Bitmap) container(key uint64) *Container { - return b.conts.Get(key) + return b.Containers.Get(key) } // IntersectionCount returns the number of set bits that would result in an @@ -350,8 +345,8 @@ func (b *Bitmap) container(key uint64) *Container { // intersecting the two and counting the result. func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { var n uint64 - iiter, _ := b.conts.Iterator(0) - jiter, _ := other.conts.Iterator(0) + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() @@ -375,8 +370,8 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { output := NewBitmap() - iiter, _ := b.conts.Iterator(0) - jiter, _ := other.conts.Iterator(0) + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() @@ -388,7 +383,7 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.conts.Put(ki, intersect(ci, cj)) + output.Containers.Put(ki, intersect(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() @@ -401,22 +396,22 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { func (b *Bitmap) Union(other *Bitmap) *Bitmap { output := NewBitmap() - iiter, _ := b.conts.Iterator(0) - jiter, _ := other.conts.Iterator(0) + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.conts.Put(kj, cj.Clone()) + output.Containers.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.conts.Put(ki, union(ci, cj)) + output.Containers.Put(ki, union(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() @@ -429,21 +424,21 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { func (b *Bitmap) Difference(other *Bitmap) *Bitmap { output := NewBitmap() - iiter, _ := b.conts.Iterator(0) - jiter, _ := other.conts.Iterator(0) + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.conts.Put(ki, difference(ci, cj)) + output.Containers.Put(ki, difference(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() @@ -456,22 +451,22 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { func (b *Bitmap) Xor(other *Bitmap) *Bitmap { output := NewBitmap() - iiter, _ := b.conts.Iterator(0) - jiter, _ := other.conts.Iterator(0) + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.conts.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.conts.Put(kj, cj.Clone()) + output.Containers.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.conts.Put(ki, xor(ci, cj)) + output.Containers.Put(ki, xor(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() @@ -482,17 +477,17 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { // removeEmptyContainers deletes all containers that have a count of zero. func (b *Bitmap) removeEmptyContainers() { - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() if c.n == 0 { - b.conts.Remove(k) + b.Containers.Remove(k) } } } func (b *Bitmap) countEmptyContainers() int { result := 0 - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() if c.n == 0 { @@ -504,7 +499,7 @@ func (b *Bitmap) countEmptyContainers() int { // Optimize converts array and bitmap containers to run containers as necessary. func (b *Bitmap) Optimize() { - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() c.Optimize() @@ -552,7 +547,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Remove empty containers before persisting. //b.removeEmptyContainers() - containerCount := b.conts.Size() - b.countEmptyContainers() + containerCount := b.Containers.Size() - b.countEmptyContainers() headerSize := headerBaseSize byte2 := make([]byte, 2) byte4 := make([]byte, 4) @@ -571,7 +566,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Descriptive header section: encode keys and cardinality. // Key and cardinality are stored interleaved here, 12 bytes per container. - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { key, c := citer.Value() // Verify container count before writing. @@ -589,7 +584,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Offset header section: write the offset for each container block. // 4 bytes per container. offset := uint32(headerSize + (containerCount * (8 + 2 + 2 + 4))) - citer, _ = b.conts.Iterator(0) + citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() if c.n > 0 { @@ -605,7 +600,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { n = int64(headerSize + (containerCount * (8 + 2 + 2 + 4))) // Container storage section: write each container block. - citer, _ = b.conts.Iterator(0) + citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() if c.n > 0 { @@ -643,7 +638,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { - b.conts.PutContainerValues( + b.Containers.PutContainerValues( binary.LittleEndian.Uint64(buf[0:8]), byte(binary.LittleEndian.Uint16(buf[8:10])), int(binary.LittleEndian.Uint16(buf[10:12]))+1, @@ -652,7 +647,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { opsOffset := headerSize + int(keyN)*12 // Read container offsets and attach data. - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { offset := binary.LittleEndian.Uint32(buf[0:4]) // Verify the offset is within the bounds of the input data. @@ -734,10 +729,10 @@ func (b *Bitmap) Iterator() *Iterator { func (b *Bitmap) Info() BitmapInfo { info := BitmapInfo{ OpN: b.opN, - Containers: make([]ContainerInfo, 0, b.conts.Size()), + Containers: make([]ContainerInfo, 0, b.Containers.Size()), } - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() ci := c.info() @@ -752,7 +747,7 @@ func (b *Bitmap) Check() error { var a ErrorList // Check each container. - citer, _ := b.conts.Iterator(0) + citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() if err := c.check(); err != nil { @@ -816,7 +811,7 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.citer, _ = itr.bitmap.conts.Iterator(highbits(seek)) + itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek)) if !itr.citer.Next() { itr.c = nil return // eof @@ -3282,8 +3277,8 @@ func BitmapsEqual(b, c *Bitmap) error { return errors.New("opNs not equal") } - biter, _ := b.conts.Iterator(0) - citer, _ := c.conts.Iterator(0) + biter, _ := b.Containers.Iterator(0) + citer, _ := c.Containers.Iterator(0) bn, cn := biter.Next(), citer.Next() for ; bn && cn; bn, cn = biter.Next(), citer.Next() { bk, bc := biter.Value() diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 8abee4e8e..0202f8b23 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1782,7 +1782,7 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} ba := NewBitmap() - ba.conts.Put(0, ca) + ba.Containers.Put(0, ca) ba2 := NewBitmap() var buf bytes.Buffer _, err := ba.WriteTo(&buf) @@ -1793,8 +1793,8 @@ func TestWriteReadArray(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(ba2.conts.Get(0).array, ca.array) { - t.Fatalf("array test expected %x, but got %x", ca.array, ba2.conts.Get(0).array) + if !reflect.DeepEqual(ba2.Containers.Get(0).array, ca.array) { + t.Fatalf("array test expected %x, but got %x", ca.array, ba2.Containers.Get(0).array) } } @@ -1805,7 +1805,7 @@ func TestWriteReadBitmap(t *testing.T) { cb.bitmap[i] = 0x5555555555555555 } bb := NewBitmap() - bb.conts.Put(0, cb) + bb.Containers.Put(0, cb) bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) @@ -1816,8 +1816,8 @@ func TestWriteReadBitmap(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.conts.Get(0).bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.conts.Get(0).bitmap) + if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap) } } @@ -1828,7 +1828,7 @@ func TestWriteReadFullBitmap(t *testing.T) { cb.bitmap[i] = 0xffffffffffffffff } bb := NewBitmap() - bb.conts.Put(0, cb) + bb.Containers.Put(0, cb) bb2 := NewBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) @@ -1839,22 +1839,22 @@ func TestWriteReadFullBitmap(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(bb2.conts.Get(0).bitmap, cb.bitmap) { - t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.conts.Get(0).bitmap) + if !reflect.DeepEqual(bb2.Containers.Get(0).bitmap, cb.bitmap) { + t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.Containers.Get(0).bitmap) } - if bb2.conts.Get(0).n != cb.n { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.conts.Get(0).n) + if bb2.Containers.Get(0).n != cb.n { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) } - if bb2.conts.Get(0).count() != cb.count() { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.conts.Get(0).n) + if bb2.Containers.Get(0).count() != cb.count() { + t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) } } func TestWriteReadRun(t *testing.T) { cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} br := NewBitmap() - br.conts.Put(0, cr) + br.Containers.Put(0, cr) br2 := NewBitmap() var buf bytes.Buffer _, err := br.WriteTo(&buf) @@ -1865,8 +1865,8 @@ func TestWriteReadRun(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling: %v", err) } - if !reflect.DeepEqual(br2.conts.Get(0).runs, cr.runs) { - t.Fatalf("run test expected %x, but got %x", cr.runs, br2.conts.Get(0).runs) + if !reflect.DeepEqual(br2.Containers.Get(0).runs, cr.runs) { + t.Fatalf("run test expected %x, but got %x", cr.runs, br2.Containers.Get(0).runs) } } @@ -2125,7 +2125,7 @@ func TestXorBitmapRun(t *testing.T) { func TestIteratorArray(t *testing.T) { // use values that span two containers b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) - if !b.conts.Get(0).isArray() { + if !b.Containers.Get(0).isArray() { t.Fatalf("wrong container type") } @@ -2178,7 +2178,7 @@ func TestIteratorBitmap(t *testing.T) { for i := uint64(75000); i < 75100; i++ { b.Add(i) } - if !b.conts.Get(0).isBitmap() { + if !b.Containers.Get(0).isBitmap() { t.Fatalf("wrong container type") } @@ -2224,7 +2224,7 @@ func TestIteratorBitmap(t *testing.T) { func TestIteratorRuns(t *testing.T) { b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() - if !b.conts.Get(0).isRun() { + if !b.Containers.Get(0).isRun() { t.Fatalf("wrong container type") } diff --git a/server/enterprise.go b/server/enterprise.go new file mode 100644 index 000000000..41bf89179 --- /dev/null +++ b/server/enterprise.go @@ -0,0 +1,13 @@ +// +build enterprise + +package server + +import ( + "github.com/pilosa/pilosa/enterprise/b" + "github.com/pilosa/pilosa/roaring" +) + +func init() { + // Replace Bitmap constructor with B+Tree implementation + roaring.NewFileBitmap = b.NewBTreeBitmap +} From b4b28cb8bdda292224f9845cbbcdd2e8a75ca3b6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 14 May 2018 17:38:39 -0500 Subject: [PATCH 30/48] Add enterprise tests to CI and Makefile --- .travis.yml | 1 + Makefile | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3a9abe355..bd22a437f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: matrix: - GOARCH=386 - GOARCH=amd64 + - GOARCH=amd64 TESTFLAGS="-tags enterprise" install: - make install-dep install-statik vendor generate-statik script: diff --git a/Makefile b/Makefile index 94081cb52..76d3199d1 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,10 @@ vendor: Gopkg.toml test: vendor go test ./... $(TESTFLAGS) +# Run EE test suite +test-ee: + make test TESTFLAGS="-tags enterprise" + # Run test suite with coverage enabled cover: vendor mkdir -p build From 251d21dee1462e9b1024e05be3b7961bf7f29818 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 14 May 2018 17:51:10 -0500 Subject: [PATCH 31/48] Remove leftover debug log line --- enterprise/b/containers_btree.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 66d20b9ca..b082f3c8b 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -2,7 +2,6 @@ package b import ( "io" - "log" "github.com/pilosa/pilosa/roaring" ) @@ -25,7 +24,6 @@ func NewBTreeContainers() *BTreeContainers { } func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { - log.Println("btree bitmap") b := &roaring.Bitmap{ Containers: NewBTreeContainers(), } From 208dc1c8dcd72e6195bf7f07de5e12ce8edc3f9f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 14 May 2018 17:54:11 -0500 Subject: [PATCH 32/48] Add new things to PHONY --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76d3199d1..b219c2ac3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test +.PHONY: build build-ee check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test test-ee CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) From 1c715aebe7ff114449a624cde3bd7cf31d1988c5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 10:16:55 -0500 Subject: [PATCH 33/48] Improve enterprise build process --- Makefile | 29 ++++++++++++----------------- cmd/root.go | 6 +++++- version.go | 8 ++++++++ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index b219c2ac3..d566c4bd6 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,15 @@ -.PHONY: build build-ee check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test test-ee +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH) +VERSION_ID := $(if $(ENTERPRISE),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) -LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" +LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(ENTERPRISE)" GO_VERSION=latest +ENTERPRISE=0 +ENTERPRISE_TAG := $(if $(ENTERPRISE),-tags=enterprise) # Run tests and compile Pilosa default: test build @@ -24,11 +26,7 @@ vendor: Gopkg.toml # Run test suite test: vendor - go test ./... $(TESTFLAGS) - -# Run EE test suite -test-ee: - make test TESTFLAGS="-tags enterprise" + go test ./... $(ENTERPRISE_TAG) $(TESTFLAGS) # Run test suite with coverage enabled cover: vendor @@ -41,11 +39,7 @@ cover-viz: cover # Compile Pilosa build: vendor - go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa - -# Compile Pilosa EE -build-ee: vendor - go build -tags release -tags enterprise -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + go build -tags release $(ENTERPRISE_TAG) -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa # Create a single release build under the build directory release-build: vendor @@ -56,12 +50,13 @@ release-build: vendor # Error out if there are untracked changes in Git check-clean: - $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) # Create release build tarballs for all supported platforms. Linux compilation happens under Docker. release: check-clean $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1 $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 ENTERPRISE=1 $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 # Create branch-tagged pre-release for client library CI jobs @@ -78,7 +73,7 @@ prerelease-upload: prerelease # Install Pilosa install: vendor - go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + go install $(ENTERPRISE_TAG) -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast @@ -102,11 +97,11 @@ docker: # Compile Pilosa inside Docker container docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build $(ENTERPRISE_TAG) -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa # Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./... + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(ENTERPRISE_TAG) $(TESTFLAGS) ./... ###################### # Build dependencies # diff --git a/cmd/root.go b/cmd/root.go index 72e59abeb..ec9d3be15 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,6 +30,10 @@ import ( var subcommandFns = map[string]func(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command{} func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + productName := "Pilosa " + pilosa.Version + if pilosa.EnterpriseEnabled { + productName = "Pilosa Enterprise " + pilosa.Version + } rc := &cobra.Command{ Use: "pilosa", Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.", @@ -41,7 +45,7 @@ tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/ -Version: ` + pilosa.Version + ` +` + productName + ` Build Time: ` + pilosa.BuildTime + "\n", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { v := viper.New() diff --git a/version.go b/version.go index 3e148ea7b..04096f7cc 100644 --- a/version.go +++ b/version.go @@ -14,5 +14,13 @@ package pilosa +var Enterprise = "0" +var EnterpriseEnabled = false var Version = "v0.0.0" var BuildTime = "not recorded" + +func init() { + if Enterprise == "1" { + EnterpriseEnabled = true + } +} From 71373945da989d2fb239e36bc0b2db463657434b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 10:22:02 -0500 Subject: [PATCH 34/48] Rename Contiterator -> ContainerIterator --- enterprise/b/containers_btree.go | 2 +- roaring/containers.go | 2 +- roaring/roaring.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index b082f3c8b..634e7725c 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -133,7 +133,7 @@ func (btc *BTreeContainers) Size() int { return btc.tree.Len() } -func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.Contiterator, found bool) { +func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { e, ok := btc.tree.Seek(key) if ok { found = true diff --git a/roaring/containers.go b/roaring/containers.go index f91360ee3..ac523676c 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -120,7 +120,7 @@ func (sc *SliceContainers) seek(key uint64) (int, bool) { return i, found } -func (sc *SliceContainers) Iterator(key uint64) (citer Contiterator, found bool) { +func (sc *SliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { i, found := sc.seek(key) return &SliceIterator{e: sc, i: i}, found } diff --git a/roaring/roaring.go b/roaring/roaring.go index 88dbf3c31..ce0e6753b 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -93,10 +93,10 @@ type Containers interface { // Iterator returns a Contiterator which after a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a // container is found at key. - Iterator(key uint64) (citer Contiterator, found bool) + Iterator(key uint64) (citer ContainerIterator, found bool) } -type Contiterator interface { +type ContainerIterator interface { Next() bool Value() (uint64, *Container) } @@ -798,7 +798,7 @@ type BitmapInfo struct { // Iterator represents an iterator over a Bitmap. type Iterator struct { bitmap *Bitmap - citer Contiterator + citer ContainerIterator key uint64 c *Container j, k int // i: container; j: array index, bit index, or run index; k: offset within the run From 89f3c10d2c35e55dcc0ae91b6755d54484e60be5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 10:25:53 -0500 Subject: [PATCH 35/48] Clarify comment --- roaring/roaring.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ce0e6753b..820a0756f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -121,7 +121,8 @@ func NewBitmap(a ...uint64) *Bitmap { return b } -// NewFileBitmap returns a Bitmap with an initial set of values. +// NewFileBitmap returns a Bitmap with an initial set of values, used for file storage. +// By default, this is a copy of NewBitmap, but is replaced with B+Tree in server/enterprise.go var NewFileBitmap func(a ...uint64) *Bitmap = NewBitmap // Clone returns a heap allocated copy of the bitmap. From 57e904f1fcee575c54cf0249aba36c10510acfb0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 10:41:33 -0500 Subject: [PATCH 36/48] Re-add Todd\'s benchmark functions lost during merge --- fragment_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/fragment_test.go b/fragment_test.go index 2b0c363b4..b46869003 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1148,3 +1148,94 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } + +func BenchmarkFragment_Snapshot(b *testing.B) { + if *FragmentPath == "" { + b.Skip("no fragment specified") + } + + b.ReportAllocs() + // Open the fragment specified by the path. + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + if err := f.Open(); err != nil { + b.Fatal(err) + } + defer f.Close() + b.ResetTimer() + + // Reset timer and execute benchmark. + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + err := f.Snapshot() + if err != nil { + b.Fatalf("unexpected count (reopen): %s", err) + } + } +} + +func BenchmarkFragment_FullSnapshot(b *testing.B) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + // Generate some intersecting data. + maxX := 1048576 / 2 + sz := maxX + rows := make([]uint64, sz, sz) + cols := make([]uint64, sz, sz) + + max := 0 + for row := 0; row < 100; row++ { + val := 1 + i := 0 + for col := 0; col < SliceWidth/2; col++ { + rows[i] = uint64(row) + cols[i] = uint64(val) + val += 2 + i++ + } + if err := f.Import(rows, cols); err != nil { + b.Fatalf("Error Building Sample: %s", err) + } + if row > max { + max = row + } + } + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + if err := f.Snapshot(); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkFragment_Import(b *testing.B) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + maxX := 1048576 * 5 * 2 + sz := maxX + rows := make([]uint64, sz, sz) + cols := make([]uint64, sz, sz) + i := 0 + for row := 0; row < 100; row++ { + val := 1 + for col := 0; col < SliceWidth/2; col++ { + rows[i] = uint64(row) + cols[i] = uint64(val) + val += 2 + i++ + } + if i == maxX { + break + } + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := f.Import(rows, cols); err != nil { + b.Fatalf("Error Building Sample: %s", err) + } + } +} From 36aab7e2239f9beb6b2c937810c91e966fc9b115 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 10:56:57 -0500 Subject: [PATCH 37/48] Use new enterprise flag --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bd22a437f..f16be8363 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ env: matrix: - GOARCH=386 - GOARCH=amd64 - - GOARCH=amd64 TESTFLAGS="-tags enterprise" + - GOARCH=amd64 ENTERPRISE=1 install: - make install-dep install-statik vendor generate-statik script: From e8fcb0f05588ddf253f9e7f85d125db71fc64182 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 11:22:19 -0500 Subject: [PATCH 38/48] Use NewFileBitmap for tests to test btree/slice containers separately. --- Makefile | 1 - enterprise/enterprise.go | 11 ++ roaring/roaring_internal_test.go | 36 +++--- roaring/roaring_test.go | 186 +++++++++++++++---------------- server/enterprise.go | 8 +- 5 files changed, 123 insertions(+), 119 deletions(-) create mode 100644 enterprise/enterprise.go diff --git a/Makefile b/Makefile index d566c4bd6..b5c0a8afe 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(ENTERPRISE)" GO_VERSION=latest -ENTERPRISE=0 ENTERPRISE_TAG := $(if $(ENTERPRISE),-tags=enterprise) # Run tests and compile Pilosa diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go new file mode 100644 index 000000000..0807dae7b --- /dev/null +++ b/enterprise/enterprise.go @@ -0,0 +1,11 @@ +package enterprise + +import ( + "github.com/pilosa/pilosa/enterprise/b" + "github.com/pilosa/pilosa/roaring" +) + +func init() { + // Replace Bitmap constructor with B+Tree implementation + roaring.NewFileBitmap = b.NewBTreeBitmap +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 0202f8b23..52d92464a 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1513,7 +1513,7 @@ func MakeBitmap(start []uint64) []uint64 { return b } func MakeLastBitSet() []uint64 { - obj := NewBitmap(65535) + obj := NewFileBitmap(65535) c := obj.container(0) c.arrayToBitmap() return c.bitmap @@ -1781,9 +1781,9 @@ func TestDifferenceRunRun(t *testing.T) { func TestWriteReadArray(t *testing.T) { ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} - ba := NewBitmap() + ba := NewFileBitmap() ba.Containers.Put(0, ca) - ba2 := NewBitmap() + ba2 := NewFileBitmap() var buf bytes.Buffer _, err := ba.WriteTo(&buf) if err != nil { @@ -1804,9 +1804,9 @@ func TestWriteReadBitmap(t *testing.T) { for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } - bb := NewBitmap() + bb := NewFileBitmap() bb.Containers.Put(0, cb) - bb2 := NewBitmap() + bb2 := NewFileBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1827,9 +1827,9 @@ func TestWriteReadFullBitmap(t *testing.T) { for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } - bb := NewBitmap() + bb := NewFileBitmap() bb.Containers.Put(0, cb) - bb2 := NewBitmap() + bb2 := NewFileBitmap() var buf bytes.Buffer _, err := bb.WriteTo(&buf) if err != nil { @@ -1853,9 +1853,9 @@ func TestWriteReadFullBitmap(t *testing.T) { func TestWriteReadRun(t *testing.T) { cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} - br := NewBitmap() + br := NewFileBitmap() br.Containers.Put(0, cr) - br2 := NewBitmap() + br2 := NewFileBitmap() var buf bytes.Buffer _, err := br.WriteTo(&buf) if err != nil { @@ -2124,7 +2124,7 @@ func TestXorBitmapRun(t *testing.T) { func TestIteratorArray(t *testing.T) { // use values that span two containers - b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := NewFileBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) if !b.Containers.Get(0).isArray() { t.Fatalf("wrong container type") } @@ -2171,7 +2171,7 @@ func TestIteratorBitmap(t *testing.T) { // use values that span two containers // this dataset will update to bitmap after enough Adds, // but won't update to RLE until Optimize() is called - b := NewBitmap() + b := NewFileBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -2222,7 +2222,7 @@ func TestIteratorBitmap(t *testing.T) { } func TestIteratorRuns(t *testing.T) { - b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() if !b.Containers.Get(0).isRun() { t.Fatalf("wrong container type") @@ -2289,7 +2289,7 @@ func TestIteratorVarious(t *testing.T) { exp uint64 }{ { - bm: NewBitmap(3, 4, 5), + bm: NewFileBitmap(3, 4, 5), exp: 3, }, { @@ -2297,7 +2297,7 @@ func TestIteratorVarious(t *testing.T) { exp: 61221, }, { - bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + bm: NewFileBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), exp: 7, }, } @@ -2438,7 +2438,7 @@ func TestRunBinSearch(t *testing.T) { } } func TestBitmap_RemoveEmptyContainers(t *testing.T) { - bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) if bm1.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") @@ -2451,13 +2451,13 @@ func TestBitmap_RemoveEmptyContainers(t *testing.T) { } func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { - bm1 := NewBitmap(1<<16, 2<<16, 3<<16) + bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) bm1.Remove(2 << 16) var buf bytes.Buffer if _, err := bm1.WriteTo(&buf); err != nil { t.Fatalf("Failure to write to bitmap buffer. ") } - bm0 := NewBitmap() + bm0 := NewFileBitmap() bm0.UnmarshalBinary(buf.Bytes()) if bm0.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") @@ -2686,7 +2686,7 @@ func bitmapVariousContainers() *Bitmap { bits = append(bits, bitCont(7, true, true, true)...) bits = append(bits, arrCont(8, true, true, true)...) bits = append(bits, rleCont(9, true, true, true)...) - bm := NewBitmap(bits...) + bm := NewFileBitmap(bits...) bm.Optimize() return bm } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 2f2c4cda2..e0385c067 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -30,7 +30,7 @@ import ( ) func TestBitmapClone(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewFileBitmap() for i := uint64(61000); i < 71000; i++ { b.Add(i) } @@ -48,7 +48,7 @@ func TestBitmapClone(t *testing.T) { } func TestContainerCount(t *testing.T) { - b := roaring.NewBitmap(65535) + b := roaring.NewFileBitmap(65535) if b.Count() != b.CountRange(0, 65546) { t.Fatalf("Count != CountRange\n") @@ -144,7 +144,7 @@ func TestCountRange(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) { - b := roaring.NewBitmap(test.bitmap...) + b := roaring.NewFileBitmap(test.bitmap...) actual := b.CountRange(test.start, test.end) if actual != test.exp { t.Errorf("got: %d, exp: %d", actual, test.exp) @@ -154,7 +154,7 @@ func TestCountRange(t *testing.T) { } func TestCheckBitmap(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewFileBitmap() x := 0 for i := uint64(61000); i < 71000; i++ { x++ @@ -171,7 +171,7 @@ func TestCheckBitmap(t *testing.T) { } func TestCheckArray(t *testing.T) { - b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) + b := roaring.NewFileBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000) err := b.Check() if err != nil { t.Fatalf("%v\n", err) @@ -179,7 +179,7 @@ func TestCheckArray(t *testing.T) { } func TestCheckRun(t *testing.T) { - b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) + b := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005) b.Optimize() // convert to runs err := b.Check() if err != nil { @@ -187,7 +187,7 @@ func TestCheckRun(t *testing.T) { } } func TestCheckFullRun(t *testing.T) { - b := roaring.NewBitmap() + b := roaring.NewFileBitmap() for i := uint64(0); i < 2097152; i++ { if i%16384 == 0 { b.Optimize() // convert to runs @@ -208,7 +208,7 @@ func TestCheckFullRun(t *testing.T) { // Ensure that we can transition between runs and arrays when materializing the bitmap. func TestContainerTransitions(t *testing.T) { // [run, run][array][run] - b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) + b := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005) b.Optimize() // convert to runs if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) { t.Fatalf("unexpected slice: %+v", b.Slice()) @@ -216,7 +216,7 @@ func TestContainerTransitions(t *testing.T) { // Test the case where last and first bits of adjoining containers are set. // [run][array][run] - b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) + b2 := roaring.NewFileBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076) b2.Optimize() // convert to runs if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) { t.Fatalf("unexpected slice: %+v", b2.Slice()) @@ -225,26 +225,26 @@ func TestContainerTransitions(t *testing.T) { // Ensure an empty bitmap returns false if checking for existence. func TestBitmap_Contains_Empty(t *testing.T) { - if roaring.NewBitmap().Contains(1000) { + if roaring.NewFileBitmap().Contains(1000) { t.Fatal("expected false") } } // Ensure an empty bitmap does nothing when removing an element. func TestBitmap_Remove_Empty(t *testing.T) { - roaring.NewBitmap().Remove(1000) + roaring.NewFileBitmap().Remove(1000) } // Ensure a bitmap can return a slice of values. func TestBitmap_Slice(t *testing.T) { - if a := roaring.NewBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := roaring.NewFileBitmap(1, 2, 3).Slice(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected slice: %+v", a) } } // Ensure an empty bitmap returns an empty slice of values. func TestBitmap_Slice_Empty(t *testing.T) { - if a := roaring.NewBitmap().Slice(); len(a) != 0 { + if a := roaring.NewFileBitmap().Slice(); len(a) != 0 { t.Fatalf("unexpected slice: %+v", a) } } @@ -252,7 +252,7 @@ func TestBitmap_Slice_Empty(t *testing.T) { // Ensure a bitmap can return a slice of values within a range. // TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { - if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { + if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { t.Fatalf("unexpected slice: %+v", a) } } @@ -260,7 +260,7 @@ func TestBitmap_SliceRange(t *testing.T) { // Ensure a bitmap can loop over a set of values. func TestBitmap_ForEach(t *testing.T) { var a []uint64 - roaring.NewBitmap(1, 2, 3).ForEach(func(v uint64) { + roaring.NewFileBitmap(1, 2, 3).ForEach(func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{1, 2, 3}) { @@ -271,7 +271,7 @@ func TestBitmap_ForEach(t *testing.T) { // Ensure a bitmap can loop over a set of values in a range. func TestBitmap_ForEachRange(t *testing.T) { var a []uint64 - roaring.NewBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { + roaring.NewFileBitmap(1, 2, 3, 4).ForEachRange(2, 4, func(v uint64) { a = append(a, v) }) if !reflect.DeepEqual(a, []uint64{2, 3}) { @@ -281,7 +281,7 @@ func TestBitmap_ForEachRange(t *testing.T) { // Ensure bitmap can return the highest value. func TestBitmap_Max(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for i := uint64(1000); i <= 100000; i++ { bm.Add(i) @@ -297,7 +297,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { e := uint64(2010 * 1048576) start := s + (39314024 % 1048576) - bm0 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { start += 16384 @@ -315,7 +315,7 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { } func TestBitmap_BitmapCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) + bm0 := roaring.NewFileBitmap(0, 2683177) for i := uint64(628); i < 2683301; i++ { bm0.Add(i) } @@ -343,20 +343,20 @@ func TestBitmap_BitmapCountRange(t *testing.T) { } func TestBitmap_ArrayCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177, 2683313) + bm0 := roaring.NewFileBitmap(0, 2683177, 2683313) if n := bm0.CountRange(1, 2683313); n != 1 { t.Fatalf("unexpected n: %d", n) } } func TestBitmap_RunCountRange(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) + bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014) bm0.Optimize() // convert to runs if n := bm0.CountRange(15, 1000003); n != 5 { t.Fatalf("unexpected n: %d", n) } - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) bm1.Optimize() // convert to runs if n := bm1.CountRange(5, 12); n != 7 { t.Fatalf("unexpected n: %d", n) @@ -364,8 +364,8 @@ func TestBitmap_RunCountRange(t *testing.T) { } func TestBitmap_Intersectionz(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(0, 2683177) + bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -378,8 +378,8 @@ func TestBitmap_Intersectionz(t *testing.T) { } func TestBitmap_Union1(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(0, 2683177) + bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -402,8 +402,8 @@ func TestBitmap_Union1(t *testing.T) { } func TestBitmap_Intersection_Empty(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(0, 2683177) + bm1 := roaring.NewFileBitmap() result := bm0.Intersect(bm1) if n := result.Count(); n != 0 { @@ -413,8 +413,8 @@ func TestBitmap_Intersection_Empty(t *testing.T) { } func TestBitmap_IntersectArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2683, 5005) - bm1 := roaring.NewBitmap(0, 2683, 2684, 5000) + bm0 := roaring.NewFileBitmap(0, 1, 2683, 5005) + bm1 := roaring.NewFileBitmap(0, 2683, 2684, 5000) result := bm0.Intersect(bm1) if n := result.Count(); n != 2 { @@ -423,12 +423,12 @@ func TestBitmap_IntersectArrayArray(t *testing.T) { } func TestBitmap_IntersectBitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 2 { bm0.Add(i) } - bm1 := roaring.NewBitmap() + bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 3 { bm1.Add(i) } @@ -441,9 +441,9 @@ func TestBitmap_IntersectBitmapBitmap(t *testing.T) { func TestBitmap_IntersectRunRun(t *testing.T) { // Intersect two runs that result in an array. - bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) + bm0 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15) bm0.Optimize() // convert to runs - bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11) + bm1 := roaring.NewFileBitmap(5, 6, 7, 8, 9, 10, 11) bm1.Optimize() // convert to runs result := bm0.Intersect(bm1) if n := result.Count(); n != 3 { @@ -451,7 +451,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } // Intersect two runs that result in a bitmap. - bm2 := roaring.NewBitmap() + bm2 := roaring.NewFileBitmap() runLen := uint64(25) spaceLen := uint64(8) offset := (runLen / 2) + spaceLen @@ -461,7 +461,7 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } } bm2.Optimize() // convert to runs - bm3 := roaring.NewBitmap() + bm3 := roaring.NewFileBitmap() runLen = uint64(32) spaceLen = uint64(1) for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { @@ -477,8 +477,8 @@ func TestBitmap_IntersectRunRun(t *testing.T) { } func TestBitmap_Difference(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(0, 2683177) + bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { bm1.Add(i) } @@ -489,8 +489,8 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) + bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) result := bm0.Difference(bm1) if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) @@ -498,8 +498,8 @@ func TestBitmap_Difference2(t *testing.T) { } func TestBitmap_Difference_Empty(t *testing.T) { - bm0 := roaring.NewBitmap(0, 2683177) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(0, 2683177) + bm1 := roaring.NewFileBitmap() result := bm0.Difference(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -507,8 +507,8 @@ func TestBitmap_Difference_Empty(t *testing.T) { } func TestBitmap_DifferenceArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20) - bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18) + bm0 := roaring.NewFileBitmap(0, 4, 8, 12, 16, 20) + bm1 := roaring.NewFileBitmap(1, 3, 6, 9, 12, 15, 18) result := bm0.Difference(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -516,9 +516,9 @@ func TestBitmap_DifferenceArrayArray(t *testing.T) { } func TestBitmap_DifferenceArrayRun(t *testing.T) { - bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) + bm0 := roaring.NewFileBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44) - bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) + bm1 := roaring.NewFileBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36) bm1.Optimize() // convert to runs result := bm0.Difference(bm1) if n := result.Count(); n != 6 { @@ -527,8 +527,8 @@ func TestBitmap_DifferenceArrayRun(t *testing.T) { } func TestBitmap_Union(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) result := bm0.Union(bm1) if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) @@ -537,7 +537,7 @@ func TestBitmap_Union(t *testing.T) { func TestBitmap_Xor(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewBitmap(0, 1, 2, 3) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3) result := bm1.Xor(bm0) if n := result.Count(); n != 75011 { t.Fatalf("unexpected n: %d", n) @@ -555,8 +555,8 @@ func TestBitmap_Xor(t *testing.T) { } func TestBitmap_Xor_ArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) result := bm0.Xor(bm1) if n := result.Count(); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -572,8 +572,8 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) { //empty array test func TestBitmap_Xor_Empty(t *testing.T) { - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) - empty := roaring.NewBitmap() + bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) + empty := roaring.NewFileBitmap() result := bm1.Xor(empty) if n := result.Count(); n != 4 { @@ -581,8 +581,8 @@ func TestBitmap_Xor_Empty(t *testing.T) { } } func TestBitmap_Xor_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) } @@ -603,7 +603,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { t.Fatalf("test 3 unexpected n: %d", n) } - empty := roaring.NewBitmap() + empty := roaring.NewFileBitmap() result = bm1.Xor(empty) if n := result.Count(); n != 5000 { t.Fatalf("unexpected n: %d", n) @@ -611,8 +611,8 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { } func TestBitmap_Xor_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap() + bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { bm1.Add(i) @@ -630,7 +630,7 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) { // Ensure bitmap contents alternate. func TestBitmap_Flip_Empty(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() results := bm.Flip(0, 10) if n := results.Count(); n != 11 { t.Fatalf("unexpected n: %d", n) @@ -643,7 +643,7 @@ func TestBitmap_Flip_Empty(t *testing.T) { // Test Subrange Flip should not affect bits outside of Range func TestBitmap_Flip_Array(t *testing.T) { - bm := roaring.NewBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) + bm := roaring.NewFileBitmap(0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024) results := bm.Flip(0, 4) if !reflect.DeepEqual(results.Slice(), []uint64{8, 16, 32, 64, 128, 256, 512, 1024}) { t.Fatalf("unexpected %v ", results.Slice()) @@ -657,7 +657,7 @@ func TestBitmap_Flip_Array(t *testing.T) { // Ensure Flip works with underlying Bitmap container. func TestBitmap_Flip_Bitmap(t *testing.T) { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() size := uint64(10000) for i := uint64(0); i < size; i += 2 { bm.Add(i) @@ -674,7 +674,7 @@ func TestBitmap_Flip_Bitmap(t *testing.T) { // Verify Flip works correctly with in different regions of bitmap, beginning, middle, and end. func TestBitmap_Flip_After(t *testing.T) { - bm := roaring.NewBitmap(0, 2, 4, 8) + bm := roaring.NewFileBitmap(0, 2, 4, 8) results := bm.Flip(9, 10) if !reflect.DeepEqual(results.Slice(), []uint64{0, 2, 4, 8, 9, 10}) { @@ -693,8 +693,8 @@ func TestBitmap_Flip_After(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002) + bm0 := roaring.NewFileBitmap(0, 1, 1000001, 1000002, 1000003) + bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) if n := bm0.IntersectionCount(bm1); n != 3 { t.Fatalf("unexpected n: %d", n) @@ -705,8 +705,8 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { - bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003) - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 3 { @@ -718,9 +718,9 @@ func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_RunRun(t *testing.T) { - bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) + bm0 := roaring.NewFileBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004) bm0.Optimize() // convert to runs - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 6 { @@ -732,11 +732,11 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { - bm0 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap() for i := uint64(3); i <= 1000006; i += 2 { bm0.Add(i) } - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs if n := bm0.IntersectionCount(bm1); n != 4 { @@ -748,8 +748,8 @@ func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { - bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098) - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) + bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { bm1.Add(i) } @@ -763,8 +763,8 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { // Ensure bitmap can return the number of intersecting bits in two bitmaps. func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { - bm0 := roaring.NewBitmap() - bm1 := roaring.NewBitmap() + bm0 := roaring.NewFileBitmap() + bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { bm0.Add(i) bm1.Add(i + 1) @@ -784,8 +784,8 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { } func TestBitmap_IntersectionCount_Mixed(t *testing.T) { bm0 := testBM() - bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) - bm3 := roaring.NewBitmap(131072) + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536) + bm3 := roaring.NewFileBitmap(131072) if n := bm0.IntersectionCount(bm0); n != bm0.Count() { t.Fatalf("unexpected n: %d", n) @@ -807,7 +807,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { quick.Check(func(a []uint64) bool { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() m := make(map[uint64]struct{}) // Add values to the bitmap and set. @@ -894,7 +894,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { quick.Check(func(a0, a1 []uint64) bool { // Create bitmap with initial values set. - bm := roaring.NewBitmap(a0...) + bm := roaring.NewFileBitmap(a0...) set := make(map[uint64]struct{}) for _, v := range a0 { @@ -923,7 +923,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { data := buf.Bytes() // Create new bitmap from ops log data. - bm2 := roaring.NewBitmap() + bm2 := roaring.NewFileBitmap() if err := bm2.UnmarshalBinary(data); err != nil { t.Fatal(err) } @@ -952,7 +952,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { // TODO duplicate for all container types func TestIterator(t *testing.T) { t.Run("bitmap", func(t *testing.T) { - itr := roaring.NewBitmap(1, 2, 3).Iterator() + itr := roaring.NewFileBitmap(1, 2, 3).Iterator() itr.Seek(0) var a []uint64 @@ -966,13 +966,13 @@ func TestIterator(t *testing.T) { }) t.Run("run", func(t *testing.T) { - bm1 := roaring.NewBitmap() + bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 11; i += 1 { bm1.Add(i) } bm1.Optimize() - bm2 := roaring.NewBitmap() + bm2 := roaring.NewFileBitmap() for i := uint64(0); i < 12; i += 1 { bm2.Add(i) } @@ -1005,7 +1005,7 @@ func TestIterator(t *testing.T) { // testBM creates a bitmap with 3 containers: array, bitmap, and run. func testBM() *roaring.Bitmap { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() //the array for i := uint64(0); i < 1024; i += 4 { bm.Add((1 << 16) + i) @@ -1068,19 +1068,19 @@ func getBenchData() *struct{ a, b, r *roaring.Bitmap } { const max = (1 << 24) / 64 // Build bitmap with array container. - data.a = roaring.NewBitmap() + data.a = roaring.NewFileBitmap() for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ { data.a.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. - data.b = roaring.NewBitmap() + data.b = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal/3; i < n; i++ { data.b.Add(uint64(i * 3)) } // build bitmap with run container - data.r = roaring.NewBitmap() + data.r = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { data.r.Add(uint64(i)) } @@ -1176,7 +1176,7 @@ const ( func BenchmarkContainerLinear(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1187,7 +1187,7 @@ func BenchmarkContainerLinear(b *testing.B) { func BenchmarkContainerReverse(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1198,7 +1198,7 @@ func BenchmarkContainerReverse(b *testing.B) { func BenchmarkContainerColumn(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1210,7 +1210,7 @@ func BenchmarkContainerColumn(b *testing.B) { func BenchmarkContainerOutsideIn(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { @@ -1224,7 +1224,7 @@ func BenchmarkContainerOutsideIn(b *testing.B) { func BenchmarkContainerInsideOut(b *testing.B) { middle := NumRows / uint64(2) for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) @@ -1236,7 +1236,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for col := uint64(0); col < pilosa.SliceWidth; col++ { bm.Add(col) } @@ -1245,7 +1245,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { - bm := roaring.NewBitmap() + bm := roaring.NewFileBitmap() for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { bm.Add(col) } diff --git a/server/enterprise.go b/server/enterprise.go index 41bf89179..320c90a4c 100644 --- a/server/enterprise.go +++ b/server/enterprise.go @@ -3,11 +3,5 @@ package server import ( - "github.com/pilosa/pilosa/enterprise/b" - "github.com/pilosa/pilosa/roaring" + _ "github.com/pilosa/pilosa/enterprise" ) - -func init() { - // Replace Bitmap constructor with B+Tree implementation - roaring.NewFileBitmap = b.NewBTreeBitmap -} From cfb914651528106c520c685692a46246a8b3db6e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 11:30:37 -0500 Subject: [PATCH 39/48] Use NewFileBitmap to get container implementation for testing ContainersIterator. --- roaring/containers_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/roaring/containers_test.go b/roaring/containers_test.go index cb92e5d50..c7d847c05 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -4,15 +4,11 @@ import ( "testing" ) -//func TestContainersSliceIterator(t *testing.T) { -// btc := NewBTreeContainers() -// testContainersIterator(btc, t) -//} -func TestContainersBTreeIterator(t *testing.T) { - slc := NewSliceContainers() +func TestContainersIterator(t *testing.T) { + slc := NewFileBitmap().Containers testContainersIterator(slc, t) - } + func testContainersIterator(cs Containers, t *testing.T) { itr, found := cs.Iterator(0) if found { From 5103bfd8cec88d5530717e4383304c31246b7c24 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 11:34:47 -0500 Subject: [PATCH 40/48] Remove errant "z". --- roaring/roaring_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index e0385c067..f206e86e5 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -363,7 +363,7 @@ func TestBitmap_RunCountRange(t *testing.T) { } } -func TestBitmap_Intersectionz(t *testing.T) { +func TestBitmap_Intersection(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { From 1f925324d753630b7567596fed38ad8bb009b175 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 12:20:45 -0500 Subject: [PATCH 41/48] Add enterprise license --- Makefile | 1 + NOTICE | 24 +- enterprise/COPYING | 661 +++++++++++++++++++++++++++++++ enterprise/b/containers_btree.go | 17 + enterprise/enterprise.go | 17 + 5 files changed, 718 insertions(+), 2 deletions(-) create mode 100644 enterprise/COPYING diff --git a/Makefile b/Makefile index b5c0a8afe..1c3b6aff9 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ build: vendor release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) + %(if $(ENTERPRISE),cp enterprise/COPYING build/pilosa-$(VERSION_ID)) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz diff --git a/NOTICE b/NOTICE index dc9506565..99ea00db4 100644 --- a/NOTICE +++ b/NOTICE @@ -1,7 +1,7 @@ Software license ================ -Copyright 2017 Pilosa Corp. +Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"). You may obtain a copy of the License at @@ -14,6 +14,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +Enterprise Edition software license +=================================== + +Files contained under the directory `enterprise` are subject to the following +license notice (Full license included in the file `COPYING`): + + Copyright (C) 2018 Pilosa Corp. All rights reserved. + + Pilosa Enterprise Edition is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Pilosa Enterprise Edition is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Pilosa Enterprise Edition. If not, see . Third-party software licenses ============================= @@ -35,7 +55,7 @@ The file /pilosa/lru/lru.go contains a redistribution of lru See the License for the specific language governing permissions and limitations under the License. -The file /pilosa/roaring/btree.go contains a modified redistribution of b +The file /enterprise/b/btree.go contains a modified redistribution of b (https://github.com/cznic/b); the license follows: Copyright (c) 2014 The b Authors. All rights reserved. diff --git a/enterprise/COPYING b/enterprise/COPYING new file mode 100644 index 000000000..be3f7b28e --- /dev/null +++ b/enterprise/COPYING @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 634e7725c..257134a82 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -1,3 +1,20 @@ +// Copyright (c) 2018 Pilosa Corp. All rights reserved. +// +// This file is part of Pilosa Enterprise Edition. +// +// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Pilosa Enterprise Edition is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with Pilosa Enterprise Edition. If not, see . + package b import ( diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index 0807dae7b..ed8ea0e8e 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -1,3 +1,20 @@ +// Copyright (c) 2018 Pilosa Corp. All rights reserved. +// +// This file is part of Pilosa Enterprise Edition. +// +// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Pilosa Enterprise Edition is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with Pilosa Enterprise Edition. If not, see . + package enterprise import ( From b1eb137a20186ef0605aa01746fe5a3a98ef27e3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 15 May 2018 12:26:33 -0500 Subject: [PATCH 42/48] Add missing license headers. --- roaring/containers.go | 14 ++++++++++++++ roaring/containers_test.go | 14 ++++++++++++++ server/enterprise.go | 14 ++++++++++++++ utils_test.go | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/roaring/containers.go b/roaring/containers.go index ac523676c..08c8d25eb 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -1,3 +1,17 @@ +// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring type SliceContainers struct { diff --git a/roaring/containers_test.go b/roaring/containers_test.go index c7d847c05..ea3106a96 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -1,3 +1,17 @@ +// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import ( diff --git a/server/enterprise.go b/server/enterprise.go index 320c90a4c..6dd3e5540 100644 --- a/server/enterprise.go +++ b/server/enterprise.go @@ -1,3 +1,17 @@ +// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // +build enterprise package server diff --git a/utils_test.go b/utils_test.go index 05afeac19..6f9eca75a 100644 --- a/utils_test.go +++ b/utils_test.go @@ -1,3 +1,17 @@ +// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( From b363debd8309dbe6b6d83cae06a2e719a35852e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 15 May 2018 12:18:02 -0500 Subject: [PATCH 43/48] replace "Pilosa starting..." log line --- server/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/server.go b/server/server.go index eb660fc23..27c6c06e5 100644 --- a/server/server.go +++ b/server/server.go @@ -159,6 +159,12 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "setting up logger") } + productName := "Pilosa" + if pilosa.EnterpriseEnabled { + productName += " Enterprise" + } + m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) + handler := pilosa.NewHandler() handler.Logger = m.logger handler.FileSystem = &statik.FileSystem{} From f9aa8549187911e768155a1207d8a2d89bd60b54 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 15 May 2018 15:02:22 -0500 Subject: [PATCH 44/48] add Licenses section to readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5ae041dfd..7c1c7cfda 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,14 @@ There are supported libraries for the following languages: - [Java](https://www.pilosa.com/docs/client-libraries/#java) - [Python](https://www.pilosa.com/docs/client-libraries/#python) +## Licenses + +The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0. +If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General +Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise) +directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and +[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses. + ## Get Support There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support. From 00a39aa3385d169e6d4b04dde0f5ae8ea09999cb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 15 May 2018 15:17:03 -0500 Subject: [PATCH 45/48] implement Container.equals and get rid of reflect --- roaring/roaring.go | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 820a0756f..0fd697b2f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -22,7 +22,6 @@ import ( "hash/fnv" "io" "math/bits" - "reflect" "sort" "unsafe" ) @@ -2388,6 +2387,43 @@ func (c *Container) bitmapZeroRange(i, j uint64) { } } +func (c *Container) equals(c2 *Container) bool { + if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n { + return false + } + if c.containerType == ContainerArray { + if len(c.array) != len(c2.array) { + return false + } + for i := 0; i < len(c.array); i++ { + if c.array[i] != c2.array[i] { + return false + } + } + } else if c.containerType == ContainerBitmap { + if len(c.bitmap) != len(c2.bitmap) { + return false + } + for i := 0; i < len(c.bitmap); i++ { + if c.bitmap[i] != c2.bitmap[i] { + return false + } + } + } else if c.containerType == ContainerRun { + if len(c.runs) != len(c2.runs) { + return false + } + for i := 0; i < len(c.runs); i++ { + if c.runs[i] != c2.runs[i] { + return false + } + } + } else { + panic(fmt.Sprintf("unknown container type: %v", c.containerType)) + } + return true +} + func unionArrayBitmap(a, b *Container) *Container { output := b.Clone() for _, v := range a.array { @@ -3287,7 +3323,7 @@ func BitmapsEqual(b, c *Bitmap) error { if bk != ck { return errors.New("keys not equal") } - if !reflect.DeepEqual(bc, cc) { + if !bc.equals(cc) { return errors.New("containers not equal") } } From 61fcf99f3ea155866423a624a9a45c1c9cc0fdd8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 15 May 2018 15:53:26 -0500 Subject: [PATCH 46/48] unexport bitmapsEqual --- roaring/roaring.go | 2 +- roaring/roaring_internal_test.go | 18 ++++++++++++++++++ roaring/roaring_test.go | 18 ------------------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 0fd697b2f..cc8466a5f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3306,7 +3306,7 @@ func xorBitmapRun(a, b *Container) *Container { return output } -func BitmapsEqual(b, c *Bitmap) error { +func bitmapsEqual(b, c *Bitmap) error { if b.OpWriter != c.OpWriter { return errors.New("opWriters not equal") } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 52d92464a..6e1f105bf 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2609,6 +2609,24 @@ func TestIntersectArrayBitmap(t *testing.T) { } } +func TestBitmapClone(t *testing.T) { + b := NewFileBitmap() + for i := uint64(61000); i < 71000; i++ { + b.Add(i) + } + c := b.Clone() + if err := bitmapsEqual(b, c); err != nil { + t.Fatalf("Clone Objects not equal: %v\n", err) + } + d := func() *Bitmap { //anybody know how to declare a nil value? + return nil + }() + e := d.Clone() + if e != nil { + t.Fatalf("Clone nil Objects not equal\n") + } +} + // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f206e86e5..c1291d1f7 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -29,24 +29,6 @@ import ( _ "github.com/pilosa/pilosa/test" ) -func TestBitmapClone(t *testing.T) { - b := roaring.NewFileBitmap() - for i := uint64(61000); i < 71000; i++ { - b.Add(i) - } - c := b.Clone() - if err := roaring.BitmapsEqual(b, c); err != nil { - t.Fatalf("Clone Objects not equal: %v\n", err) - } - d := func() *roaring.Bitmap { //anybody know how to declare a nil value? - return nil - }() - e := d.Clone() - if e != nil { - t.Fatalf("Clone nil Objects not equal\n") - } -} - func TestContainerCount(t *testing.T) { b := roaring.NewFileBitmap(65535) From c27faaa2718743a2533b4f07f02ad46ea1c515f8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 15 May 2018 16:07:58 -0500 Subject: [PATCH 47/48] add GOARCH=386 ENTERPRISE=1 to travisCI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f16be8363..6d5674662 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" matrix: - GOARCH=386 + - GOARCH=386 ENTERPRISE=1 - GOARCH=amd64 - GOARCH=amd64 ENTERPRISE=1 install: From 4f3a4864f0e361f4d645bd9639b9c8851073fed1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 15 May 2018 16:08:22 -0500 Subject: [PATCH 48/48] add godoc comment to enterprise/enterprise.go --- enterprise/enterprise.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index ed8ea0e8e..db9d6e2dd 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -15,6 +15,10 @@ // You should have received a copy of the GNU Affero General Public License // along with Pilosa Enterprise Edition. If not, see . +// Package enterprise injects enterprise implementations of various Pilosa +// features when Pilosa is built with "ENTERPRISE=1 make install". These +// features are dual-licensed separately from Pilosa community edition under the +// AGPL and Pilosa's commercial license. package enterprise import (