mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-08 03:47:51 +00:00
wip on adding RLE to roaring, add, countRange, contains
This commit is contained in:
Matt Jaffee
parent
ddb4adcc47
commit
c2203d19c1
2 changed files with 275 additions and 12 deletions
|
|
@ -902,7 +902,23 @@ type container struct {
|
|||
n int // number of integers in container
|
||||
array []uint32 // used for array containers
|
||||
bitmap []uint64 // used for bitmap containers
|
||||
mapped bool // mapped directly to a byte slice when true
|
||||
runs []interval16
|
||||
mapped bool // mapped directly to a byte slice when true
|
||||
}
|
||||
|
||||
type interval16 struct {
|
||||
start uint16
|
||||
last uint16
|
||||
}
|
||||
|
||||
// runlen returns the count of integers in the interval.
|
||||
func (iv interval16) runlen() int {
|
||||
return 1 + int(iv.last) - int(iv.start)
|
||||
}
|
||||
|
||||
// String produces a human viewable string of the contents.
|
||||
func (iv interval16) String() string {
|
||||
return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
|
||||
}
|
||||
|
||||
// newContainer returns a new instance of container.
|
||||
|
|
@ -911,7 +927,17 @@ func newContainer() *container {
|
|||
}
|
||||
|
||||
// isArray returns true if the container is an array container.
|
||||
func (c *container) isArray() bool { return c.bitmap == nil }
|
||||
func (c *container) isArray() bool {
|
||||
return c.bitmap == nil && c.runs == nil
|
||||
}
|
||||
|
||||
func (c *container) isBitmap() bool {
|
||||
return c.array == nil && c.runs == nil
|
||||
}
|
||||
|
||||
func (c *container) isRun() bool {
|
||||
return c.array == nil && c.bitmap == nil
|
||||
}
|
||||
|
||||
// unmap creates copies of the containers data in the heap.
|
||||
//
|
||||
|
|
@ -932,6 +958,11 @@ func (c *container) unmap() {
|
|||
copy(tmp, c.bitmap)
|
||||
c.bitmap = tmp
|
||||
}
|
||||
if c.runs != nil {
|
||||
tmp := make([]interval16, len(c.runs))
|
||||
copy(tmp, c.runs)
|
||||
c.runs = tmp
|
||||
}
|
||||
c.mapped = false
|
||||
}
|
||||
|
||||
|
|
@ -944,8 +975,10 @@ func (c *container) count() (n int) {
|
|||
func (c *container) countRange(start, end uint32) (n int) {
|
||||
if c.isArray() {
|
||||
return c.arrayCountRange(start, end)
|
||||
} else if c.isBitmap() {
|
||||
return c.bitmapCountRange(start, end)
|
||||
}
|
||||
return c.bitmapCountRange(start, end)
|
||||
return c.runCountRange(start, end)
|
||||
}
|
||||
|
||||
func (c *container) arrayCountRange(start, end uint32) (n int) {
|
||||
|
|
@ -991,12 +1024,50 @@ func (c *container) bitmapCountRange(start, end uint32) int {
|
|||
return int(n)
|
||||
}
|
||||
|
||||
// add adds a value to the container.
|
||||
func (c *container) add(v uint32) bool {
|
||||
if c.isArray() {
|
||||
return c.arrayAdd(v)
|
||||
func (c *container) runCountRange(start, end uint32) (n int) {
|
||||
start16, end16 := uint16(start), uint16(end)
|
||||
for _, iv := range c.runs {
|
||||
// iv is before range
|
||||
if iv.last < start16 {
|
||||
continue
|
||||
}
|
||||
// iv is after range
|
||||
if end16 < iv.start {
|
||||
break
|
||||
}
|
||||
// iv is superset of range
|
||||
if iv.start < start16 && iv.last > end16 {
|
||||
return int(end16 - start16)
|
||||
}
|
||||
// iv is subset of range
|
||||
if iv.start >= start16 && iv.last < end16 {
|
||||
n += iv.runlen()
|
||||
}
|
||||
// iv overlaps beginning of range
|
||||
if iv.start < start16 && iv.last < end16 {
|
||||
n += int(iv.last - start16 + 1)
|
||||
}
|
||||
// iv overlaps end of range
|
||||
if iv.start > start16 && iv.last >= end16 {
|
||||
n += int(end16 - iv.start)
|
||||
}
|
||||
}
|
||||
return c.bitmapAdd(v)
|
||||
return n
|
||||
}
|
||||
|
||||
// add adds a value to the container.
|
||||
func (c *container) add(v uint32) (added bool) {
|
||||
if c.isArray() {
|
||||
added = c.arrayAdd(v)
|
||||
} else if c.isBitmap() {
|
||||
added = c.bitmapAdd(v)
|
||||
} else {
|
||||
added = c.runAdd(v)
|
||||
}
|
||||
if added {
|
||||
c.n++
|
||||
}
|
||||
return added
|
||||
}
|
||||
|
||||
func (c *container) arrayAdd(v uint32) bool {
|
||||
|
|
@ -1004,7 +1075,6 @@ func (c *container) arrayAdd(v uint32) bool {
|
|||
if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v {
|
||||
c.unmap()
|
||||
c.array = append(c.array, v)
|
||||
c.n++
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -1026,7 +1096,6 @@ func (c *container) arrayAdd(v uint32) bool {
|
|||
c.array = append(c.array, 0)
|
||||
copy(c.array[i+1:], c.array[i:])
|
||||
c.array[i] = v
|
||||
c.n++
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -1036,7 +1105,50 @@ func (c *container) bitmapAdd(v uint32) bool {
|
|||
}
|
||||
c.unmap()
|
||||
c.bitmap[v/64] |= (1 << uint64(v%64))
|
||||
c.n++
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *container) runAdd(v uint32) bool {
|
||||
v16 := uint16(v)
|
||||
if len(c.runs) == 0 {
|
||||
c.unmap()
|
||||
c.runs = []interval16{{start: v16, last: v16}}
|
||||
return true
|
||||
}
|
||||
i := 0
|
||||
var iv interval16
|
||||
for i, iv = range c.runs {
|
||||
if iv.last >= v16 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if v16 >= iv.start && iv.last >= v16 {
|
||||
return false
|
||||
}
|
||||
c.unmap()
|
||||
if iv.last < v16 {
|
||||
if iv.last == v16-1 {
|
||||
c.runs[i].last += 1
|
||||
} else {
|
||||
c.runs = append(c.runs, interval16{start: v16, last: v16})
|
||||
}
|
||||
} else if v16+1 == iv.start {
|
||||
// combining two intervals
|
||||
if i > 0 && c.runs[i-1].last == v16-1 {
|
||||
c.runs[i-1].last = iv.last
|
||||
c.runs = append(c.runs[:i], c.runs[i+1:]...)
|
||||
return true
|
||||
}
|
||||
// just before an interval
|
||||
c.runs[i].start -= 1
|
||||
} else if i > 0 && v16-1 == c.runs[i-1].last {
|
||||
// just after an interval
|
||||
c.runs[i-1].last += 1
|
||||
} else {
|
||||
// alone
|
||||
newIv := interval16{start: v16, last: v16}
|
||||
c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -1056,6 +1168,20 @@ func (c *container) bitmapContains(v uint32) bool {
|
|||
return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
|
||||
}
|
||||
|
||||
func (c *container) runContains(v uint32) bool {
|
||||
v16 := uint16(v)
|
||||
for _, iv := range c.runs {
|
||||
if v16 > iv.last {
|
||||
continue
|
||||
}
|
||||
if v16 < iv.start {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// remove adds a value to the container.
|
||||
func (c *container) remove(v uint32) bool {
|
||||
if c.isArray() {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,138 @@
|
|||
|
||||
package roaring
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInterval16RunLen(t *testing.T) {
|
||||
iv := interval16{start: 7, last: 9}
|
||||
if iv.runlen() != 3 {
|
||||
t.Fatalf("should be 3")
|
||||
}
|
||||
iv = interval16{start: 7, last: 7}
|
||||
if iv.runlen() != 1 {
|
||||
t.Fatalf("should be 1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerRunAdd(t *testing.T) {
|
||||
c := container{runs: make([]interval16, 0)}
|
||||
tests := []struct {
|
||||
op uint32
|
||||
exp []interval16
|
||||
}{
|
||||
{1, []interval16{{start: 1, last: 1}}},
|
||||
{2, []interval16{{start: 1, last: 2}}},
|
||||
{4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}},
|
||||
{3, []interval16{{start: 1, last: 4}}},
|
||||
{10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}},
|
||||
{7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
|
||||
{6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
|
||||
{0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
|
||||
{8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
ret := c.add(test.op)
|
||||
if !ret {
|
||||
t.Fatalf("result of adding new bit should be true: %v", c.runs)
|
||||
}
|
||||
if !reflect.DeepEqual(c.runs, test.exp) {
|
||||
t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs, test.op)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerRunAdd2(t *testing.T) {
|
||||
c := container{runs: make([]interval16, 0)}
|
||||
ret := c.add(0)
|
||||
if !ret {
|
||||
t.Fatalf("result of adding new bit should be true: %v", c.runs)
|
||||
}
|
||||
if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) {
|
||||
t.Fatalf("should have 1 run of length 1, but have %v", c.runs)
|
||||
}
|
||||
ret = c.add(0)
|
||||
if ret {
|
||||
t.Fatalf("result of adding existing bit should be false: %v", c.runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCountRange(t *testing.T) {
|
||||
c := container{runs: make([]interval16, 0)}
|
||||
cnt := c.runCountRange(2, 9)
|
||||
if cnt != 0 {
|
||||
t.Fatalf("should get 0 from empty container, but got: %v", cnt)
|
||||
}
|
||||
c.add(5)
|
||||
c.add(6)
|
||||
c.add(7)
|
||||
|
||||
cnt = c.runCountRange(2, 9)
|
||||
if cnt != 3 {
|
||||
t.Fatalf("should get 3 from interval within range, but got: %v", cnt)
|
||||
}
|
||||
|
||||
c.add(8)
|
||||
c.add(9)
|
||||
c.add(10)
|
||||
c.add(11)
|
||||
|
||||
cnt = c.runCountRange(6, 8)
|
||||
if cnt != 2 {
|
||||
t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
|
||||
}
|
||||
|
||||
cnt = c.runCountRange(3, 9)
|
||||
if cnt != 4 {
|
||||
t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt)
|
||||
}
|
||||
|
||||
cnt = c.runCountRange(9, 14)
|
||||
if cnt != 3 {
|
||||
t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt)
|
||||
}
|
||||
|
||||
c.add(17)
|
||||
c.add(18)
|
||||
c.add(19)
|
||||
|
||||
cnt = c.runCountRange(1, 22)
|
||||
if cnt != 10 {
|
||||
t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt)
|
||||
}
|
||||
|
||||
c.add(13)
|
||||
c.add(14)
|
||||
|
||||
cnt = c.runCountRange(6, 18)
|
||||
if cnt != 9 {
|
||||
t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunContains(t *testing.T) {
|
||||
c := container{runs: make([]interval16, 0)}
|
||||
if c.runContains(5) {
|
||||
t.Fatalf("empty run container should not contain 5")
|
||||
}
|
||||
c.add(5)
|
||||
if !c.runContains(5) {
|
||||
t.Fatalf("run container with 5 should contain 5")
|
||||
}
|
||||
|
||||
c.add(6)
|
||||
c.add(7)
|
||||
|
||||
c.add(9)
|
||||
c.add(10)
|
||||
c.add(11)
|
||||
|
||||
if !c.runContains(10) {
|
||||
t.Fatalf("run container with 10 in second run should contain 10")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapCountRange(t *testing.T) {
|
||||
c := container{}
|
||||
|
|
@ -38,6 +169,12 @@ func TestBitmapCountRange(t *testing.T) {
|
|||
t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
|
||||
}
|
||||
}
|
||||
|
||||
c = container{bitmap: []uint64{0xF0000001, 0xFF000000, 0xFF00000000000000}}
|
||||
cnt := c.bitmapCountRange(62, 129)
|
||||
if cnt != 10 {
|
||||
t.Fatalf("count of %v from 62 to 129 should be 10, but got %v", c.bitmap, cnt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntersectionCountArrayBitmap2(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue