mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-08 11:57:51 +00:00
xorArrayRun;xorRunRun of rle
This commit is contained in:
Todd Gruben
Matt Jaffee
parent
f6d3efe29d
commit
bd90254282
2 changed files with 300 additions and 21 deletions
|
|
@ -28,7 +28,7 @@ import (
|
|||
const (
|
||||
// cookie is the first four bytes in a roaring bitmap file.
|
||||
cookieNoRuns = uint32(12346)
|
||||
cookie = uint32(12347)
|
||||
cookie = uint32(12347)
|
||||
|
||||
// headerBaseSize is the size of the cookie and key count at the beginning of a file.
|
||||
// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
|
||||
|
|
@ -521,7 +521,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
|
|||
|
||||
// Create bitset indicating runs, record whether any runs present.
|
||||
containsRuns := false
|
||||
runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size
|
||||
runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size
|
||||
var k uint8 = 0
|
||||
for _, c := range b.containers {
|
||||
if c.n == 0 {
|
||||
|
|
@ -529,7 +529,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
|
|||
}
|
||||
if c.isRun() {
|
||||
containsRuns = true
|
||||
runFlagBitset[k/8] |= (1 << (k%8)) // TODO verify
|
||||
runFlagBitset[k/8] |= (1 << (k % 8)) // TODO verify
|
||||
}
|
||||
k++
|
||||
}
|
||||
|
|
@ -630,10 +630,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
|
|||
|
||||
headerSize := headerBaseSize
|
||||
|
||||
runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size
|
||||
runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size
|
||||
if containsRuns {
|
||||
// Read runFlag bitset.
|
||||
for i := 0; i<len(runFlagBitset); i++ {
|
||||
for i := 0; i < len(runFlagBitset); i++ {
|
||||
runFlagBitset[i] = data[8+i]
|
||||
}
|
||||
headerSize += len(runFlagBitset)
|
||||
|
|
@ -661,11 +661,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
|
|||
// Map byte slice directly to the container data.
|
||||
c := b.containers[i]
|
||||
if c.n <= ArrayMaxSize {
|
||||
if containsRuns && (runFlagBitset[uint8(i)/8] & (1 << uint8(i)%8)) != 0 {
|
||||
if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i)%8)) != 0 {
|
||||
// Read runs.
|
||||
runCount := binary.LittleEndian.Uint16(data[offset:offset+2])
|
||||
runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
|
||||
c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
|
||||
opsOffset = int(offset) + 2 + len(c.runs)*8 // TODO verify
|
||||
opsOffset = int(offset) + 2 + len(c.runs)*8 // TODO verify
|
||||
} else {
|
||||
// Read array.
|
||||
c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
|
||||
|
|
@ -1445,7 +1445,7 @@ func (c *container) bitmapToRun() {
|
|||
current := c.bitmap[0]
|
||||
var i, start, last uint32
|
||||
for {
|
||||
// skip while empty
|
||||
// skip while empty
|
||||
for current == 0 && i < bitmapN-1 {
|
||||
i++
|
||||
current = c.bitmap[i]
|
||||
|
|
@ -1455,10 +1455,10 @@ func (c *container) bitmapToRun() {
|
|||
break
|
||||
}
|
||||
currentStart := uint32(trailingZeroN(current))
|
||||
start = 64 * i + currentStart
|
||||
start = 64*i + currentStart
|
||||
|
||||
// pad LSBs with 1s
|
||||
current = current | (current-1)
|
||||
current = current | (current - 1)
|
||||
|
||||
// find next 0
|
||||
for current == maxBitmap && i < bitmapN-1 {
|
||||
|
|
@ -1468,16 +1468,16 @@ func (c *container) bitmapToRun() {
|
|||
|
||||
last = 0
|
||||
if current == maxBitmap {
|
||||
last = 64*i + 64 // TODO verify
|
||||
c.runs = append(c.runs, interval32{start, last-1})
|
||||
last = 64*i + 64 // TODO verify
|
||||
c.runs = append(c.runs, interval32{start, last - 1})
|
||||
break
|
||||
}
|
||||
currentLast := uint32(trailingZeroN(^current))
|
||||
last = 64 * i + currentLast
|
||||
c.runs = append(c.runs, interval32{start, last-1})
|
||||
last = 64*i + currentLast
|
||||
c.runs = append(c.runs, interval32{start, last - 1})
|
||||
|
||||
// pad LSBs with 0s
|
||||
current = current & (current+1)
|
||||
current = current & (current + 1)
|
||||
}
|
||||
|
||||
c.bitmap = nil
|
||||
|
|
@ -1489,7 +1489,7 @@ func (c *container) arrayToRun() {
|
|||
c.runs = make([]interval32, 0, c.n) // what capacity to use?
|
||||
start := c.array[0]
|
||||
for i, v := range c.array[1:] {
|
||||
if v - c.array[i] > 1 {
|
||||
if v-c.array[i] > 1 {
|
||||
// if current-previous > 1, one run ends and another begins
|
||||
c.runs = append(c.runs, interval32{start, c.array[i]})
|
||||
start = v
|
||||
|
|
@ -1574,7 +1574,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
|
|||
return 0, err
|
||||
}
|
||||
nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)])
|
||||
return int64(2+nn), err
|
||||
return int64(2 + nn), err
|
||||
}
|
||||
|
||||
// size returns the encoded size of the container, in bytes.
|
||||
|
|
@ -2910,3 +2910,182 @@ func assert(condition bool, format string, a ...interface{}) {
|
|||
panic(fmt.Sprintf(format, a...))
|
||||
}
|
||||
}
|
||||
|
||||
func xorArrayRun(a, b *container) *container {
|
||||
output := &container{}
|
||||
na, nb := len(a.array), len(b.runs)
|
||||
var vb interval32
|
||||
var va uint32
|
||||
last_i, last_j := -1, -1
|
||||
for i, j := 0, 0; i < na || j < nb; {
|
||||
if i < na && i != last_i {
|
||||
va = a.array[i]
|
||||
}
|
||||
if j < nb && j != last_j {
|
||||
vb = b.runs[j]
|
||||
}
|
||||
last_i = i
|
||||
last_j = j
|
||||
|
||||
if i < na && (j >= nb || va < vb.start) { //before
|
||||
output.n += output.runAppendInterval(interval32{start: va, last: va})
|
||||
i++
|
||||
} else if j < nb && (i >= na || va > vb.last) { //after
|
||||
output.n += output.runAppendInterval(vb)
|
||||
j++
|
||||
} else if va > vb.start {
|
||||
if va < vb.last {
|
||||
output.n += output.runAppendInterval(interval32{start: vb.start, last: va - 1})
|
||||
vb.start = va + 1
|
||||
i++
|
||||
if vb.start > vb.last {
|
||||
j++
|
||||
}
|
||||
} else if va > vb.last {
|
||||
output.n += output.runAppendInterval(vb)
|
||||
j++
|
||||
} else { // va == vb.last
|
||||
vb.last--
|
||||
if vb.start < vb.last {
|
||||
output.n += output.runAppendInterval(vb)
|
||||
}
|
||||
j++
|
||||
i++
|
||||
}
|
||||
|
||||
} else {
|
||||
vb.start++
|
||||
i++
|
||||
}
|
||||
}
|
||||
if output.n < ArrayMaxSize {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > RunMaxSize {
|
||||
output.runToBitmap()
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func xorCompare(x *stm) (r1 interval32, has_data bool) {
|
||||
has_data = false
|
||||
if !x.va_valid || !x.vb_valid {
|
||||
if x.vb_valid {
|
||||
x.vb_valid = false
|
||||
r1 = x.vb
|
||||
has_data = true
|
||||
return
|
||||
}
|
||||
if x.va_valid {
|
||||
x.va_valid = false
|
||||
r1 = x.va
|
||||
has_data = true
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if x.va.last < x.vb.start { //a before
|
||||
x.va_valid = false
|
||||
r1 = x.va
|
||||
has_data = true
|
||||
} else if x.vb.last < x.va.start { // b before
|
||||
x.vb_valid = false
|
||||
r1 = x.va
|
||||
has_data = true
|
||||
} else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal
|
||||
x.va_valid = false
|
||||
x.vb_valid = false
|
||||
} else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { // b inside
|
||||
x.vb_valid = false
|
||||
if x.va.start != x.vb.start {
|
||||
r1 = interval32{start: x.va.start, last: x.vb.start - 1}
|
||||
has_data = true
|
||||
}
|
||||
x.va.start = x.vb.last + 1
|
||||
if x.va.start > x.va.last {
|
||||
x.va_valid = false
|
||||
}
|
||||
|
||||
} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //a inside
|
||||
x.va_valid = false
|
||||
if x.vb.start != x.va.start {
|
||||
r1 = interval32{start: x.vb.start, last: x.va.start - 1}
|
||||
has_data = true
|
||||
}
|
||||
|
||||
x.vb.start = x.va.last + 1
|
||||
if x.vb.start > x.vb.last {
|
||||
x.vb_valid = false
|
||||
}
|
||||
|
||||
} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //a first overlap
|
||||
x.va_valid = false
|
||||
r1 = interval32{start: x.va.start, last: x.vb.start - 1}
|
||||
has_data = true
|
||||
x.vb.start = x.va.last + 1
|
||||
if x.vb.start > x.vb.last {
|
||||
x.vb_valid = false
|
||||
}
|
||||
} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //b first overlap
|
||||
x.vb_valid = false
|
||||
r1 = interval32{start: x.vb.start, last: x.va.start - 1}
|
||||
has_data = true
|
||||
x.va.start = x.vb.last + 1
|
||||
if x.va.start > x.va.last {
|
||||
x.va_valid = false
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type stm struct {
|
||||
va_valid, vb_valid bool
|
||||
va, vb interval32
|
||||
}
|
||||
|
||||
func xorRunRun(a, b *container) *container {
|
||||
na, nb := len(a.runs), len(b.runs)
|
||||
if na == 0 {
|
||||
return b.clone()
|
||||
}
|
||||
if nb == 0 {
|
||||
return a.clone()
|
||||
}
|
||||
output := &container{}
|
||||
|
||||
last_i, last_j := -1, -1
|
||||
|
||||
state := &stm{}
|
||||
|
||||
for i, j := 0, 0; i < na || j < nb; { //how to terminate?
|
||||
if i < na && last_i != i {
|
||||
state.va = a.runs[i]
|
||||
state.va_valid = true
|
||||
}
|
||||
|
||||
if j < nb && last_j != j {
|
||||
state.vb = b.runs[j]
|
||||
state.vb_valid = true
|
||||
}
|
||||
last_i, last_j = i, j
|
||||
|
||||
r1, ok := xorCompare(state)
|
||||
if ok {
|
||||
output.n += output.runAppendInterval(r1)
|
||||
}
|
||||
if !state.va_valid {
|
||||
i++
|
||||
}
|
||||
if !state.vb_valid {
|
||||
j++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > RunMaxSize {
|
||||
output.runToBitmap()
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
|
|
|||
|
|
@ -884,6 +884,36 @@ func TestArrayToRun(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRunToArray(t *testing.T) {
|
||||
a := &container{}
|
||||
tests := []struct {
|
||||
runs []interval32
|
||||
exp []uint32
|
||||
}{
|
||||
{
|
||||
runs: []interval32{{start: 0, last: 0}},
|
||||
exp: []uint32{0},
|
||||
},
|
||||
{
|
||||
runs: []interval32{{start: 0, last: 4}},
|
||||
exp: []uint32{0, 1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
|
||||
exp: []uint32{2, 5, 6, 7, 13, 14, 17},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
a.runs = test.runs
|
||||
a.n = len(test.exp)
|
||||
a.runToArray()
|
||||
if !reflect.DeepEqual(a.array, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapZeroRange(t *testing.T) {
|
||||
c := &container{bitmap: make([]uint64, bitmapN)}
|
||||
tests := []struct {
|
||||
|
|
@ -1234,8 +1264,8 @@ func TestWriteReadArray(t *testing.T) {
|
|||
|
||||
func TestWriteReadBitmap(t *testing.T) {
|
||||
// create bitmap containing > 4096 bits
|
||||
cb := &container{bitmap: make([]uint64, bitmapN), n: 129*32}
|
||||
for i := 0; i < 129 ; i++ {
|
||||
cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32}
|
||||
for i := 0; i < 129; i++ {
|
||||
cb.bitmap[i] = 0x5555555555555555
|
||||
}
|
||||
bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
|
||||
|
|
@ -1249,7 +1279,7 @@ func TestWriteReadBitmap(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestWriteReadRun(t *testing.T) {
|
||||
cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last:109}}, n: 20}
|
||||
cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20}
|
||||
br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
|
||||
br2 := &Bitmap{}
|
||||
var buf bytes.Buffer
|
||||
|
|
@ -1259,3 +1289,73 @@ func TestWriteReadRun(t *testing.T) {
|
|||
t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXorArrayRun(t *testing.T) {
|
||||
a := &container{array: []uint32{1, 5, 10, 11, 12}}
|
||||
b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
|
||||
exp := []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
|
||||
|
||||
ret := xorArrayRun(a, b)
|
||||
if !reflect.DeepEqual(ret.array, exp) {
|
||||
t.Fatalf("test expected %v, but got %v", exp, ret.array)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXorRunRun(t *testing.T) {
|
||||
a := &container{}
|
||||
b := &container{}
|
||||
tests := []struct {
|
||||
aruns []interval32
|
||||
bruns []interval32
|
||||
exp []interval32
|
||||
}{
|
||||
{
|
||||
aruns: []interval32{},
|
||||
bruns: []interval32{{start: 5, last: 10}},
|
||||
exp: []interval32{{start: 5, last: 10}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 5, last: 12}},
|
||||
bruns: []interval32{{start: 5, last: 10}},
|
||||
exp: []interval32{{start: 11, last: 12}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
|
||||
bruns: []interval32{{start: 5, last: 10}},
|
||||
exp: []interval32{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
|
||||
bruns: []interval32{{start: 2, last: 65535}},
|
||||
exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 2, last: 65535}},
|
||||
bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
|
||||
exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
|
||||
bruns: []interval32{{start: 0, last: 65535}},
|
||||
exp: []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 0, last: 65535}},
|
||||
bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
|
||||
exp: []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
|
||||
},
|
||||
{
|
||||
aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
|
||||
bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
|
||||
exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
a.runs = test.aruns
|
||||
b.runs = test.bruns
|
||||
ret := xorRunRun(a, b)
|
||||
if !reflect.DeepEqual(ret.runs, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue