From bd90254282a3d875e2122e6c4a8667262d916435 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 9 Jun 2017 17:47:22 -0500 Subject: [PATCH] xorArrayRun;xorRunRun of rle --- roaring/roaring.go | 215 ++++++++++++++++++++++++++++--- roaring/roaring_internal_test.go | 106 ++++++++++++++- 2 files changed, 300 insertions(+), 21 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 92f9e7c2e..2da7511b1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -28,7 +28,7 @@ import ( const ( // cookie is the first four bytes in a roaring bitmap file. cookieNoRuns = uint32(12346) - cookie = uint32(12347) + cookie = uint32(12347) // headerBaseSize is the size of the cookie and key count at the beginning of a file. // Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8. @@ -521,7 +521,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Create bitset indicating runs, record whether any runs present. containsRuns := false - runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size + runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size var k uint8 = 0 for _, c := range b.containers { if c.n == 0 { @@ -529,7 +529,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } if c.isRun() { containsRuns = true - runFlagBitset[k/8] |= (1 << (k%8)) // TODO verify + runFlagBitset[k/8] |= (1 << (k % 8)) // TODO verify } k++ } @@ -630,10 +630,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { headerSize := headerBaseSize - runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size + runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size if containsRuns { // Read runFlag bitset. - for i := 0; i 1 { + if v-c.array[i] > 1 { // if current-previous > 1, one run ends and another begins c.runs = append(c.runs, interval32{start, c.array[i]}) start = v @@ -1574,7 +1574,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) { return 0, err } nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)]) - return int64(2+nn), err + return int64(2 + nn), err } // size returns the encoded size of the container, in bytes. @@ -2910,3 +2910,182 @@ func assert(condition bool, format string, a ...interface{}) { panic(fmt.Sprintf(format, a...)) } } + +func xorArrayRun(a, b *container) *container { + output := &container{} + na, nb := len(a.array), len(b.runs) + var vb interval32 + var va uint32 + last_i, last_j := -1, -1 + for i, j := 0, 0; i < na || j < nb; { + if i < na && i != last_i { + va = a.array[i] + } + if j < nb && j != last_j { + vb = b.runs[j] + } + last_i = i + last_j = j + + if i < na && (j >= nb || va < vb.start) { //before + output.n += output.runAppendInterval(interval32{start: va, last: va}) + i++ + } else if j < nb && (i >= na || va > vb.last) { //after + output.n += output.runAppendInterval(vb) + j++ + } else if va > vb.start { + if va < vb.last { + output.n += output.runAppendInterval(interval32{start: vb.start, last: va - 1}) + vb.start = va + 1 + i++ + if vb.start > vb.last { + j++ + } + } else if va > vb.last { + output.n += output.runAppendInterval(vb) + j++ + } else { // va == vb.last + vb.last-- + if vb.start < vb.last { + output.n += output.runAppendInterval(vb) + } + j++ + i++ + } + + } else { + vb.start++ + i++ + } + } + if output.n < ArrayMaxSize { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} + +func xorCompare(x *stm) (r1 interval32, has_data bool) { + has_data = false + if !x.va_valid || !x.vb_valid { + if x.vb_valid { + x.vb_valid = false + r1 = x.vb + has_data = true + return + } + if x.va_valid { + x.va_valid = false + r1 = x.va + has_data = true + return + } + return + } + + if x.va.last < x.vb.start { //a before + x.va_valid = false + r1 = x.va + has_data = true + } else if x.vb.last < x.va.start { // b before + x.vb_valid = false + r1 = x.va + has_data = true + } else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal + x.va_valid = false + x.vb_valid = false + } else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { // b inside + x.vb_valid = false + if x.va.start != x.vb.start { + r1 = interval32{start: x.va.start, last: x.vb.start - 1} + has_data = true + } + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } + + } else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //a inside + x.va_valid = false + if x.vb.start != x.va.start { + r1 = interval32{start: x.vb.start, last: x.va.start - 1} + has_data = true + } + + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } + + } else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //a first overlap + x.va_valid = false + r1 = interval32{start: x.va.start, last: x.vb.start - 1} + has_data = true + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } + } else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //b first overlap + x.vb_valid = false + r1 = interval32{start: x.vb.start, last: x.va.start - 1} + has_data = true + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } + } + return +} + +type stm struct { + va_valid, vb_valid bool + va, vb interval32 +} + +func xorRunRun(a, b *container) *container { + na, nb := len(a.runs), len(b.runs) + if na == 0 { + return b.clone() + } + if nb == 0 { + return a.clone() + } + output := &container{} + + last_i, last_j := -1, -1 + + state := &stm{} + + for i, j := 0, 0; i < na || j < nb; { //how to terminate? + if i < na && last_i != i { + state.va = a.runs[i] + state.va_valid = true + } + + if j < nb && last_j != j { + state.vb = b.runs[j] + state.vb_valid = true + } + last_i, last_j = i, j + + r1, ok := xorCompare(state) + if ok { + output.n += output.runAppendInterval(r1) + } + if !state.va_valid { + i++ + } + if !state.vb_valid { + j++ + } + + } + + if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + output.runToArray() + } else if len(output.runs) > RunMaxSize { + output.runToBitmap() + } + return output +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index be2b07590..c9607f5a8 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -884,6 +884,36 @@ func TestArrayToRun(t *testing.T) { } } +func TestRunToArray(t *testing.T) { + a := &container{} + tests := []struct { + runs []interval32 + exp []uint32 + }{ + { + runs: []interval32{{start: 0, last: 0}}, + exp: []uint32{0}, + }, + { + runs: []interval32{{start: 0, last: 4}}, + exp: []uint32{0, 1, 2, 3, 4}, + }, + { + runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, + exp: []uint32{2, 5, 6, 7, 13, 14, 17}, + }, + } + + for i, test := range tests { + a.runs = test.runs + a.n = len(test.exp) + a.runToArray() + if !reflect.DeepEqual(a.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) + } + } +} + func TestBitmapZeroRange(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN)} tests := []struct { @@ -1234,8 +1264,8 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &container{bitmap: make([]uint64, bitmapN), n: 129*32} - for i := 0; i < 129 ; i++ { + cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32} + for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}} @@ -1249,7 +1279,7 @@ func TestWriteReadBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last:109}}, n: 20} + cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20} br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}} br2 := &Bitmap{} var buf bytes.Buffer @@ -1259,3 +1289,73 @@ func TestWriteReadRun(t *testing.T) { t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs) } } + +func TestXorArrayRun(t *testing.T) { + a := &container{array: []uint32{1, 5, 10, 11, 12}} + b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + exp := []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16} + + ret := xorArrayRun(a, b) + if !reflect.DeepEqual(ret.array, exp) { + t.Fatalf("test expected %v, but got %v", exp, ret.array) + } +} + +func TestXorRunRun(t *testing.T) { + a := &container{} + b := &container{} + tests := []struct { + aruns []interval32 + bruns []interval32 + exp []interval32 + }{ + { + aruns: []interval32{}, + bruns: []interval32{{start: 5, last: 10}}, + exp: []interval32{{start: 5, last: 10}}, + }, + { + aruns: []interval32{{start: 5, last: 12}}, + bruns: []interval32{{start: 5, last: 10}}, + exp: []interval32{{start: 11, last: 12}}, + }, + { + aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval32{{start: 5, last: 10}}, + exp: []interval32{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}}, + }, + { + aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval32{{start: 2, last: 65535}}, + exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval32{{start: 2, last: 65535}}, + bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + bruns: []interval32{{start: 0, last: 65535}}, + exp: []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval32{{start: 0, last: 65535}}, + bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}}, + exp: []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}}, + }, + { + aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}}, + bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, + exp: []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}}, + }, + } + for i, test := range tests { + a.runs = test.aruns + b.runs = test.bruns + ret := xorRunRun(a, b) + if !reflect.DeepEqual(ret.runs, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) + } + } +}