mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 09:05:55 +00:00
WIP bug fix, cleanup, additional tests
This commit is contained in:
Alan Bernstein
Matt Jaffee
parent
f4f07c0aa2
commit
bf7a1bc08a
3 changed files with 159 additions and 54 deletions
|
|
@ -17,6 +17,7 @@ package pilosa_test
|
|||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
|
|
@ -248,6 +249,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
|
|||
// Set bits on rows 0 - 999. Higher rows have higher bit counts.
|
||||
for i := uint64(0); i < 1000; i++ {
|
||||
for j := uint64(0); j < i; j++ {
|
||||
//fmt.Printf("testfragment: setting %d, %d\n", i, j)
|
||||
f.MustSetBits(i, j)
|
||||
}
|
||||
}
|
||||
|
|
@ -272,6 +274,45 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// temporary test for catching op log bug
|
||||
func TestFragment_TopN_Intersect_Moderate(t *testing.T) {
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
|
||||
defer f.Close()
|
||||
|
||||
// Create an intersecting input row.
|
||||
src := pilosa.NewBitmap(
|
||||
80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
||||
90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
|
||||
)
|
||||
|
||||
// Set bits on rows 0 - 99. Higher rows have higher bit counts.
|
||||
for i := uint64(0); i < 100; i++ {
|
||||
for j := uint64(0); j < i; j++ {
|
||||
//fmt.Printf("testfragment: setting %d, %d\n", i, j)
|
||||
f.MustSetBits(i, j)
|
||||
}
|
||||
}
|
||||
f.RecalculateCache()
|
||||
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{ID: 99, Count: 19},
|
||||
{ID: 98, Count: 18},
|
||||
{ID: 97, Count: 17},
|
||||
{ID: 96, Count: 16},
|
||||
{ID: 95, Count: 15},
|
||||
{ID: 94, Count: 14},
|
||||
{ID: 93, Count: 13},
|
||||
{ID: 92, Count: 12},
|
||||
{ID: 91, Count: 11},
|
||||
{ID: 90, Count: 10},
|
||||
}) {
|
||||
t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can return top rows when specified by ID.
|
||||
func TestFragment_TopN_IDs(t *testing.T) {
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
|
||||
|
|
@ -789,3 +830,34 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
|
|||
t.Fatalf("unexpected pair(1): %v", pairs[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFragment_Snapshot_Run(t *testing.T) {
|
||||
fmt.Printf("mustopenfragment\n")
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
|
||||
defer f.Close()
|
||||
|
||||
fmt.Printf("\n---\nset bits\n")
|
||||
// Set bits on the fragment.
|
||||
for i := uint64(1); i < 3; i++ {
|
||||
if _, err := f.SetBit(1000, i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n---\nsnapshot\n")
|
||||
// Snapshot bitmap and verify data.
|
||||
if err := f.Snapshot(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Row(1000).Count(); n != 2 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
fmt.Printf("\n---\nreopen\n")
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Row(1000).Count(); n != 2 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,9 +26,17 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// cookie is the first four bytes in a roaring bitmap file.
|
||||
cookieNoRuns = uint32(12346)
|
||||
cookie = uint32(12347)
|
||||
// magicNumber is an identifier, in bytes 0-1 of the file.
|
||||
magicNumberNoRuns = uint32(12346)
|
||||
magicNumber = uint32(12347)
|
||||
|
||||
// storageVersion indicates the storage version, in bytes 2-3.
|
||||
storageVersion = uint32(0)
|
||||
|
||||
// cookie is the first four bytes in a roaring bitmap file,
|
||||
// formed by joining magicNumber and storageVersion
|
||||
cookieNoRuns = magicNumberNoRuns<<16 + storageVersion
|
||||
cookie = magicNumber<<16 + storageVersion
|
||||
|
||||
// headerBaseSize is the size of the cookie and key count at the beginning of a file.
|
||||
// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
|
||||
|
|
@ -661,7 +669,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
|
|||
// Map byte slice directly to the container data.
|
||||
c := b.containers[i]
|
||||
if c.n <= ArrayMaxSize {
|
||||
if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i)%8)) != 0 {
|
||||
if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i%8))) != 0 {
|
||||
// Read runs.
|
||||
runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
|
||||
c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
|
||||
|
|
@ -680,7 +688,6 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
|
|||
c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
|
||||
opsOffset = int(offset) + len(c.bitmap)*8
|
||||
}
|
||||
|
||||
// Verify container count on load.
|
||||
// TODO: instead of commenting this out, we need to make it a configuration option
|
||||
//count := c.count()
|
||||
|
|
@ -809,8 +816,8 @@ type BitmapInfo struct {
|
|||
|
||||
// Iterator represents an iterator over a Bitmap.
|
||||
type Iterator struct {
|
||||
bitmap *Bitmap
|
||||
i, j int
|
||||
bitmap *Bitmap
|
||||
i, j, k int // i: container; j: array index, bit index, or run index; k:
|
||||
}
|
||||
|
||||
// eof returns true if the iterator is at the end of the bitmap.
|
||||
|
|
@ -829,7 +836,8 @@ func (itr *Iterator) Seek(seek uint64) {
|
|||
|
||||
// Move to the correct value index inside the array container.
|
||||
lb := lowbits(seek)
|
||||
if c := itr.bitmap.containers[itr.i]; c.isArray() {
|
||||
c := itr.bitmap.containers[itr.i]
|
||||
if c.isArray() {
|
||||
// Find index in the container.
|
||||
itr.j = search32(c.array, lb)
|
||||
if itr.j < 0 {
|
||||
|
|
@ -845,6 +853,15 @@ func (itr *Iterator) Seek(seek uint64) {
|
|||
return
|
||||
}
|
||||
|
||||
if c.isRun() {
|
||||
// TODO work for seek!=0
|
||||
itr.i, itr.j, itr.k = 0, 0, -1
|
||||
if seek != 0 {
|
||||
panic("cant seeek nonzero")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// If it's a bitmap container then move to index before the value and call next().
|
||||
itr.j = int(lb) - 1
|
||||
}
|
||||
|
|
@ -868,6 +885,35 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
|
|||
itr.j++
|
||||
return itr.peek(), false
|
||||
}
|
||||
|
||||
if c.isRun() {
|
||||
if itr.j >= len(c.runs)-1 {
|
||||
r := c.runs[itr.j]
|
||||
runLength := int(r.last - r.start)
|
||||
|
||||
if itr.k >= runLength {
|
||||
itr.i++
|
||||
itr.j = -1
|
||||
continue
|
||||
} else {
|
||||
itr.k++
|
||||
return itr.peek(), false
|
||||
}
|
||||
}
|
||||
|
||||
r := c.runs[itr.j]
|
||||
runLength := int(r.last - r.start)
|
||||
|
||||
if itr.k >= runLength {
|
||||
itr.k = 0
|
||||
itr.j++
|
||||
} else {
|
||||
itr.k++
|
||||
}
|
||||
|
||||
return itr.peek(), false
|
||||
}
|
||||
|
||||
// Move to the next possible index in the bitmap container.
|
||||
itr.j++
|
||||
|
||||
|
|
@ -904,6 +950,9 @@ func (itr *Iterator) peek() uint64 {
|
|||
if c.isArray() {
|
||||
return uint64(key)<<16 | uint64(c.array[itr.j])
|
||||
}
|
||||
if c.isRun() {
|
||||
return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint32(itr.k))
|
||||
}
|
||||
return uint64(key)<<16 | uint64(itr.j)
|
||||
}
|
||||
|
||||
|
|
@ -1002,10 +1051,12 @@ func (c *container) isArray() bool {
|
|||
return c.bitmap == nil && c.runs == nil
|
||||
}
|
||||
|
||||
// isBitmap returns true if the container is a bitmap container
|
||||
func (c *container) isBitmap() bool {
|
||||
return c.array == nil && c.runs == nil
|
||||
}
|
||||
|
||||
// isRun returns true if the container is a run-length-encoded container
|
||||
func (c *container) isRun() bool {
|
||||
return c.array == nil && c.bitmap == nil
|
||||
}
|
||||
|
|
@ -1582,7 +1633,7 @@ func (c *container) size() int {
|
|||
if c.isArray() {
|
||||
return len(c.array) * 4
|
||||
} else if c.isRun() {
|
||||
return len(c.runs) * 8
|
||||
return len(c.runs)*8 + 2
|
||||
} else {
|
||||
return len(c.bitmap) * 8
|
||||
}
|
||||
|
|
@ -1597,7 +1648,7 @@ func (c *container) info() ContainerInfo {
|
|||
info.Alloc = len(c.array) * 4
|
||||
} else if c.isRun() {
|
||||
info.Type = "run"
|
||||
info.Alloc = len(c.runs) * 8
|
||||
info.Alloc = len(c.runs)*8 + 2
|
||||
} else {
|
||||
info.Type = "bitmap"
|
||||
info.Alloc = len(c.bitmap) * 8
|
||||
|
|
@ -2323,26 +2374,28 @@ func differenceArrayArray(a, b *container) *container {
|
|||
func differenceArrayRun(a, b *container) *container {
|
||||
// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
|
||||
|
||||
if b.n == 0 {
|
||||
if a.n == 0 || b.n == 0 {
|
||||
return a.clone()
|
||||
}
|
||||
|
||||
output := &container{array: make([]uint32, 0, a.n)}
|
||||
// cardinality upper bound: card(A)
|
||||
|
||||
i := 0 // array index
|
||||
j := 0 // run index
|
||||
i := 0 // array index
|
||||
j := 0 // run index
|
||||
|
||||
// keep all array elements before beginning of runs
|
||||
for ; i < b.runs[j].start; i++ {
|
||||
for ; i < int(b.runs[j].start); i++ {
|
||||
output.array = append(output.array, a.array[i])
|
||||
}
|
||||
|
||||
// handle overlap
|
||||
for ; i < a.n; i++ {
|
||||
// if array element in run, keep
|
||||
if !(a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last) {
|
||||
output.array = append(output.array, a.array[i])
|
||||
}
|
||||
// update current run
|
||||
if i >= int(b.runs[j].last) {
|
||||
j++
|
||||
if j == len(b.runs) {
|
||||
|
|
@ -2354,7 +2407,7 @@ func differenceArrayRun(a, b *container) *container {
|
|||
|
||||
// keep all array elements after end of runs
|
||||
output.array = append(output.array, a.array[i:]...)
|
||||
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2372,27 +2425,33 @@ func differenceBitmapRun(a, b *container) *container {
|
|||
|
||||
func differenceRunArray(a, b *container) *container {
|
||||
// TODO
|
||||
if a.n == 0 || b.n == 0 {
|
||||
return a.clone()
|
||||
}
|
||||
|
||||
output := &container{runs: make([]interval32, 0, a.n)}
|
||||
return output
|
||||
}
|
||||
|
||||
func differenceRunBitmap(a, b *container) *container {
|
||||
// TODO
|
||||
if a.n == 0 || b.n == 0 {
|
||||
return a.clone()
|
||||
}
|
||||
|
||||
output := &container{runs: make([]interval32, 0, a.n)}
|
||||
itr := newBufIterator(newBitmapIterator(b.bitmap))
|
||||
|
||||
fmt.Printf("\ndifferenceRunBitmap\n")
|
||||
for i := 0; ; {
|
||||
vb, eof := itr.next()
|
||||
if eof || {
|
||||
if eof {
|
||||
break
|
||||
}
|
||||
|
||||
fmt.Println(i, vb, eof)
|
||||
fmt.Println(i, vb)
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2403,8 +2462,8 @@ func differenceRunRun(a, b *container) *container {
|
|||
return a.clone()
|
||||
}
|
||||
|
||||
apos := 0 // current a-run index
|
||||
bpos := 0 // current b-run index
|
||||
apos := 0 // current a-run index
|
||||
bpos := 0 // current b-run index
|
||||
astart := a.runs[apos].start
|
||||
alast := a.runs[apos].last
|
||||
bstart := b.runs[bpos].start
|
||||
|
|
@ -2412,7 +2471,7 @@ func differenceRunRun(a, b *container) *container {
|
|||
alen := len(a.runs)
|
||||
blen := len(b.runs)
|
||||
|
||||
output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else
|
||||
output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else
|
||||
// cardinality upper bound: sum of number of runs
|
||||
// each B-run could split an A-run in two, up to len(b.runs) times
|
||||
|
||||
|
|
|
|||
|
|
@ -996,36 +996,6 @@ func TestUnionBitmapRun(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRunToArray(t *testing.T) {
|
||||
a := &container{}
|
||||
tests := []struct {
|
||||
runs []interval32
|
||||
exp []uint32
|
||||
}{
|
||||
{
|
||||
runs: []interval32{{start: 0, last: 0}},
|
||||
exp: []uint32{0},
|
||||
},
|
||||
{
|
||||
runs: []interval32{{start: 0, last: 4}},
|
||||
exp: []uint32{0, 1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
|
||||
exp: []uint32{2, 5, 6, 7, 13, 14, 17},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
a.runs = test.runs
|
||||
a.n = len(test.exp)
|
||||
a.runToArray()
|
||||
if !reflect.DeepEqual(a.array, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapCountRuns(t *testing.T) {
|
||||
c := &container{bitmap: make([]uint64, bitmapN)}
|
||||
tests := []struct {
|
||||
|
|
@ -1133,6 +1103,7 @@ func TestDifferenceArrayRun(t *testing.T) {
|
|||
}
|
||||
for i, test := range tests {
|
||||
a.array = test.array
|
||||
a.n = len(a.array)
|
||||
b.runs = test.runs
|
||||
b.n = b.runCountRange(0, 100)
|
||||
ret := differenceArrayRun(a, b)
|
||||
|
|
@ -1160,6 +1131,7 @@ func TestDifferenceRunArray(t *testing.T) {
|
|||
a.runs = test.runs
|
||||
a.n = a.runCountRange(0, 100)
|
||||
b.array = test.array
|
||||
b.n = len(b.array)
|
||||
ret := differenceRunArray(a, b)
|
||||
if !reflect.DeepEqual(ret.array, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
|
||||
|
|
@ -1187,6 +1159,7 @@ func TestDifferenceRunBitmap(t *testing.T) {
|
|||
for i, v := range test.bitmap {
|
||||
b.bitmap[i] = v
|
||||
}
|
||||
b.n = b.bitmapCountRange(0, 100)
|
||||
ret := differenceRunBitmap(a, b)
|
||||
if !reflect.DeepEqual(ret.runs, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
|
||||
|
|
@ -1212,11 +1185,12 @@ func TestDifferenceBitmapRun(t *testing.T) {
|
|||
for i, v := range test.bitmap {
|
||||
a.bitmap[i] = v
|
||||
}
|
||||
a.n = a.bitmapCountRange(0, 100)
|
||||
b.runs = test.runs
|
||||
b.n = b.runCountRange(0, 100)
|
||||
ret := differenceBitmapRun(a, b)
|
||||
if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap[:len(test.exp)])
|
||||
t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue