mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 00:55:55 +00:00
rbf: keep ElemN and BitN up to date.
- fix a bug in computing leafCell.BitN in a run after a bit Remove - shrink bitmaps on remove - util_test.go has Cursor.DebugSlowCheckAllPages to verify; used by cursor_test.go
This commit is contained in:
Jason E. Aten
parent
dc0ac0ccd0
commit
d46b603cd6
11 changed files with 402 additions and 328 deletions
|
|
@ -111,7 +111,7 @@ func (cmd *RBFPageCommand) printLeafPage(page *rbf.LeafPage) {
|
|||
fmt.Fprintf(cmd.Stdout, "Type: leaf\n")
|
||||
fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
|
||||
for i, cell := range page.Cells {
|
||||
if cell.Type == "bitmap-ptr" {
|
||||
if cell.Type == rbf.ContainerTypeBitmapPtr {
|
||||
fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s pgno=%d\n", i, cell.Key, cell.Type, cell.Pgno)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s values=%v\n", i, cell.Key, cell.Type, cell.Values)
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -58,4 +58,4 @@ require (
|
|||
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
|
||||
)
|
||||
|
||||
go 1.13
|
||||
go 1.14
|
||||
|
|
|
|||
164
rbf/cursor.go
164
rbf/cursor.go
|
|
@ -17,7 +17,6 @@ import (
|
|||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
"sort"
|
||||
"unsafe"
|
||||
|
||||
|
|
@ -39,6 +38,7 @@ type Cursor struct {
|
|||
rle [RLEMaxSize + 1]roaring.Interval16
|
||||
leafCells [PageSize / 8]leafCell
|
||||
|
||||
// stack holds branches
|
||||
stack struct {
|
||||
index int
|
||||
elems [32]stackElem
|
||||
|
|
@ -85,8 +85,9 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
|
|||
return runs, true
|
||||
}
|
||||
|
||||
// checkRun is only called by Cursor.Add()
|
||||
func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
|
||||
// maybeConvertOversizedRunToBitmap is only called by Cursor.Add().
|
||||
// bitN must be the correct new bit count.
|
||||
func maybeConvertOversizedRunToBitmap(runs []roaring.Interval16, bitN int, key uint64) leafCell {
|
||||
if len(runs) >= RLEMaxSize {
|
||||
//convertToBitmap
|
||||
bitmap := make([]uint64, BitmapN)
|
||||
|
|
@ -120,12 +121,8 @@ func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
|
|||
words[i] = ^uint64(0)
|
||||
}
|
||||
}
|
||||
// TODO: take this out once we know bitN matches
|
||||
n := uint64(0)
|
||||
for _, v := range bitmap {
|
||||
n += popcount(v)
|
||||
}
|
||||
|
||||
// note that ElemN should be left 0 for ContainerTypeBitmap.
|
||||
return leafCell{Key: key, BitN: int(bitN), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
|
||||
}
|
||||
return leafCell{Key: key, ElemN: len(runs), BitN: int(bitN), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
|
||||
|
|
@ -162,11 +159,10 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
|
|||
|
||||
case ContainerTypeRLE:
|
||||
runs := toInterval16(cell.Data)
|
||||
//TODO Look at this again with fresh eyes
|
||||
copy(c.rle[:], runs)
|
||||
run, added := runAdd(c.rle[:len(runs)], lo)
|
||||
if added {
|
||||
leaf := checkRun(run, cell.BitN+1, cell.Key)
|
||||
leaf := maybeConvertOversizedRunToBitmap(run, cell.BitN+1, cell.Key)
|
||||
return true, c.putLeafCell(leaf)
|
||||
}
|
||||
return false, nil
|
||||
|
|
@ -188,9 +184,9 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
|
|||
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// TODO(bbj): Update parent cell with new BitN.
|
||||
|
||||
return true, nil
|
||||
// Update with new BitN.
|
||||
cell.BitN++
|
||||
return true, c.putLeafCell(cell)
|
||||
default:
|
||||
return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
|
||||
}
|
||||
|
|
@ -267,19 +263,41 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
|
|||
}
|
||||
a := cloneArray64(bm)
|
||||
if a[lo/64]&(1<<uint64(lo%64)) == 0 {
|
||||
// not present.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// TODO(3736): Handle shrinking bitmap container to an array container.
|
||||
|
||||
// Clear bit and rewrite page.
|
||||
// clear the bit
|
||||
a[lo/64] &^= 1 << uint64(lo%64)
|
||||
cell.BitN--
|
||||
|
||||
if cell.BitN == 0 {
|
||||
if err := c.tx.freePgno(pgno); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, c.deleteLeafCell(cell.Key)
|
||||
}
|
||||
|
||||
// shrink if we've gotten small.
|
||||
if cell.BitN <= ArrayMaxSize {
|
||||
cbm := roaring.NewContainerBitmap(cell.BitN, a)
|
||||
// convert to array
|
||||
cbm = roaring.Optimize(cbm)
|
||||
|
||||
leafCell1 := ConvertToLeafArgs(cell.Key, cbm)
|
||||
// ConvertToLeafArgs returns leafCell1 with BitN and ElemN updated.
|
||||
if err := c.tx.freePgno(pgno); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, c.putLeafCell(leafCell1)
|
||||
}
|
||||
|
||||
// rewrite page, still as a bitmap.
|
||||
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, c.putLeafCell(cell)
|
||||
|
||||
// TODO(bbj): Update parent cell to decrement BitN.
|
||||
return true, nil
|
||||
default:
|
||||
return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
|
||||
}
|
||||
|
|
@ -346,6 +364,8 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
}
|
||||
cell.Data = fromPgno(bitmapPgno)
|
||||
cell.Type = ContainerTypeBitmapPtr
|
||||
cell.BitN = in.BitN
|
||||
cell.ElemN = in.ElemN
|
||||
}
|
||||
// Shift cells over if this is an insertion.
|
||||
cells = append(cells, leafCell{})
|
||||
|
|
@ -361,6 +381,9 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
}
|
||||
cell.Type = ContainerTypeBitmapPtr
|
||||
cell.Data = fromPgno(bitmapPgno)
|
||||
// update the BitN too
|
||||
cell.BitN = in.BitN
|
||||
cell.ElemN = in.ElemN
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -963,107 +986,6 @@ func (c *Cursor) Prev() error {
|
|||
}
|
||||
}
|
||||
|
||||
// Union performs a bitwise OR operation on row and a given row id in the bitmap.
|
||||
func (c *Cursor) Union(rowID uint64, row []uint64) error {
|
||||
base := rowID * ShardWidth
|
||||
|
||||
if _, err := c.Seek(base >> 16); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
err := c.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cell := c.cell()
|
||||
key := cell.Key << 16
|
||||
if key >= base+ShardWidth {
|
||||
return nil
|
||||
}
|
||||
offset := key - base
|
||||
switch cell.Type {
|
||||
case ContainerTypeArray:
|
||||
for _, v := range toArray16(cell.Data) {
|
||||
row[(offset+uint64(v))/64] |= 1 << uint64(v%64)
|
||||
}
|
||||
case ContainerTypeRLE:
|
||||
panic("TODO(BBJ): rbf.Bitmap.Union() RLE support")
|
||||
case ContainerTypeBitmapPtr:
|
||||
_, bm, err := c.tx.leafCellBitmap(toPgno(cell.Data))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "union")
|
||||
}
|
||||
for i, v := range bm {
|
||||
row[(offset/64)+uint64(i)] |= v
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("rbf.Bitmap.Union(): invalid container type: %d", cell.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Intersect performs a bitwise AND operation on row and a given row id in the bitmap.
|
||||
func (c *Cursor) Intersect(rowID uint64, row []uint64) error {
|
||||
base := rowID * ShardWidth
|
||||
c.stack.index = 0
|
||||
|
||||
keyExists := make([]bool, ShardWidth/(1<<16))
|
||||
|
||||
if _, err := c.Seek(base >> 16); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
err := c.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cell := c.cell()
|
||||
key := cell.Key << 16
|
||||
if key >= base+ShardWidth {
|
||||
return nil
|
||||
}
|
||||
offset := key - base
|
||||
|
||||
keyExists[offset/(1<<16)] = true
|
||||
|
||||
switch cell.Type {
|
||||
case ContainerTypeArray:
|
||||
for i, v := range cell.Bitmap(c.tx) {
|
||||
row[(offset/64)+uint64(i)] &= v
|
||||
}
|
||||
case ContainerTypeRLE:
|
||||
panic("TODO(BBJ): rbf.Bitmap.Intersect() RLE support")
|
||||
case ContainerTypeBitmapPtr:
|
||||
_, bm, err := c.tx.leafCellBitmap(toPgno(cell.Data))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cursor.Intersect")
|
||||
}
|
||||
for i, v := range bm {
|
||||
row[(offset/64)+uint64(i)] &= v
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("rbf.Bitmap.Intersect(): invalid container type: %d", cell.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// Clear any missing keys.
|
||||
for i, ok := range keyExists {
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
for j := 0; j < (1 << 16); j += 64 {
|
||||
row[((i*(1<<16))+j)/64] = 0
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Values returns the values for the container the cursor is currently pointing to.
|
||||
func (c *Cursor) Values() []uint16 {
|
||||
elem := &c.stack.elems[c.stack.index]
|
||||
|
|
@ -1226,10 +1148,6 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) {
|
|||
return changed, nil
|
||||
}
|
||||
|
||||
func popcount(x uint64) uint64 {
|
||||
return uint64(bits.OnesCount64(x))
|
||||
}
|
||||
|
||||
func (c *Cursor) RemoveRoaring(bm *roaring.Bitmap) (changed bool, err error) {
|
||||
itr, _ := bm.Containers.Iterator(0)
|
||||
for itr.Next() {
|
||||
|
|
|
|||
|
|
@ -267,162 +267,6 @@ func TestCursor_LastPrev_Quick(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestCursor_Union(t *testing.T) {
|
||||
t.Run("OK", func(t *testing.T) {
|
||||
db := MustOpenDB(t)
|
||||
defer MustCloseDB(t, db)
|
||||
tx := MustBegin(t, db, true)
|
||||
defer tx.Rollback()
|
||||
|
||||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
row := make([]uint64, rbf.ShardWidth/64)
|
||||
|
||||
if _, err := tx.Add("x", 1, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := tx.Add("x", rbf.ShardWidth+1, rbf.ShardWidth+2, rbf.ShardWidth+7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c, err := tx.Cursor("x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Union(0, row); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if row[0] != 0b00001010 {
|
||||
t.Fatalf("unexpected row[0]: 0b%b", row[0])
|
||||
}
|
||||
|
||||
if err := c.Union(1, row); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if row[0] != 0b10001110 {
|
||||
t.Fatalf("unexpected row[0]: 0b%b", row[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Quick", func(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("-short enabled, skipping")
|
||||
} else if is32Bit() {
|
||||
t.Skip("32-bit build, skipping quick check tests")
|
||||
}
|
||||
|
||||
QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
|
||||
t.Parallel()
|
||||
|
||||
db := MustOpenDB(t)
|
||||
defer MustCloseDB(t, db)
|
||||
tx := MustBegin(t, db, true)
|
||||
defer tx.Rollback()
|
||||
values := GenerateValues(rand, 10000)
|
||||
rows := ToRows(values)
|
||||
|
||||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
MustAddRandom(t, rand, tx, "x", values...)
|
||||
|
||||
// Iterate over rows and randomly choose another row to union.
|
||||
for i, row0 := range rows {
|
||||
row1 := rows[rand.Intn(len(rows))]
|
||||
|
||||
bitmap := row0.Bitmap()
|
||||
c, err := tx.Cursor("x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := c.Union(row1.ID, bitmap); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if got, want := len(rbf.RowValues(bitmap)), len(row0.Union(row1)); got != want {
|
||||
t.Fatalf("%d. len()=%d, want %d", i, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCursor_Intersect(t *testing.T) {
|
||||
t.Run("OK", func(t *testing.T) {
|
||||
db := MustOpenDB(t)
|
||||
defer MustCloseDB(t, db)
|
||||
tx := MustBegin(t, db, true)
|
||||
defer tx.Rollback()
|
||||
|
||||
row := make([]uint64, rbf.ShardWidth/64)
|
||||
|
||||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := tx.Add("x", 1, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := tx.Add("x", rbf.ShardWidth+1, rbf.ShardWidth+2, rbf.ShardWidth+7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c, err := tx.Cursor("x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := c.Union(0, row); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if row[0] != 0b00001010 {
|
||||
t.Fatalf("unexpected row[0]: %#v", row[0])
|
||||
}
|
||||
|
||||
if err := c.Intersect(1, row); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if row[0] != 0b00000010 {
|
||||
t.Fatalf("unexpected row[0]: %#v", row[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Quick", func(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("-short enabled, skipping")
|
||||
} else if is32Bit() {
|
||||
t.Skip("32-bit build, skipping quick check tests")
|
||||
}
|
||||
|
||||
QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
|
||||
t.Parallel()
|
||||
|
||||
db := MustOpenDB(t)
|
||||
defer MustCloseDB(t, db)
|
||||
tx := MustBegin(t, db, true)
|
||||
defer tx.Rollback()
|
||||
values := GenerateValues(rand, rand.Intn(10000))
|
||||
rows := ToRows(values)
|
||||
|
||||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
MustAddRandom(t, rand, tx, "x", values...)
|
||||
|
||||
// Iterate over rows and randomly choose another row to union.
|
||||
for i, row0 := range rows {
|
||||
row1 := rows[rand.Intn(len(rows))]
|
||||
|
||||
bitmap := row0.Bitmap()
|
||||
if c, err := tx.Cursor("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := c.Intersect(row1.ID, bitmap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got, want := len(rbf.RowValues(bitmap)), len(row0.Intersect(row1)); got != want {
|
||||
t.Fatalf("%d. len()=%d, want %d", i, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func makeBitmap(bit []uint16) (n int, ret []uint64) {
|
||||
ret = make([]uint64, 1024)
|
||||
for _, v := range bit {
|
||||
|
|
@ -731,6 +575,7 @@ func TestCursor_RLEConversion(t *testing.T) {
|
|||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exp := 0 // expected count of BitN
|
||||
want := make([]uint16, 0, rbf.ArrayMaxSize)
|
||||
rb := func() *roaring.Bitmap {
|
||||
bm := roaring.NewBitmap()
|
||||
|
|
@ -741,6 +586,7 @@ func TestCursor_RLEConversion(t *testing.T) {
|
|||
want = append(want, x)
|
||||
want = append(want, x+1)
|
||||
x += 3
|
||||
exp += 2
|
||||
}
|
||||
bm.Put(0, roaring.NewContainerRun(runs))
|
||||
return bm
|
||||
|
|
@ -775,6 +621,18 @@ func TestCursor_RLEConversion(t *testing.T) {
|
|||
t.Fatalf("Should Not Contain %v", 0x6)
|
||||
}
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
|
||||
//i := 0
|
||||
for x := uint64(65408); x < 65536; x++ {
|
||||
_, err = tx.Add("x", x)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
//vv("on i=%v, x = %v", i, x)
|
||||
c.DebugSlowCheckAllPages()
|
||||
}
|
||||
|
||||
//add a few bits to create another run
|
||||
_, err = tx.Add("x",
|
||||
func() []uint64 {
|
||||
|
|
@ -792,10 +650,171 @@ func TestCursor_RLEConversion(t *testing.T) {
|
|||
if err := c.First(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := c.Values(), want; !reflect.DeepEqual(got, want) {
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
//vv("past 2nd check")
|
||||
|
||||
got, want := c.Values(), want
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Values()=%#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
// Split a run.
|
||||
// There's a run at the end from 0xff80 - 0xffff (65408 - 65535).
|
||||
// Split it in half.
|
||||
removeMe := uint64(65471)
|
||||
_, err = c.Remove(removeMe)
|
||||
if err != nil {
|
||||
t.Fatalf("remove failed")
|
||||
}
|
||||
|
||||
got2 := c.Values()
|
||||
target := uint16(removeMe)
|
||||
for _, v := range got2 {
|
||||
if v == target {
|
||||
t.Fatalf("removal of %v from rle did not succeed", removeMe)
|
||||
}
|
||||
}
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
|
||||
// add it back
|
||||
_, err = c.Add(removeMe)
|
||||
if err != nil {
|
||||
t.Fatalf("add failed")
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Values()=%#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
}
|
||||
|
||||
// test shrinking from bitmap down to array when a bit is removed
|
||||
func TestCursor_BitmapToArrayConversion(t *testing.T) {
|
||||
db := MustOpenDB(t)
|
||||
defer MustCloseDB(t, db)
|
||||
tx := MustBegin(t, db, true)
|
||||
defer tx.Rollback()
|
||||
|
||||
// Setup a Bitmap with more than ArrayMaxSize, then
|
||||
// delete bits to down to below ArrayMaxSize, and
|
||||
// verify the container type got converted to an array
|
||||
// and that the BitN and ElemN are always correct.
|
||||
//
|
||||
if err := tx.CreateBitmap("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// start with n = 4084 bits, where ArrayMaxSize =
|
||||
rb := roaring.NewBitmap()
|
||||
bits := make([]uint64, rbf.BitmapN)
|
||||
n := 0
|
||||
for i := range bits {
|
||||
bits[i] = 15 // ^uint64(0)
|
||||
n += 4
|
||||
if n > rbf.ArrayMaxSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
//vv("ArrayMaxSize=%v; n = %v", rbf.ArrayMaxSize, n)
|
||||
rb.Put(0, roaring.NewContainerBitmap(n, bits))
|
||||
|
||||
// setup to delete random bits down
|
||||
values := rb.Slice()
|
||||
valmap := make(map[uint64]bool)
|
||||
for _, v := range values {
|
||||
valmap[v] = true
|
||||
}
|
||||
|
||||
_, err := tx.AddRoaring("x", rb)
|
||||
if err != nil {
|
||||
t.Errorf("Add Roaring Failed %v", err)
|
||||
}
|
||||
c, err := tx.Cursor("x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := c.First(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if c.CurrentPageType() != rbf.ContainerTypeBitmapPtr {
|
||||
t.Fatalf("Should Be BitmapPtr but is: %v\n", c.CurrentPageType())
|
||||
}
|
||||
|
||||
exists, err := c.Contains(0x3)
|
||||
if err != nil {
|
||||
t.Fatalf("ERR:%v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("Should Contain %v", 0x3)
|
||||
}
|
||||
|
||||
exists, err = c.Contains(0x4)
|
||||
if err != nil {
|
||||
t.Fatalf("ERR:%v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatalf("Should Not Contain %v", 0x4)
|
||||
}
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
|
||||
for len(valmap) > 0 {
|
||||
|
||||
// pick a bit to evict
|
||||
var removeMe uint64
|
||||
for removeMe = range valmap {
|
||||
break
|
||||
}
|
||||
|
||||
c.DebugSlowCheckAllPages()
|
||||
|
||||
exists, err := c.Contains(removeMe)
|
||||
if err != nil {
|
||||
t.Fatalf("ERR:%v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("Should Contain %v", removeMe)
|
||||
}
|
||||
|
||||
_, err = c.Remove(removeMe)
|
||||
if err != nil {
|
||||
t.Fatalf("remove failed")
|
||||
}
|
||||
//vv("removed %v, have %v left", removeMe, len(valmap))
|
||||
c.DebugSlowCheckAllPages()
|
||||
|
||||
exists, err = c.Contains(removeMe)
|
||||
if err != nil {
|
||||
t.Fatalf("ERR:%v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatalf("Should NOT Contain %v", removeMe)
|
||||
}
|
||||
|
||||
delete(valmap, removeMe)
|
||||
n := len(valmap)
|
||||
|
||||
if n > rbf.ArrayMaxSize {
|
||||
if c.CurrentPageType() != rbf.ContainerTypeBitmapPtr {
|
||||
t.Fatalf("Should Be BitmapPtr but is: %v\n", c.CurrentPageType())
|
||||
}
|
||||
} else {
|
||||
if n > 0 { // no pages if n == 0, so cannot get a CurrentPageType()
|
||||
if c.CurrentPageType() == rbf.ContainerTypeBitmapPtr {
|
||||
t.Fatalf("Should NOT Be BitmapPtr but is indeed: %v\n", c.CurrentPageType())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// check it is empty
|
||||
va := c.Values()
|
||||
if len(va) != 0 {
|
||||
t.Fatalf("expected empty container, but see %v values left: '%#v'", len(va), va)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type EasyWalker struct {
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ func (c *Cursor) Row(shard, rowID uint64) (*roaring.Bitmap, error) {
|
|||
|
||||
// CurrentPageType returns the type of the container currently pointed to by cursor used in testing
|
||||
// sometimes the cursor needs to be positions prior to this call with First/Last etc.
|
||||
func (c *Cursor) CurrentPageType() int {
|
||||
func (c *Cursor) CurrentPageType() ContainerType {
|
||||
cell := c.cell()
|
||||
return cell.Type
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
|
|||
fmt.Fprintf(writer, "%s[label=\"BRANCH(%d)| n=%d\"]\n %s->%s\n", p, pgno, readCellN(page), parent, p)
|
||||
for i, n := 0, readCellN(page); i < n; i++ {
|
||||
cell := readBranchCell(page, i)
|
||||
if cell.Flags&ContainerTypeBitmap == 0 { // leaf/branch child page
|
||||
if cell.Flags&uint32(ContainerTypeBitmap) == 0 { // leaf/branch child page
|
||||
dumpdot(tx, cell.Pgno, p, writer)
|
||||
} else {
|
||||
b := fmt.Sprintf("bm%d", cell.Pgno)
|
||||
|
|
|
|||
|
|
@ -34,35 +34,41 @@ func rbfName(index, field, view string, shard uint64) string {
|
|||
return string(txkey.Prefix(index, field, view, shard))
|
||||
}
|
||||
|
||||
var _ = rbfName // keep linter happy
|
||||
|
||||
/*
|
||||
// rbtree uses 15% memory and needs half the ingest time
|
||||
// for our 10K view ingest.
|
||||
//
|
||||
// previous master with slice copying instead of rbtree:
|
||||
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:14:09.778839Z m0.TotalAlloc = 728408
|
||||
ingest_test.go:144 2020-11-13T03:14:37.492104Z m1.TotalAlloc = 41,816,617,216
|
||||
--- PASS: TestIngest_lots_of_views (27.71s)
|
||||
|
||||
// lots_views with rbtree
|
||||
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:11:01.540076Z m0.TotalAlloc = 726072
|
||||
ingest_test.go:144 2020-11-13T03:11:15.003591Z m1.TotalAlloc = 35,510,273,184
|
||||
--- PASS: TestIngest_lots_of_views (13.46s)
|
||||
*/
|
||||
func TestIngest_lots_of_views(t *testing.T) {
|
||||
|
||||
// skip unless studying perf because is long (15-30 seconds)
|
||||
//return
|
||||
// realistic
|
||||
//nCt := 10000
|
||||
|
||||
// fast CI
|
||||
nCt := 10
|
||||
|
||||
var m0, m1 runtime.MemStats
|
||||
runtime.ReadMemStats(&m0)
|
||||
vv("m0.TotalAlloc = %v", m0.TotalAlloc)
|
||||
//vv("m0.TotalAlloc = %v", m0.TotalAlloc)
|
||||
defer func() {
|
||||
runtime.ReadMemStats(&m1)
|
||||
vv("m1.TotalAlloc = %v", m1.TotalAlloc)
|
||||
//vv("m1.TotalAlloc = %v", m1.TotalAlloc)
|
||||
}()
|
||||
// rbtree uses 15% memory and needs half the ingest time
|
||||
// for our 10K view ingest.
|
||||
//
|
||||
// previous master with slice copying instead of rbtree:
|
||||
/*
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:14:09.778839Z m0.TotalAlloc = 728408
|
||||
ingest_test.go:144 2020-11-13T03:14:37.492104Z m1.TotalAlloc = 41,816,617,216
|
||||
--- PASS: TestIngest_lots_of_views (27.71s)
|
||||
*/
|
||||
// lots_views with rbtree
|
||||
/*
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:11:01.540076Z m0.TotalAlloc = 726072
|
||||
ingest_test.go:144 2020-11-13T03:11:15.003591Z m1.TotalAlloc = 35,510,273,184
|
||||
--- PASS: TestIngest_lots_of_views (13.46s)
|
||||
*/
|
||||
|
||||
path, err := ioutil.TempDir("", "rbf_ingest_lots_of_views")
|
||||
panicOn(err)
|
||||
|
|
@ -93,15 +99,14 @@ func TestIngest_lots_of_views(t *testing.T) {
|
|||
|
||||
// put a raw-bitmap container to many views.
|
||||
bits := []uint16{}
|
||||
for i := 0; i < 1<<16; i++ {
|
||||
//for i := 0; i < 100; i++ {
|
||||
//for i := 0; i < 1<<16; i++ {
|
||||
for i := 0; i < 100; i++ {
|
||||
if i%2 == 0 {
|
||||
bits = append(bits, uint16(i))
|
||||
}
|
||||
}
|
||||
ct := roaring.NewContainerArray(bits)
|
||||
|
||||
nCt := 10000
|
||||
ckey := uint64(0)
|
||||
shard := ckey / ShardWidth
|
||||
|
||||
|
|
@ -133,23 +138,10 @@ func TestIngest_lots_of_views(t *testing.T) {
|
|||
sz, err := DiskUse(path, "")
|
||||
panicOn(err)
|
||||
_ = sz
|
||||
vv("sz in bytes= %v", sz)
|
||||
|
||||
//vv("sz in bytes= %v", sz)
|
||||
db.Close()
|
||||
}
|
||||
|
||||
// func (s *rr) last() (r *RootRecord) {
|
||||
// it := s.tree.Max()
|
||||
// if it == s.tree.NegativeLimit() {
|
||||
// return nil
|
||||
// }
|
||||
// rec := it.Item().(RootRecord)
|
||||
// r = &rec
|
||||
// return
|
||||
// }
|
||||
|
||||
// var _ = (&rr{}).last // happy linter
|
||||
|
||||
func DiskUse(root string, requiredSuffix string) (tot int, err error) {
|
||||
if !DirExists(root) {
|
||||
return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
|
||||
|
|
|
|||
10
rbf/rbf.go
10
rbf/rbf.go
|
|
@ -66,9 +66,11 @@ const (
|
|||
MetaPageFlagRollback = 2
|
||||
)
|
||||
|
||||
type ContainerType int
|
||||
|
||||
// Container types.
|
||||
const (
|
||||
ContainerTypeNone = iota
|
||||
ContainerTypeNone ContainerType = iota
|
||||
ContainerTypeArray
|
||||
ContainerTypeRLE
|
||||
ContainerTypeBitmap
|
||||
|
|
@ -76,7 +78,7 @@ const (
|
|||
)
|
||||
|
||||
// ContainerTypeString returns a string representation of the container type.
|
||||
func ContainerTypeString(typ int) string {
|
||||
func (typ ContainerType) String() string {
|
||||
switch typ {
|
||||
case ContainerTypeNone:
|
||||
return "none"
|
||||
|
|
@ -289,7 +291,7 @@ func align8(offset int) int {
|
|||
// leafCell represents a leaf cell.
|
||||
type leafCell struct {
|
||||
Key uint64
|
||||
Type int // container type
|
||||
Type ContainerType
|
||||
|
||||
// ElemN is the number of "things" in Data:
|
||||
// for an array container the number of integers in the array.
|
||||
|
|
@ -477,7 +479,7 @@ func readLeafCell(page []byte, i int) leafCell {
|
|||
|
||||
var cell leafCell
|
||||
cell.Key = *(*uint64)(unsafe.Pointer(&buf[0]))
|
||||
cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8])))
|
||||
cell.Type = ContainerType(*(*uint32)(unsafe.Pointer(&buf[8])))
|
||||
cell.ElemN = int(*(*uint16)(unsafe.Pointer(&buf[12])))
|
||||
cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14])))
|
||||
|
||||
|
|
|
|||
|
|
@ -126,6 +126,8 @@ func MustBegin(tb testing.TB, db *rbf.DB, writable bool) *rbf.Tx {
|
|||
return tx
|
||||
}
|
||||
|
||||
var _ = MustAddRandom
|
||||
|
||||
// MustAddRandom adds values to a bitmap in a random order.
|
||||
func MustAddRandom(tb testing.TB, rand *rand.Rand, tx *rbf.Tx, name string, values ...uint64) {
|
||||
tb.Helper()
|
||||
|
|
@ -147,6 +149,8 @@ func GenerateValues(rand *rand.Rand, n int) []uint64 {
|
|||
return a
|
||||
}
|
||||
|
||||
var _ = ToRows
|
||||
|
||||
// ToRows returns a sorted list of rows from a set of values.
|
||||
func ToRows(values []uint64) []*Row {
|
||||
m := make(map[uint64][]uint64)
|
||||
|
|
@ -163,6 +167,8 @@ func ToRows(values []uint64) []*Row {
|
|||
return a
|
||||
}
|
||||
|
||||
var _ = Row{}
|
||||
|
||||
type Row struct {
|
||||
ID uint64
|
||||
Values []uint64
|
||||
|
|
|
|||
|
|
@ -1733,7 +1733,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) {
|
|||
for _, cell := range readLeafCells(buf, cells) {
|
||||
other := &LeafCell{
|
||||
Key: cell.Key,
|
||||
Type: ContainerTypeString(cell.Type),
|
||||
Type: cell.Type,
|
||||
}
|
||||
|
||||
switch cell.Type {
|
||||
|
|
@ -1979,7 +1979,7 @@ type LeafPage struct {
|
|||
// LeafCell represents a leaf cell in the public API.
|
||||
type LeafCell struct {
|
||||
Key uint64
|
||||
Type string // container type
|
||||
Type ContainerType
|
||||
Pgno uint32 // bitmap pointer only
|
||||
Values []uint16 // array & rle containers only
|
||||
}
|
||||
|
|
|
|||
137
rbf/util_test.go
Normal file
137
rbf/util_test.go
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package rbf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
"io"
|
||||
)
|
||||
|
||||
// util_test adds reusable utilities for testing.
|
||||
// Here we catch BitN and ElemN mis-settings by
|
||||
// scanning all data under an rbf-root (logically equivalent
|
||||
// to a single roaring.Bitmap with multiple rows).
|
||||
|
||||
// verify that BitN and ElemN are correct.
|
||||
func (c_orig *Cursor) DebugSlowCheckAllPages() {
|
||||
|
||||
// work with a totally new Cursor, so we don't impact our current cursor
|
||||
// so any test using the cursor isn't disturbed.
|
||||
c2 := Cursor{tx: c_orig.tx}
|
||||
c2.stack.elems[0] = c_orig.stack.elems[0]
|
||||
err := c2.First()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// ok, can be empty
|
||||
return
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
c2.Dump("debug.dot")
|
||||
|
||||
checkElemNBitN(c2.tx, 0)
|
||||
}
|
||||
|
||||
// checkElemNBitN recursively writes the tree representation starting from a given page to STDERR.
|
||||
func checkElemNBitN(tx *Tx, pgno uint32) {
|
||||
page, err := tx.readPage(pgno)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if IsMetaPage(page) {
|
||||
visitor := func(pgno uint32, records []*RootRecord) {
|
||||
for _, record := range records {
|
||||
checkElemNBitN(tx, record.Pgno)
|
||||
}
|
||||
}
|
||||
Walk(tx, readMetaRootRecordPageNo(page), visitor)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle each type of page
|
||||
switch typ := readFlags(page); typ {
|
||||
case PageTypeBranch:
|
||||
for i, n := 0, readCellN(page); i < n; i++ {
|
||||
cell := readBranchCell(page, i)
|
||||
if cell.Flags&uint32(ContainerTypeBitmap) == 0 { // leaf/branch child page
|
||||
checkElemNBitN(tx, cell.Pgno)
|
||||
}
|
||||
// else is a bitmap
|
||||
}
|
||||
case PageTypeLeaf:
|
||||
cellCheckElemNBitN(tx, page)
|
||||
}
|
||||
}
|
||||
|
||||
func cellCheckElemNBitN(tx *Tx, b []byte) {
|
||||
pgno := readPageNo(b)
|
||||
if pgno == Magic32() {
|
||||
// skip meta page
|
||||
return
|
||||
}
|
||||
|
||||
flags := readFlags(b)
|
||||
cellN := readCellN(b)
|
||||
|
||||
switch {
|
||||
case flags&PageTypeLeaf != 0:
|
||||
for i := 0; i < cellN; i++ {
|
||||
cell := readLeafCell(b, i)
|
||||
verifyElemNBitN(tx, cell)
|
||||
}
|
||||
}
|
||||
}
|
||||
func verifyElemNBitN(tx *Tx, lc leafCell) {
|
||||
|
||||
obsElemN := int32(-1)
|
||||
obsBitN := int32(-1)
|
||||
|
||||
typ := ""
|
||||
var c *roaring.Container
|
||||
switch lc.Type {
|
||||
case ContainerTypeArray:
|
||||
typ = "array"
|
||||
a := toArray16(lc.Data)
|
||||
c = roaring.NewContainerArray(a)
|
||||
obsBitN = c.N()
|
||||
obsElemN = int32(len(a))
|
||||
|
||||
case ContainerTypeRLE:
|
||||
typ = "rle"
|
||||
a := toInterval16(lc.Data)
|
||||
c = roaring.NewContainerRun(a)
|
||||
obsBitN = c.N()
|
||||
obsElemN = int32(len(a))
|
||||
|
||||
case ContainerTypeBitmap:
|
||||
panic("should never get an actual bitmap!")
|
||||
|
||||
case ContainerTypeBitmapPtr:
|
||||
typ = "bitmap_ptr"
|
||||
_, bm, _ := tx.leafCellBitmap(toPgno(lc.Data))
|
||||
c = roaring.NewContainerBitmap(-1, bm)
|
||||
obsBitN = int32(c.N())
|
||||
obsElemN = 0 // by definition, should always see 0 for raw bitmaps's ElemN
|
||||
}
|
||||
if lc.BitN != int(obsBitN) {
|
||||
panic(fmt.Sprintf("lc.BitN(%v) != obsBitN(%v); typ='%v'", lc.BitN, obsBitN, typ))
|
||||
}
|
||||
if lc.ElemN != int(obsElemN) {
|
||||
panic(fmt.Sprintf("lc.ElemN(%v) != obsElemN(%v); typ='%v'", lc.ElemN, obsElemN, typ))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue