scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-15 16:51:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor putleaf

Browse source
This commit is contained in:
Todd Gruben 2020-07-13 10:48:25 -05:00
parent 3c619fa2ef
commit b11b43af43
9 changed files with 631 additions and 294 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
rbf/array.go
View file

@ -1,17 +1,3 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
@ -30,6 +16,12 @@ func fromArray16(a []uint16) []byte {
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
}
func cloneArray16(a []uint16) []uint16 {
other := make([]uint16, len(a))
copy(other, a)
return other
}
// arrayIndex returns the insertion index of v in a. Returns true if exact match.
func arrayIndex(a []uint16, v uint16) (int, bool) {
return search(len(a), func(i int) int {
@ -67,3 +59,9 @@ func toInterval16(a []byte) []roaring.Interval16 {
func fromInterval16(a []roaring.Interval16) []byte {
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
}
func cloneInterval16(a []roaring.Interval16) []roaring.Interval16 {
other := make([]roaring.Interval16, len(a))
copy(other, a)
return other
}

391
rbf/cursor.go
View file

@ -1,20 +1,7 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
"encoding/binary"
"fmt"
"io"
"math/bits"
@ -25,7 +12,7 @@ import (
)
const (
bitmapN = (1 << 16) / 64
BitmapN = (1 << 16) / 64
)
type Cursor struct {
@ -83,10 +70,10 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
}
return runs, true
}
func checkRun(runs []roaring.Interval16, key uint64) leafCell {
func checkRun(runs []roaring.Interval16, key uint64) leafArgs {
if len(runs) >= RLEMaxSize {
//convertToBitmap
bitmap := make([]uint64, bitmapN)
bitmap := make([]uint64, BitmapN)
for _, iv := range runs {
w1, w2 := iv.Start/64, iv.Last/64
b1, b2 := iv.Start&63, iv.Last&63
@ -122,9 +109,9 @@ func checkRun(runs []roaring.Interval16, key uint64) leafCell {
n += popcount(v)
}
return leafCell{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
return leafArgs{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
}
return leafCell{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
return leafArgs{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
}
// Add sets a bit on the underlying bitmap.
@ -135,7 +122,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
if exact, err := c.Seek(hi); err != nil {
return false, err
} else if !exact {
return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})})
return true, c.putLeafCell(leafArgs{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})})
}
// If the container exists and bit is not set then update the page.
@ -154,7 +141,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
copy(other, a[:i])
other[i] = lo
copy(other[i+1:], a[i:])
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
return true, c.putLeafCell(leafArgs{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
case ContainerTypeRLE:
runs := toInterval16(cell.Data)
@ -168,14 +155,19 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
return false, nil
case ContainerTypeBitmap:
// Exit if bit set in bitmap container.
a := cloneArray64(toArray64(cell.Data))
pgno, bm, err := cell.GetBitmap(c.tx)
if err != nil {
return false, err
}
a := cloneArray64(bm)
if a[lo/64]&(1<<uint64(lo%64)) != 0 {
return false, nil
}
// Insert new value and rewrite page.
a[lo/64] |= 1 << uint64(lo%64)
if err := c.tx.writeBitmapPage(c.stack.elems[c.stack.index].pgno, fromArray64(a)); err != nil {
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
return false, err
}
return true, nil
@ -213,14 +205,40 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
other := make([]uint16, len(a)-1)
copy(other[:i], a[:i])
copy(other[i:], a[i+1:])
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
return true, c.putLeafCell(leafArgs{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
case ContainerTypeRLE:
panic("TODO(BBJ): Implement RLE")
r := toInterval16(cell.Data)
i, contains := roaring.BinSearchRuns(lo, r)
if !contains {
return false, nil
}
copy(c.rle[:], r)
runs := c.rle[:len(r)]
if lo == runs[i].Last && lo == runs[i].Start {
runs = append(runs[:i], runs[i+1:]...)
} else if lo == runs[i].Last {
runs[i].Last--
} else if lo == c.rle[i].Start {
runs[i].Start++
} else if lo > runs[i].Start {
last := runs[i].Last
runs[i].Last = lo - 1
runs = append(runs, roaring.Interval16{})
copy(runs[i+2:], runs[i+1:])
runs[i+1] = roaring.Interval16{Start: lo + 1, Last: last}
}
if len(runs) == 0 {
return true, c.deleteLeafCell(cell.Key)
}
return true, c.putLeafCell(leafArgs{Key: cell.Key, Type: ContainerTypeRLE, N: len(runs), Data: fromInterval16(runs)})
case ContainerTypeBitmap:
// Exit if bit not set in bitmap container.
a := cloneArray64(toArray64(cell.Data))
pgno, bm, err := cell.GetBitmap(c.tx)
if err != nil {
return false, err
}
a := cloneArray64(bm)
if a[lo/64]&(1<<uint64(lo%64)) == 0 {
return false, nil
}
@ -229,7 +247,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
// Clear bit and rewrite page.
a[lo/64] &^= 1 << uint64(lo%64)
if err := c.tx.writeBitmapPage(c.stack.elems[c.stack.index].pgno, fromArray64(a)); err != nil {
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
return false, err
}
return true, nil
@ -266,24 +284,62 @@ func (c *Cursor) Contains(v uint64) (exists bool, err error) {
}
return false, nil
case ContainerTypeBitmap:
a := toArray64(cell.Data)
return a[lo/64]&(1<<uint64(lo%64)) != 0, nil
_, a, err := cell.GetBitmap(c.tx)
return a[lo/64]&(1<<uint64(lo%64)) != 0, err
default:
return false, fmt.Errorf("rbf.Cursor.Contains(): invalid container type: %d", cell.Type)
}
}
// putLeafCell writes a cell to the currently positioned page & index.
// If the new cell causes the size to exceed the page size then split into multiple pages.
func (c *Cursor) putLeafCell(cell leafCell) (err error) {
// TODO(78720): Handle empty leaf cells.
func fromPgno(val uint32) []byte {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, val)
// return (*[4]byte)(unsafe.Pointer(&val))[:]
return buf
}
func toPgno(val []byte) uint32 {
return binary.LittleEndian.Uint32(val)
}
func (c *Cursor) putLeafCell(in leafArgs) (err error) {
cells := readLeafCells(c.leafPage, c.leafCells[:])
elem := &c.stack.elems[c.stack.index]
cells := readLeafCells(c.leafPage, elem.isBitmap, c.leafCells[:])
// Shift cells over if this is an insertion.
cell := leafCell{Key: in.Key, Type: in.Type, N: in.N, Data: in.Data}
if elem.index >= len(cells) || c.Key() != cell.Key {
//new cell
if cell.Type == ContainerTypeBitmap {
//allocated bitmap()
bitmapPgno, _ := c.tx.allocate()
cell.Data = fromPgno(bitmapPgno)
}
// Shift cells over if this is an insertion.
cells = append(cells, leafCell{})
copy(cells[elem.index+1:], cells[elem.index:])
} else if in.Type == ContainerTypeBitmap {
ecell := cells[elem.index]
if ecell.Type != ContainerTypeBitmap {
bitmapPgno, _ := c.tx.allocate()
cell.Data = fromPgno(bitmapPgno)
} else {
cell.Data = ecell.Data //fill in the old pgno
}
}
if in.Type == ContainerTypeArray && in.N > ArrayMaxSize {
//convert to bitmap
in.Type = ContainerTypeBitmap
a := make([]uint64, PageSize/8)
for _, v := range toArray16(in.Data) {
a[v/64] |= 1 << uint64(v%64)
}
in.Data = fromArray64(a)
cell.Type = ContainerTypeBitmap
bitmapPgno, _ := c.tx.allocate()
cell.Data = fromPgno(bitmapPgno)
}
cells[elem.index] = cell
// Split into multiple pages if page size is exceeded.
@ -291,23 +347,16 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) {
if leafCellsPageSize(cells) >= PageSize {
groups = splitLeafCells(cells)
}
// Write each group to a separate page.
var hasBitmap bool
for _, group := range groups {
if len(group) == 1 && (group[0].Type == ContainerTypeBitmap || group[0].N > ArrayMaxSize) && (group[0].Type != ContainerTypeRLE) {
hasBitmap = true
}
}
newRoot := (len(groups) > 1) && (c.stack.index == 0)
var parents []branchCell
origPgno := elem.pgno
newRoot := (len(groups) > 1 || hasBitmap) && c.stack.index == 0
// newRoot if split occured and bottom of the stack
for i, group := range groups {
// First page should overwrite the original.
// Subsequent pages should allocate new pages.
parent := branchCell{Key: group[0].Key}
parent := branchCell{Key: group[0].Key} //<<< this is the key spot for making sure that key is correct
if i == 0 && !newRoot {
parent.Pgno = origPgno
} else {
@ -316,33 +365,26 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) {
}
}
// If cell exceeds threshold then write out bitmap page.
// Otherwise encode leaf page normally.
// if the cell is a bitmap write out its page
if in.Type == ContainerTypeBitmap {
var bm [PageSize]byte
copy(bm[:], fromArray64(toArray64(in.Data)))
c.tx.writeBitmapPage(toPgno(cell.Data), bm[:])
}
var buf [PageSize]byte
if len(group) == 1 && (group[0].Type == ContainerTypeBitmap || group[0].N > ArrayMaxSize) && (group[0].Type != ContainerTypeRLE) {
// Write cells to page.
writePageNo(buf[:], parent.Pgno)
writeFlags(buf[:], PageTypeLeaf)
writeCellN(buf[:], len(group))
hasBitmap = true
parent.Flags |= ContainerTypeBitmap
copy(buf[:], fromArray64(cell.Bitmap()))
offset := dataOffset(len(group))
for j, cell := range group {
writeLeafCell(buf[:], j, offset, cell)
offset += align8(cell.Size())
}
if err := c.tx.writeBitmapPage(parent.Pgno, buf[:]); err != nil {
return err
}
} else {
// Write cells to page.
writePageNo(buf[:], parent.Pgno)
writeFlags(buf[:], PageTypeLeaf)
writeCellN(buf[:], len(group))
offset := dataOffset(len(group))
for j, cell := range group {
writeLeafCell(buf[:], j, offset, cell)
offset += align8(cell.Size())
}
if err := c.tx.writePage(buf[:]); err != nil {
return err
}
if err := c.tx.writePage(buf[:]); err != nil {
return err
}
parents = append(parents, parent)
@ -350,9 +392,8 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) {
// TODO(BBJ): Update page in buffer & cursor stack.
// If this is not a split and we have no bitmap containers, then exit now.
// Bitmap containers require a parent and the parent's flag must be set.
if len(groups) == 1 && !hasBitmap {
// If this is not a split then exit now.
if len(groups) == 1 {
return nil
}
@ -369,9 +410,15 @@ func (c *Cursor) putLeafCell(cell leafCell) (err error) {
// deleteLeafCell removes a cell from the currently positioned page & index.
func (c *Cursor) deleteLeafCell(key uint64) (err error) {
cells := readLeafCells(c.leafPage, c.leafCells[:])
elem := &c.stack.elems[c.stack.index]
cells := readLeafCells(c.leafPage, elem.isBitmap, c.leafCells[:])
oldPageKey := cells[0].Key
cell := c.cell()
if cell.Type == ContainerTypeBitmap {
if err := c.tx.deallocate(toPgno(cell.Data)); err != nil {
return err
}
}
// If no more cells exist and we have a parent, remove from parent.
if c.stack.index > 0 && len(cells) == 1 {
@ -570,7 +617,7 @@ func (c *Cursor) deleteBranchCell(stackIndex int, key uint64) (err error) {
return err
}
if stackIndex > 0 && oldPageKey != cells[0].Key {
if stackIndex > 0 && len(cells) > 0 && oldPageKey != cells[0].Key {
return c.updateBranchCell(stackIndex-1, cells[0].Key)
}
return nil
@ -606,6 +653,9 @@ func splitLeafCells(cells []leafCell) [][]leafCell {
// half a page then create a new group of cells.
if cellN != 0 && (dataOffset(cellN+1)+dataSize+sz) > (PageSize*60)/100 {
slices, dataSize = append(slices, nil), 0
} else if cellN != 0 && cell.Type == ContainerTypeArray && cell.N > ArrayMaxSize {
slices, dataSize = append(slices, nil), 0
sz = PageSize
}
// Append to current slice & increase total cell data size.
@ -644,18 +694,12 @@ func splitBranchCells(cells []branchCell) [][]branchCell {
// Key returns the key that the cursor is currently positioned over.
func (c *Cursor) Key() uint64 {
elem := &c.stack.elems[c.stack.index]
if elem.isBitmap {
return elem.key
}
offset := readCellOffset(c.leafPage, elem.index)
return *(*uint64)(unsafe.Pointer(&c.leafPage[offset]))
}
func (c *Cursor) cell() leafCell {
elem := &c.stack.elems[c.stack.index]
if elem.isBitmap {
return leafCell{Type: ContainerTypeBitmap, Key: elem.key, Data: c.leafPage[:]}
}
return readLeafCell(c.leafPage[:], elem.index)
}
@ -677,21 +721,10 @@ func (c *Cursor) First() error {
// Read cell pgno into the next stack level.
cell := readBranchCell(buf, elem.index)
isBitmap := cell.Flags&ContainerTypeBitmap != 0
c.stack.elems[c.stack.index+1] = stackElem{
pgno: cell.Pgno,
key: cell.Key,
isBitmap: isBitmap,
}
// If cell points at a bitmap page then increment stack but exit immediately.
if isBitmap {
c.stack.index++
if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil {
return err
}
return nil
pgno: cell.Pgno,
key: cell.Key,
}
case PageTypeLeaf:
@ -726,21 +759,9 @@ func (c *Cursor) Last() error {
// Read cell pgno into the next stack level.
cell := readBranchCell(buf, elem.index)
isBitmap := cell.Flags&ContainerTypeBitmap != 0
c.stack.elems[c.stack.index+1] = stackElem{
pgno: cell.Pgno,
key: cell.Key,
isBitmap: isBitmap,
}
// If cell points at a bitmap page then increment stack but exit immediately.
if isBitmap {
c.stack.index++
if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil {
return err
}
return nil
pgno: cell.Pgno,
key: cell.Key,
}
case PageTypeLeaf:
@ -780,6 +801,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
}
return 1
})
//if not found (ok) the cell
if !ok && index > 0 {
index--
}
@ -788,21 +810,10 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
// Read cell pgno into the next stack level.
cell := readBranchCell(buf, elem.index)
isBitmap := cell.Flags&ContainerTypeBitmap != 0
c.stack.elems[c.stack.index+1] = stackElem{
pgno: cell.Pgno,
key: cell.Key,
isBitmap: isBitmap,
}
// If cell points at a bitmap page then increment stack but exit immediately.
if isBitmap {
c.stack.index++
if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil {
return false, err
}
return ok, nil
pgno: cell.Pgno,
key: cell.Key,
}
case PageTypeLeaf:
@ -833,7 +844,7 @@ func (c *Cursor) Next() error {
}
// Move forward to the next leaf element if available.
if elem := &c.stack.elems[c.stack.index]; !elem.isBitmap && elem.index < readCellN(c.leafPage)-1 {
if elem := &c.stack.elems[c.stack.index]; elem.index < readCellN(c.leafPage)-1 {
elem.index++
return nil
}
@ -848,7 +859,7 @@ func (c *Cursor) Prev() error {
}
// Move forward to the next leaf element if available.
if elem := &c.stack.elems[c.stack.index]; !elem.isBitmap && elem.index > 0 {
if elem := &c.stack.elems[c.stack.index]; elem.index > 0 {
elem.index--
return nil
}
@ -880,21 +891,10 @@ func (c *Cursor) Prev() error {
switch typ := readFlags(buf); typ {
case PageTypeBranch:
cell := readBranchCell(buf, elem.index)
isBitmap := cell.Flags&ContainerTypeBitmap != 0
c.stack.elems[c.stack.index+1] = stackElem{
pgno: cell.Pgno,
key: cell.Key,
isBitmap: isBitmap,
}
// If cell points at a bitmap page then increment stack but exit immediately.
if isBitmap {
c.stack.index++
if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil {
return err
}
return nil
pgno: cell.Pgno,
key: cell.Key,
}
case PageTypeLeaf:
@ -936,7 +936,8 @@ func (c *Cursor) Union(rowID uint64, row []uint64) error {
case ContainerTypeRLE:
panic("TODO(BBJ): rbf.Bitmap.Union() RLE support")
case ContainerTypeBitmap:
for i, v := range toArray64(cell.Data) {
_, bm, _ := cell.GetBitmap(c.tx)
for i, v := range bm {
row[(offset/64)+uint64(i)] |= v
}
default:
@ -974,13 +975,17 @@ func (c *Cursor) Intersect(rowID uint64, row []uint64) error {
switch cell.Type {
case ContainerTypeArray:
for i, v := range cell.Bitmap() {
for i, v := range cell.Bitmap(c.tx) {
row[(offset/64)+uint64(i)] &= v
}
case ContainerTypeRLE:
panic("TODO(BBJ): rbf.Bitmap.Intersect() RLE support")
case ContainerTypeBitmap:
for i, v := range toArray64(cell.Data) {
_, bm, err := cell.GetBitmap(c.tx)
if err != nil {
return err
}
for i, v := range bm {
row[(offset/64)+uint64(i)] &= v
}
default:
@ -1003,21 +1008,15 @@ func (c *Cursor) Intersect(rowID uint64, row []uint64) error {
// Values returns the values for the container the cursor is currently pointing to.
func (c *Cursor) Values() []uint16 {
elem := &c.stack.elems[c.stack.index]
var cell leafCell
if elem.isBitmap {
cell = leafCell{Type: ContainerTypeBitmap, Key: elem.key, Data: c.leafPage}
} else {
cell = readLeafCell(c.leafPage[:], elem.index)
}
return cell.Values()
cell := readLeafCell(c.leafPage[:], elem.index)
return cell.Values(c.tx)
}
// stackElem represents a single element on the cursor stack.
type stackElem struct {
pgno uint32 // current page number
index int // cell index
key uint64 // element key
isBitmap bool // if true, entire page is a bitmap
pgno uint32 // current page number
index int // cell index
key uint64 // element key
}
func (c *Cursor) goNextPage() error {
@ -1048,23 +1047,10 @@ func (c *Cursor) goNextPage() error {
switch typ := readFlags(buf); typ {
case PageTypeBranch:
cell := readBranchCell(buf, elem.index)
isBitmap := cell.Flags&ContainerTypeBitmap != 0
c.stack.elems[c.stack.index+1] = stackElem{
pgno: cell.Pgno,
key: cell.Key,
isBitmap: isBitmap,
pgno: cell.Pgno,
key: cell.Key,
}
// If cell points at a bitmap page then increment stack but exit immediately.
if isBitmap {
c.stack.index++
if c.leafPage, err = c.tx.readPage(cell.Pgno); err != nil {
return err
}
return nil
}
case PageTypeLeaf:
elem.index = 0
c.leafPage = buf
@ -1075,8 +1061,7 @@ func (c *Cursor) goNextPage() error {
}
}
func ConvertToLeaf(key uint64, c *roaring.Container) (result leafCell) {
//TODO(twg) clean up roaring constant import export
func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafArgs) {
result.Key = key
result.N = int(c.N())
result.Type = ContainerTypeNone
@ -1110,7 +1095,6 @@ func ConvertToLeaf(key uint64, c *roaring.Container) (result leafCell) {
result.Type = ContainerTypeRLE
result.Data = fromInterval16(r)
return
}
return
}
@ -1123,16 +1107,18 @@ func (c *Cursor) merge(key uint64, data *roaring.Container) (bool, error) {
d := toArray16(cell.Data)
container = roaring.NewContainerArray(d)
case ContainerTypeBitmap:
d := toArray64(cell.Data)
_, d, err := cell.GetBitmap(c.tx)
if err != nil {
return false, err
}
container = roaring.NewContainerBitmap(cell.N, d)
case ContainerTypeRLE:
d := toInterval16(cell.Data)
container = roaring.NewContainerRun(d)
}
res := roaring.Union(data, container)
if res.N() != data.N() {
leaf := ConvertToLeaf(key, res)
leaf := ConvertToLeafArgs(key, res)
err := c.putLeafCell(leaf)
return true, err
}
@ -1144,7 +1130,7 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) {
itr, _ := bm.Containers.Iterator(0)
for itr.Next() {
hi, cont := itr.Value()
leaf := ConvertToLeaf(hi, cont)
leaf := ConvertToLeafArgs(hi, cont)
if leaf.N == 0 {
continue
}
@ -1160,7 +1146,6 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) {
changed = true
continue
}
// If the container exists and bit is not set then update the page.
u, err := c.merge(hi, cont)
if err != nil {
@ -1176,3 +1161,59 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) {
func popcount(x uint64) uint64 {
return uint64(bits.OnesCount64(x))
}
func (c *Cursor) RemoveRoaring(bm *roaring.Bitmap) (changed bool, err error) {
itr, _ := bm.Containers.Iterator(0)
for itr.Next() {
hi, cont := itr.Value()
if cont.N() == 0 {
continue
}
// Move cursor to the key of the container.
// Insert new container if it doesn't exist.
if exact, err := c.Seek(hi); err != nil {
return false, err
} else if exact {
f, err := c.difference(hi, cont)
if err != nil {
return f, err
}
if f {
changed = true
}
}
}
return
}
func (c *Cursor) difference(key uint64, data *roaring.Container) (bool, error) {
cell := c.cell()
var container *roaring.Container
switch cell.Type {
case ContainerTypeArray:
d := toArray16(cell.Data)
container = roaring.NewContainerArray(d)
case ContainerTypeBitmap:
_, d, err := cell.GetBitmap(c.tx)
if err != nil {
return false, err
}
container = roaring.NewContainerBitmap(cell.N, d)
case ContainerTypeRLE:
d := toInterval16(cell.Data)
container = roaring.NewContainerRun(d)
}
res := roaring.Difference(container, data)
if res == nil {
return true, c.deleteLeafCell(cell.Key)
}
if res.N() != container.N() {
leaf := ConvertToLeafArgs(key, res)
err := c.putLeafCell(leaf)
return true, err
}
return false, nil
}

251
rbf/cursor_test.go
View file

@ -19,6 +19,7 @@ import (
"math/rand"
"reflect"
"sort"
"strings"
"testing"
"github.com/pilosa/pilosa/v2/rbf"
@ -762,6 +763,7 @@ func TestCursor_RLEConversion(t *testing.T) {
} else if err := c.First(); err != nil {
t.Fatal(err)
}
if c.CurrentPageType() != rbf.ContainerTypeRLE {
t.Fatalf("Should Be RLE but is: %v\n", c.CurrentPageType())
}
@ -772,6 +774,14 @@ func TestCursor_RLEConversion(t *testing.T) {
if !exists {
t.Fatalf("Should Contain %v", 0x7)
}
exists, err = c.Contains(0x6)
if err != nil {
t.Fatalf("ERR:%v", err)
}
if exists {
t.Fatalf("Should Not Contain %v", 0x6)
}
//add a few bits to create another run
_, err = tx.Add("x",
func() []uint64 {
@ -797,3 +807,244 @@ func TestCursor_RLEConversion(t *testing.T) {
}
}
type EasyWalker struct {
tx *rbf.Tx
path strings.Builder
}
func (e *EasyWalker) Visitor(pgno uint32, records []*rbf.RootRecord) {
for _, record := range records {
e.VisitRoot(record.Pgno, record.Name)
rbf.Page(e.tx, record.Pgno, e)
}
}
func (e *EasyWalker) VisitRoot(pgno uint32, name string) {
e.path.WriteString("R")
}
func (e *EasyWalker) VisitBranch(pgno uint32) {
e.path.WriteString("B")
}
func (e *EasyWalker) VisitLeaf(pgno uint32) {
e.path.WriteString("L")
}
func (e *EasyWalker) VisitBitmap(pgno uint32) {
}
func (e *EasyWalker) String() string {
return e.path.String()
}
func TestCursor_UpdateBranchCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
c, err := tx.Cursor("x")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
}
changed, err := c.Add(1)
if changed {
if err := c.First(); err != nil {
t.Fatal(err)
}
if got, want := c.Values(), []uint16{uint16(1)}; !reflect.DeepEqual(got, want) {
t.Fatal(err)
}
} else {
t.Fatal("Expected Add Change")
}
changed, err = c.Remove(1)
if changed {
c.First()
if got, want := c.Values(), []uint16{}; !reflect.DeepEqual(got, want) {
t.Fatal(err)
}
} else {
t.Fatal("Expected Remove Change")
}
rb := func() *roaring.Bitmap {
bm := roaring.NewBitmap()
bm.Put(0, roaring.NewContainerRun([]roaring.Interval16{{Start: 1, Last: 2}}))
return bm
}()
_, err = tx.AddRoaring("x", rb)
if err != nil {
t.Fatal(err)
}
changed, err = c.Remove(2)
if changed {
c.First()
if got, want := c.Values(), []uint16{1}; !reflect.DeepEqual(got, want) {
t.Fatal(err)
}
} else {
t.Fatal("Expected Remove Change")
}
changed, err = c.Remove(1)
if changed {
c.First()
if got, want := c.Values(), []uint16{}; !reflect.DeepEqual(got, want) {
t.Fatal(err)
}
} else {
t.Fatal("Expected Remove Change")
}
}
func TestCursor_SplitBranchCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
rb := func(key uint64) *roaring.Bitmap {
bm := roaring.NewBitmap()
bits := make([]uint64, rbf.BitmapN)
n := 0
for i := range bits {
bits[i] = ^uint64(0)
n += 64
}
bm.Put(key, roaring.NewContainerBitmap(n, bits))
return bm
}
//634 == offset, 24== size of leafcell with bitmap
// measured should split at 634+(24*314)
numContainers := 314
for i := 0; i < numContainers; i++ { //need to calculate how many will force a split
b := rb(uint64(i))
tx.AddRoaring("x", b)
}
before := &EasyWalker{tx: tx}
rbf.Page(tx, 0, before)
if before.String() != "RL" {
t.Fatalf("Expecting RL (one branch) got %v", before.String())
}
// adding one more container should split it
tx.AddRoaring("x", rb(uint64(numContainers)))
after := &EasyWalker{tx: tx}
rbf.Page(tx, 0, after)
if after.String() != "RBLL" {
t.Fatalf("Expecting RBLL (a branch split) got %v", after.String())
}
}
func TestCursor_RemoveCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
cur, _ := tx.Cursor("x")
rb := func(key uint64) *roaring.Bitmap {
bm := roaring.NewBitmap()
bits := make([]uint64, rbf.BitmapN)
n := 0
for i := range bits {
bits[i] = ^uint64(0)
n += 64
}
bm.Put(key, roaring.NewContainerBitmap(n, bits))
return bm
}
numContainers := 455 //enough containers to cause a split
for i := 0; i < numContainers; i++ {
b := rb(uint64(i))
tx.AddRoaring("x", b)
}
for i := numContainers; i >= 1; i-- {
cur.RemoveRoaring(rb(uint64(i)))
}
cur.RemoveRoaring(rb(uint64(0)))
//f, err := os.OpenFile("before.dot", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 066)
}
//These aren't test i'm just using to generate graphs to look at structure
func TestCursor_PlayContainer(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
many := func(c *rbf.Cursor, start, count uint64) {
for i := start; i < start+count; i++ {
c.Add(i)
}
}
cur, _ := tx.Cursor("x")
offset := uint64(0)
many(cur, 0, rbf.ArrayMaxSize+offset)
many(cur, 65536, rbf.ArrayMaxSize+offset)
/*
many(cur, 2*65536, rbf.ArrayMaxSize+offset)
many(cur, 3*65536, rbf.ArrayMaxSize) //+offset)
many(cur, 4*65536, rbf.ArrayMaxSize) //+offset)
many(cur, 5*65536, rbf.ArrayMaxSize) //+offset)
many(cur, 6*65536, 10) //+offset)
many(cur, 7*65536, 10) //+offset)
many(cur, 8*65536, rbf.ArrayMaxSize+10) //+offset)
many(cur, 9*65536, rbf.ArrayMaxSize+10) //+offset)
*/
cur.First()
cur.Dump("fun.dot")
}
func TestCursor_OneBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
rb := func(key uint64) *roaring.Bitmap {
bm := roaring.NewBitmap()
bits := make([]uint64, rbf.BitmapN)
n := 0
for i := range bits {
bits[i] = ^uint64(0)
n += 64
}
bm.Put(key, roaring.NewContainerBitmap(n, bits))
return bm
}
numContainers := 4
for i := 0; i < numContainers; i++ { //need to calculate how many will force a split
b := rb(uint64(i)) // measured at i=454 seems reasonable should occur at Len(branchcells)+header >8192
tx.AddRoaring("x", b)
}
cur, err := tx.Cursor("x")
if err != nil {
panic(err)
}
cur.First()
cur.Dump("fun.dot")
}

69
rbf/cursorx.go
View file

@ -27,9 +27,7 @@ import (
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
shardVsContainerExponent := uint(4) //needs constant exported from roaring package
if err := c.First(); err != nil {
return nil, err
}
c.First()
rows := make([]uint64, 0)
var err error
var lastRow uint64 = math.MaxUint64
@ -48,7 +46,6 @@ func (c *Cursor) Rows() ([]uint64, error) {
}
return rows, err
}
func (tx *Tx) FieldViews() []string {
r, _ := tx.rootRecords()
res := make([]string, len(r))
@ -57,23 +54,17 @@ func (tx *Tx) FieldViews() []string {
}
return res
}
func (c *Cursor) DumpKeys() error {
if err := c.First(); err != nil {
return err
}
func (c *Cursor) DumpKeys() {
c.First()
for {
err := c.Next()
if err == io.EOF {
return nil
} else if err != nil {
return err
break
}
cell := c.cell()
fmt.Println("key", cell.Key)
}
}
func (c *Cursor) DumpStack() {
fmt.Println("STACK")
for i := c.stack.index; i >= 0; i-- {
@ -81,18 +72,18 @@ func (c *Cursor) DumpStack() {
}
fmt.Println()
}
func (c *Cursor) Dump() {
bufStdout := bufio.NewWriter(os.Stdout)
defer bufStdout.Flush()
func (c *Cursor) Dump(name string) {
writer, _ := os.Create(name)
defer writer.Close()
bufStdout := bufio.NewWriter(writer)
fmt.Fprintf(bufStdout, "digraph RBF{\n")
fmt.Fprintf(bufStdout, "rankdir=\"LR\"\n")
fmt.Fprintf(bufStdout, "node [shape=record height=.1]\n")
dumpdot(c.tx, 0, " ", bufStdout)
fmt.Fprintf(bufStdout, "\n}")
bufStdout.Flush()
}
func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) {
base := rowID * ShardWidth
@ -108,9 +99,7 @@ func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) {
elem := &c.stack.elems[c.stack.index]
n := readCellN(c.leafPage)
if elem.index >= n {
if err := c.goNextPage(); err != nil {
return nil, err
}
c.goNextPage()
}
}
other := roaring.NewSliceBitmap()
@ -149,3 +138,41 @@ func toContainer(l leafCell) *roaring.Container {
}
return nil
}
type Walker interface {
Visitor(pgno uint32, records []*RootRecord)
VisitRoot(pgno uint32, name string)
VisitBranch(pgno uint32)
VisitLeaf(pgno uint32)
VisitBitmap(pgno uint32)
}
func Page(tx *Tx, pgno uint32, walker Walker) {
page, err := tx.readPage(pgno)
if err != nil {
panic(err)
}
if IsMetaPage(page) {
Walk(tx, readMetaRootRecordPageNo(page), walker.Visitor)
return
}
// Handle
switch typ := readFlags(page); typ {
case PageTypeBranch:
walker.VisitBranch(pgno)
for i, n := 0, readCellN(page); i < n; i++ {
cell := readBranchCell(page, i)
if cell.Flags&ContainerTypeBitmap == 0 { // leaf/branch child page
Page(tx, cell.Pgno, walker)
} else {
walker.VisitBitmap(cell.Pgno)
}
}
case PageTypeLeaf:
walker.VisitLeaf(pgno)
default:
panic(err)
}
}

33
rbf/dot.go
View file

@ -1,17 +1,4 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package rbf implements the roaring b-tree file format.
package rbf
import (
@ -32,7 +19,8 @@ func dotCell(b []byte, parent string, writer io.Writer) {
switch {
case flags&PageTypeLeaf != 0:
fmt.Fprintf(writer, "cell%d [ shape=none label=<<table border=\"0\" cellspacing=\"0\">\n", pgno)
fmt.Fprintf(writer, "<tr><td border=\"1\">CELL</td></tr>\n")
fmt.Fprintf(writer, "<tr><td border=\"1\">CELL (%d) </td></tr>\n", pgno)
links := make([]string, 0)
for i := 0; i < cellN; i++ {
cell := readLeafCell(b, i)
switch cell.Type {
@ -41,12 +29,19 @@ func dotCell(b []byte, parent string, writer io.Writer) {
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"green\"><font color=\"white\">[%d]: key=%d type=array n=%d</font></td></tr>\n", i, cell.Key, cell.N)
case ContainerTypeRLE:
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"blue\"><font color=\"white\">[%d]: key=%d type=rle n=%d</font></td></tr>\n", i, cell.Key, cell.N)
case ContainerTypeBitmap:
bpn := toPgno(cell.Data)
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"red\" port=\"%d\"><font color=\"white\">[%d]: key=%d type=bitmap n=%d </font></td></tr>\n", bpn, i, cell.Key, cell.N)
links = append(links, fmt.Sprintf("bitmap%d[label=\"bitmap (%d)\"]\n cell%d:%d -> bitmap%d\n", bpn, bpn, pgno, i, bpn))
default:
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"red\">[%d]: key=%d type=unknown<%d> n=%d</td></tr>\n", i, cell.Key, cell.Type, cell.N)
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"yellow\">[%d]: key=%d type=unknown<%d> n=%d</td></tr>\n", i, cell.Key, cell.Type, cell.N)
}
}
fmt.Fprintf(writer, "</table>>]\n")
fmt.Fprintf(writer, "%s -> cell%d\n", parent, pgno)
for _, link := range links {
fmt.Fprintf(writer, "%s", link)
}
default:
//should not happen
fmt.Fprintf(writer, "==!PAGE %d flags=%d\n", pgno, flags)
@ -76,7 +71,7 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
}
}
rrdump(tx, readMetaRootRecordPageNo(page), visitor)
Walk(tx, readMetaRootRecordPageNo(page), visitor)
return
}
@ -92,14 +87,12 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
dumpdot(tx, cell.Pgno, p, writer)
} else {
b := fmt.Sprintf("bm%d", cell.Pgno)
fmt.Fprintf(writer, "%s[label=\"BITMAP(%d)\"]\n %s -> %s\n", b, cell.Pgno, p, b)
fmt.Fprintf(writer, "%s[label=\"BITMAP(%d) key=%d \"]\n %s -> %s\n", b, cell.Pgno, cell.Key, p, b)
}
}
case PageTypeLeaf:
p := fmt.Sprintf("leaf%d", pgno)
fmt.Fprintf(writer, "%s[label=\"LEAF(%d)| n=%d\"]\n%s->%s\n", p, pgno, readCellN(page), parent, p)
dotCell(page, p, writer)
default:
panic(err)
}
}

2
rbf/internal_test.go
View file

@ -21,6 +21,6 @@ func TestUsed(t *testing.T) {
t.Skip("This function is always skipped")
dump(nil)
hexdump(nil)
pagedump(nil, "", nil)
pagedumpi(nil, "", nil)
treedump(nil, 0, "", nil)
}

97
rbf/rbf.go
View file

@ -1,17 +1,3 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package rbf implements the roaring b-tree file format.
package rbf
@ -113,13 +99,12 @@ func writeMetaRootRecordPageNo(page []byte, pgno uint32) { binary.BigEndian.PutU
func readMetaFreelistPageNo(page []byte) uint32 { return binary.BigEndian.Uint32(page[24:]) }
func writeMetaFreelistPageNo(page []byte, pgno uint32) { binary.BigEndian.PutUint32(page[24:], pgno) }
// func readMetaChecksum(page []byte) uint32 {
// return binary.BigEndian.Uint32(page[PageSize-4 : PageSize])
// }
// func writeMetaChecksum(page []byte, chksum uint32) {
// binary.BigEndian.PutUint32(page[PageSize-4:PageSize], chksum)
// }
func readMetaChecksum(page []byte) uint32 {
return binary.BigEndian.Uint32(page[PageSize-4 : PageSize])
}
func writeMetaChecksum(page []byte, chksum uint32) {
binary.BigEndian.PutUint32(page[PageSize-4:PageSize], chksum)
}
// Root record page helpers
@ -164,7 +149,6 @@ func readCellN(page []byte) int { return int(binary.BigEndian.Uint16(page[8:
func writeCellN(page []byte, v int) { binary.BigEndian.PutUint16(page[8:10], uint16(v)) }
func readCellOffset(page []byte, i int) int {
assert(i < readCellN(page))
return int(binary.BigEndian.Uint16(page[10+(i*2):]))
}
@ -177,8 +161,11 @@ func dataOffset(n int) int {
}
func IsBitmapHeader(page []byte) bool {
if readFlags(page) != PageTypeBitmapHeader {
return false
}
// TODO(BBJ): Verify checksum.
return readFlags(page) == PageTypeBitmapHeader
return true
}
type RootRecord struct {
@ -262,17 +249,23 @@ type leafCell struct {
N int
Data []byte
}
type leafArgs leafCell
// Size returns the size of the leaf cell, in bytes.
func (c *leafCell) Size() int {
if c.Type == ContainerTypeBitmap {
return PageSize
}
return leafCellHeaderSize + len(c.Data)
}
func (c *leafCell) GetBitmap(tx *Tx) (pgno uint32, bm []uint64, err error) {
pgno = toPgno(c.Data)
page, err := tx.readPage(pgno)
if err != nil {
return 0, nil, err
}
return pgno, toArray64(page), err
}
// Bitmap returns a bitmap representation of the cell data.
func (c *leafCell) Bitmap() []uint64 {
func (c *leafCell) Bitmap(tx *Tx) []uint64 {
switch c.Type {
case ContainerTypeArray:
buf := make([]uint64, PageSize/8)
@ -300,14 +293,15 @@ func (c *leafCell) Bitmap() []uint64 {
}
return buf
case ContainerTypeBitmap:
return toArray64(c.Data)
_, bm, _ := c.GetBitmap(tx)
return bm
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
}
}
// Values returns a slice of 16-bit values from a container.
func (c *leafCell) Values() []uint16 {
func (c *leafCell) Values(tx *Tx) []uint16 {
switch c.Type {
case ContainerTypeArray:
return toArray16(c.Data)
@ -324,8 +318,9 @@ func (c *leafCell) Values() []uint16 {
a = a[:n]
return a
case ContainerTypeBitmap:
a := make([]uint16, 0, ArrayMaxSize)
for i, v := range toArray64(c.Data) {
a := make([]uint16, 0, BitmapN*64)
_, bm, _ := c.GetBitmap(tx)
for i, v := range bm {
for j := uint(0); j < 64; j++ {
if v&(1<<j) != 0 {
a = append(a, (uint16(i)*64)+uint16(j))
@ -333,6 +328,8 @@ func (c *leafCell) Values() []uint16 {
}
}
return a
case ContainerTypeNone:
return []uint16{}
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
}
@ -380,17 +377,15 @@ func readLeafCell(page []byte, i int) leafCell {
cell.Data = buf[16 : 16+(cell.N*2)]
case ContainerTypeRLE:
cell.Data = buf[16 : 16+(cell.N*4)]
case ContainerTypeBitmap:
cell.Data = buf[16 : 16+4]
default:
}
return cell
}
func readLeafCells(page []byte, isBitmap bool, buf []leafCell) []leafCell {
if isBitmap {
return []leafCell{{Type: ContainerTypeBitmap, Data: page}}
}
func readLeafCells(page []byte, buf []leafCell) []leafCell {
n := readCellN(page)
cells := buf[:n]
for i := 0; i < n; i++ {
@ -488,9 +483,11 @@ func search(n int, f func(int) int) (index int, exact bool) {
return i, false
}
func itohex(v int) string { return fmt.Sprintf("0x%x", v) }
func hexdump(b []byte) { println(hex.Dump(b)) }
func pagedump(b []byte, indent string, writer io.Writer) {
func pagedumpi(b []byte, indent string, writer io.Writer) {
pgno := readPageNo(b)
if pgno == Magic32() {
fmt.Fprintf(writer, "==META\n")
@ -529,6 +526,24 @@ func pagedump(b []byte, indent string, writer io.Writer) {
}
}
func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) {
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, err := tx.readPage(pgno)
if err != nil {
panic(err)
}
// Read all records on the page.
a, err := readRootRecords(page)
if err != nil {
panic(err)
}
v(pgno, a)
// Read next overflow page number.
pgno = readRootRecordOverflowPgno(page)
}
}
// treedump recursively writes the tree representation starting from a given page to STDERR.
func treedump(tx *Tx, pgno uint32, indent string, writer io.Writer) {
page, err := tx.readPage(pgno)
@ -569,7 +584,7 @@ func treedump(tx *Tx, pgno uint32, indent string, writer io.Writer) {
}
case PageTypeLeaf:
fmt.Fprintf(writer, "%s LEAF(%d) n=%d\n", fmtindent(indent), pgno, readCellN(page))
pagedump(page, fmtindent(" "+indent), writer)
pagedumpi(page, fmtindent(" "+indent), writer)
default:
panic(err)
}
@ -613,6 +628,12 @@ func RowValues(b []uint64) []uint64 {
return a
}
func onpanic(fn func()) {
if r := recover(); r != nil {
fn()
}
}
func assert(condition bool) {
if !condition {
panic("assertion failed")

45
rbf/tx.go
View file

@ -1,17 +1,3 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
@ -50,8 +36,14 @@ func (tx *Tx) Commit() error {
tx.db.pageMap = tx.pageMap
}
if err := tx.db.checkpoint(); err != nil {
return err
}
// Disconnect transaction from DB.
return tx.db.removeTx(tx)
tx.db.removeTx(tx)
return nil
}
func (tx *Tx) Rollback() error {
@ -70,8 +62,14 @@ func (tx *Tx) Rollback() error {
}
}
if err := tx.db.checkpoint(); err != nil {
return err
}
// Disconnect transaction from DB.
return tx.db.removeTx(tx)
tx.db.removeTx(tx)
return nil
}
// Root returns the root page number for a bitmap. Returns 0 if the bitmap does not exist.
@ -246,9 +244,7 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
return err
}
if err := tx.deallocate(pgno); err != nil {
return err
}
tx.deallocate(pgno)
pgno = readRootRecordOverflowPgno(page)
}
@ -410,7 +406,14 @@ func (tx *Tx) checkPageAllocations() error {
if isInuse && isFree {
return fmt.Errorf("page in-use & free: pgno=%d", pgno)
} else if !isInuse && !isFree {
return fmt.Errorf("page not in-use & not free: pgno=%d", pgno)
page, _ := tx.readPage(pgno)
flags := readFlags(page)
if flags == PageTypeBranch || flags == PageTypeLeaf {
return fmt.Errorf("page not in-use & not free: pgno=%d", pgno)
}
//assuming its a bitmap so its ok TODO ben?
return nil
}
}
@ -436,7 +439,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
}
cell := c.cell()
for _, v := range cell.Values() {
for _, v := range cell.Values(tx) {
pgno := uint32((cell.Key << 16) & uint64(v))
m[pgno] = struct{}{}
}

11
roaring/roaring.go
View file

@ -2589,7 +2589,7 @@ func (itr *Iterator) Seek(seek uint64) {
return
}
j, contains := binSearchRuns(lb, itr.c.runs())
j, contains := BinSearchRuns(lb, itr.c.runs())
if contains {
itr.j = j
itr.k = int32(lb) - int32(itr.c.runs()[j].Start) - 1
@ -3163,7 +3163,7 @@ func (c *Container) bitmapContains(v uint16) bool {
// binSearchRuns returns the index of the run containing v, and true, when v is contained;
// or the index of the next run starting after v, and false, when v is not contained.
func binSearchRuns(v uint16, a []Interval16) (int32, bool) {
func BinSearchRuns(v uint16, a []Interval16) (int32, bool) {
i := int32(sort.Search(len(a),
func(i int) bool { return a[i].Last >= v }))
if i < int32(len(a)) {
@ -3176,7 +3176,7 @@ func binSearchRuns(v uint16, a []Interval16) (int32, bool) {
// runContains determines if v is in the container assuming c is a run
// container.
func (c *Container) runContains(v uint16) bool {
_, found := binSearchRuns(v, c.runs())
_, found := BinSearchRuns(v, c.runs())
return found
}
@ -3239,7 +3239,7 @@ func (c *Container) bitmapRemove(v uint16) (*Container, bool) {
// runRemove removes v from a run container, and returns true if v was removed.
func (c *Container) runRemove(v uint16) (*Container, bool) {
runs := c.runs()
i, contains := binSearchRuns(v, runs)
i, contains := BinSearchRuns(v, runs)
if !contains {
return c, false
}
@ -6783,3 +6783,6 @@ func Optimize(c *Container) {
func Union(a, b *Container) *Container {
return union(a, b)
}
func Difference(a, b *Container) *Container {
return difference(a, b)
}