mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
rbf: use a red-black tree to manage the root records list
- 40% faster on ingest_test when putting 10K roots/containers. - 15% fewer bytes allocated total - clarifying renames leafCell.N -> ElemN, allocate -> allocatePgno, deallocate -> freePgno
This commit is contained in:
Jason E. Aten
parent
cb73c71e02
commit
eae82b72c2
11 changed files with 388 additions and 112 deletions
1
go.mod
1
go.mod
|
|
@ -14,6 +14,7 @@ require (
|
|||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
|
||||
github.com/glycerine/lmdb-go v1.9.34
|
||||
github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
github.com/gogo/protobuf v1.2.1
|
||||
github.com/golang/protobuf v1.3.3
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -56,6 +56,8 @@ github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06A
|
|||
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
|
||||
github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE=
|
||||
github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
|
||||
github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809 h1:wBr8MeUUS+Xi4oweFspffWBlDw8s1rGmRBwM4fUjxrc=
|
||||
github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809/go.mod h1:tf1G9WLJXoNEQ5TWYvCSkqsOepuCNCJebECwJ/B/64I=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
|
|
|
|||
1
rbf.go
1
rbf.go
|
|
@ -460,7 +460,6 @@ func (tx *RBFTx) UseRowCache() bool {
|
|||
|
||||
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
|
||||
func rbfName(index, field, view string, shard uint64) string {
|
||||
//return fmt.Sprintf("%s\x00%s\x00%s\x00%d", index, field, view, shard)
|
||||
return string(txkey.Prefix(index, field, view, shard))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
|
|||
if i > 0 && runs[i-1].Last == v-1 {
|
||||
runs[i-1].Last = iv.Last
|
||||
runs = append(runs[:i], runs[i+1:]...)
|
||||
//TODO check if to big
|
||||
//TODO check if too big
|
||||
return runs, true
|
||||
}
|
||||
// just before an interval
|
||||
|
|
@ -84,6 +84,8 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
|
|||
}
|
||||
return runs, true
|
||||
}
|
||||
|
||||
// checkRun is only called by Cursor.Add()
|
||||
func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
|
||||
if len(runs) >= RLEMaxSize {
|
||||
//convertToBitmap
|
||||
|
|
@ -118,14 +120,15 @@ func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
|
|||
words[i] = ^uint64(0)
|
||||
}
|
||||
}
|
||||
// TODO: take this out once we know bitN matches
|
||||
n := uint64(0)
|
||||
for _, v := range bitmap {
|
||||
n += popcount(v)
|
||||
}
|
||||
|
||||
return leafCell{Key: key, N: int(n), BitN: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
|
||||
return leafCell{Key: key, BitN: int(bitN), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
|
||||
}
|
||||
return leafCell{Key: key, N: len(runs), BitN: int(bitN + 1), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
|
||||
return leafCell{Key: key, ElemN: len(runs), BitN: int(bitN), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
|
||||
}
|
||||
|
||||
// Add sets a bit on the underlying bitmap.
|
||||
|
|
@ -136,7 +139,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
|
|||
if exact, err := c.Seek(hi); err != nil {
|
||||
return false, err
|
||||
} else if !exact {
|
||||
return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, BitN: 1, Data: fromArray16([]uint16{lo})})
|
||||
return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, ElemN: 1, BitN: 1, Data: fromArray16([]uint16{lo})})
|
||||
}
|
||||
|
||||
// If the container exists and bit is not set then update the page.
|
||||
|
|
@ -155,7 +158,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
|
|||
copy(other, a[:i])
|
||||
other[i] = lo
|
||||
copy(other[i+1:], a[i:])
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)})
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, ElemN: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)})
|
||||
|
||||
case ContainerTypeRLE:
|
||||
runs := toInterval16(cell.Data)
|
||||
|
|
@ -163,7 +166,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
|
|||
copy(c.rle[:], runs)
|
||||
run, added := runAdd(c.rle[:len(runs)], lo)
|
||||
if added {
|
||||
leaf := checkRun(run, cell.BitN, cell.Key)
|
||||
leaf := checkRun(run, cell.BitN+1, cell.Key)
|
||||
return true, c.putLeafCell(leaf)
|
||||
}
|
||||
return false, nil
|
||||
|
|
@ -219,10 +222,10 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
|
|||
}
|
||||
|
||||
// Copy container data and remove new value.
|
||||
other := make([]uint16, len(a)-1)
|
||||
other := c.array[:len(a)-1]
|
||||
copy(other[:i], a[:i])
|
||||
copy(other[i:], a[i+1:])
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)})
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, ElemN: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)})
|
||||
|
||||
case ContainerTypeRLE:
|
||||
r := toInterval16(cell.Data)
|
||||
|
|
@ -230,6 +233,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
|
|||
if !contains {
|
||||
return false, nil
|
||||
}
|
||||
// INVAR: lo is in run[i]
|
||||
copy(c.rle[:], r)
|
||||
runs := c.rle[:len(r)]
|
||||
|
||||
|
|
@ -240,16 +244,22 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
|
|||
} else if lo == c.rle[i].Start {
|
||||
runs[i].Start++
|
||||
} else if lo > runs[i].Start {
|
||||
// INVAR: Start < lo < Last.
|
||||
// We remove lo, so split into two runs:
|
||||
last := runs[i].Last
|
||||
runs[i].Last = lo - 1
|
||||
// INVAR: runs[:i] is correct, but still need to insert the new interval at i+1.
|
||||
runs = append(runs, roaring.Interval16{})
|
||||
// copy the tail first
|
||||
copy(runs[i+2:], runs[i+1:])
|
||||
// overwrite with the new interval.
|
||||
runs[i+1] = roaring.Interval16{Start: lo + 1, Last: last}
|
||||
}
|
||||
if len(runs) == 0 {
|
||||
return true, c.deleteLeafCell(cell.Key)
|
||||
}
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeRLE, N: len(runs), Data: fromInterval16(runs)})
|
||||
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeRLE, ElemN: len(runs), BitN: cell.BitN - 1, Data: fromInterval16(runs)})
|
||||
|
||||
case ContainerTypeBitmapPtr:
|
||||
pgno, bm, err := c.tx.leafCellBitmap(toPgno(cell.Data))
|
||||
if err != nil {
|
||||
|
|
@ -322,8 +332,7 @@ func toPgno(val []byte) uint32 {
|
|||
return binary.LittleEndian.Uint32(val)
|
||||
}
|
||||
func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
||||
leafPage := c.leafPage
|
||||
|
||||
leafPage := c.leafPage // the last read leaf page
|
||||
cells := readLeafCells(leafPage, c.leafCells[:])
|
||||
elem := &c.stack.elems[c.stack.index]
|
||||
cell := in
|
||||
|
|
@ -331,7 +340,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
//new cell
|
||||
if in.Type == ContainerTypeBitmap {
|
||||
//allocated bitmap()
|
||||
bitmapPgno, err := c.tx.allocate()
|
||||
bitmapPgno, err := c.tx.allocatePgno()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -346,7 +355,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
if in.Type == ContainerTypeBitmap {
|
||||
cell = cells[elem.index]
|
||||
if cell.Type != ContainerTypeBitmapPtr {
|
||||
bitmapPgno, err := c.tx.allocate()
|
||||
bitmapPgno, err := c.tx.allocatePgno()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cursor.putLeafCell")
|
||||
}
|
||||
|
|
@ -356,7 +365,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
}
|
||||
}
|
||||
|
||||
if in.Type == ContainerTypeArray && in.N > ArrayMaxSize {
|
||||
if in.Type == ContainerTypeArray && in.ElemN > ArrayMaxSize {
|
||||
//convert to bitmap
|
||||
in.Type = ContainerTypeBitmap
|
||||
a := make([]uint64, PageSize/8)
|
||||
|
|
@ -365,7 +374,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
}
|
||||
in.Data = fromArray64(a)
|
||||
cell.Type = ContainerTypeBitmapPtr
|
||||
bitmapPgno, err := c.tx.allocate()
|
||||
bitmapPgno, err := c.tx.allocatePgno()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -393,7 +402,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
|
|||
if i == 0 && !newRoot {
|
||||
parent.Pgno = origPgno
|
||||
} else {
|
||||
if parent.Pgno, err = c.tx.allocate(); err != nil {
|
||||
if parent.Pgno, err = c.tx.allocatePgno(); err != nil {
|
||||
return fmt.Errorf("cannot allocate leaf: %w", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -450,14 +459,14 @@ func (c *Cursor) deleteLeafCell(key uint64) (err error) {
|
|||
oldPageKey := cells[0].Key
|
||||
cell := c.cell()
|
||||
if cell.Type == ContainerTypeBitmapPtr {
|
||||
if err := c.tx.deallocate(toPgno(cell.Data)); err != nil {
|
||||
if err := c.tx.freePgno(toPgno(cell.Data)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// If no more cells exist and we have a parent, remove from parent.
|
||||
if c.stack.index > 0 && len(cells) == 1 {
|
||||
if err := c.tx.deallocate(elem.pgno); err != nil {
|
||||
if err := c.tx.freePgno(elem.pgno); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.deleteBranchCell(c.stack.index-1, cells[0].Key)
|
||||
|
|
@ -527,7 +536,7 @@ func (c *Cursor) putBranchCells(stackIndex int, newCells []branchCell) (err erro
|
|||
if i == 0 && !newRoot {
|
||||
parent.Pgno = origPgno
|
||||
} else {
|
||||
if parent.Pgno, err = c.tx.allocate(); err != nil {
|
||||
if parent.Pgno, err = c.tx.allocatePgno(); err != nil {
|
||||
return fmt.Errorf("cannot allocate branch: %w", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -634,7 +643,7 @@ func (c *Cursor) deleteBranchCell(stackIndex int, key uint64) (err error) {
|
|||
copy(buf, target)
|
||||
writePageNo(buf[:], elem.pgno)
|
||||
|
||||
if err := c.tx.deallocate(cells[0].Pgno); err != nil {
|
||||
if err := c.tx.freePgno(cells[0].Pgno); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.tx.writePage(buf[:])
|
||||
|
|
@ -694,7 +703,7 @@ func splitLeafCells(cells []leafCell) [][]leafCell {
|
|||
// half a page then create a new group of cells.
|
||||
if cellN != 0 && (dataOffset(cellN+1)+dataSize+sz) > (PageSize*60)/100 {
|
||||
slices, dataSize = append(slices, nil), 0
|
||||
} else if cellN != 0 && cell.Type == ContainerTypeArray && cell.N > ArrayMaxSize {
|
||||
} else if cellN != 0 && cell.Type == ContainerTypeArray && cell.ElemN > ArrayMaxSize {
|
||||
slices, dataSize = append(slices, nil), 0
|
||||
sz = PageSize
|
||||
}
|
||||
|
|
@ -1116,7 +1125,6 @@ func (c *Cursor) goNextPage() error {
|
|||
|
||||
func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
|
||||
result.Key = key
|
||||
result.N = int(c.N())
|
||||
result.BitN = int(c.N())
|
||||
result.Type = ContainerTypeNone
|
||||
if c.N() == 0 {
|
||||
|
|
@ -1129,14 +1137,17 @@ func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
|
|||
roaring.ConvertArrayToBitmap(c)
|
||||
result.Type = ContainerTypeBitmap
|
||||
result.Data = fromArray64(roaring.AsBitmap(c))
|
||||
// result.ElemN is 0 or undefined for bitmap
|
||||
return
|
||||
}
|
||||
result.Type = ContainerTypeArray
|
||||
result.Data = fromArray16(a)
|
||||
result.ElemN = int(c.N())
|
||||
return
|
||||
case 2: //bitmap
|
||||
result.Type = ContainerTypeBitmap
|
||||
result.Data = fromArray64(roaring.AsBitmap(c))
|
||||
// result.ElemN is 0 or undefined for bitmap
|
||||
return
|
||||
case 3: //run
|
||||
r := roaring.AsRuns(c)
|
||||
|
|
@ -1144,9 +1155,11 @@ func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
|
|||
roaring.ConvertRunToBitmap(c)
|
||||
result.Type = ContainerTypeBitmap
|
||||
result.Data = fromArray64(roaring.AsBitmap(c))
|
||||
// result.ElemN is 0 or undefined for bitmap
|
||||
|
||||
return
|
||||
}
|
||||
result.N = len(r) //note RBF N is number of containers
|
||||
result.ElemN = len(r) // ElemN is the number of runs
|
||||
result.Type = ContainerTypeRLE
|
||||
result.Data = fromInterval16(r)
|
||||
return
|
||||
|
|
@ -1186,7 +1199,7 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) {
|
|||
for itr.Next() {
|
||||
hi, cont := itr.Value()
|
||||
leaf := ConvertToLeafArgs(hi, cont)
|
||||
if leaf.N == 0 {
|
||||
if leaf.BitN == 0 {
|
||||
continue
|
||||
}
|
||||
// Move cursor to the key of the container.
|
||||
|
|
|
|||
|
|
@ -72,13 +72,10 @@ func (c *Cursor) Rows() ([]uint64, error) {
|
|||
return rows, err
|
||||
}
|
||||
func (tx *Tx) FieldViews() []string {
|
||||
r, _ := tx.RootRecords()
|
||||
res := make([]string, len(r))
|
||||
for i := range r {
|
||||
res[i] = r[i].Name
|
||||
}
|
||||
return res
|
||||
rr, _ := tx.RootRecords()
|
||||
return rr.sliceOfNames()
|
||||
}
|
||||
|
||||
func (c *Cursor) DumpKeys() {
|
||||
if err := c.First(); err != nil {
|
||||
//ignoring errors for this debug function
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ type DB struct {
|
|||
|
||||
data []byte // database mmap
|
||||
file *os.File // database file descriptor
|
||||
rootRecords []*RootRecord // cached root records
|
||||
rootRecords *rr // cached root records
|
||||
pageMap *immutable.Map // pgno-to-WALID mapping
|
||||
txs map[*Tx]struct{} // active transactions
|
||||
opened bool // true if open
|
||||
|
|
@ -329,7 +329,9 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) {
|
|||
}
|
||||
// Loop over each bitmap and attempt to move to the first cell.
|
||||
// If we can move to a cell then we have at least one record.
|
||||
for _, record := range records {
|
||||
|
||||
for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() {
|
||||
record := it.Item().(RootRecord)
|
||||
// Fetch cursor for bitmap.
|
||||
cur, err := tx.Cursor(record.Name)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -38,15 +38,15 @@ func dotCell(b []byte, parent string, writer io.Writer) {
|
|||
switch cell.Type {
|
||||
case ContainerTypeArray:
|
||||
//fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data))
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"green\"><font color=\"white\">[%d]: key=%d type=array n=%d</font></td></tr>\n", i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"green\"><font color=\"white\">[%d]: key=%d type=array n=%d</font></td></tr>\n", i, cell.Key, cell.BitN)
|
||||
case ContainerTypeRLE:
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"blue\"><font color=\"white\">[%d]: key=%d type=rle n=%d</font></td></tr>\n", i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"blue\"><font color=\"white\">[%d]: key=%d type=rle n=%d</font></td></tr>\n", i, cell.Key, cell.BitN)
|
||||
case ContainerTypeBitmapPtr:
|
||||
bpn := toPgno(cell.Data)
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"red\" port=\"%d\"><font color=\"white\">[%d]: key=%d type=bitmap n=%d </font></td></tr>\n", bpn, i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"red\" port=\"%d\"><font color=\"white\">[%d]: key=%d type=bitmap n=%d </font></td></tr>\n", bpn, i, cell.Key, cell.BitN)
|
||||
links = append(links, fmt.Sprintf("bitmap%d[label=\"bitmap (%d)\"]\n cell%d:%d -> bitmap%d\n", bpn, bpn, pgno, i, bpn))
|
||||
default:
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"yellow\">[%d]: key=%d type=unknown<%d> n=%d</td></tr>\n", i, cell.Key, cell.Type, cell.N)
|
||||
fmt.Fprintf(writer, "<tr><td border=\"1\" bgcolor=\"yellow\">[%d]: key=%d type=unknown<%d> n=%d</td></tr>\n", i, cell.Key, cell.Type, cell.BitN)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(writer, "</table>>]\n")
|
||||
|
|
|
|||
173
rbf/ingest_test.go
Normal file
173
rbf/ingest_test.go
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rbf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strings"
|
||||
"testing"
|
||||
//"time"
|
||||
|
||||
"github.com/pilosa/pilosa/v2/rbf/cfg"
|
||||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
"github.com/pilosa/pilosa/v2/txkey"
|
||||
)
|
||||
|
||||
func rbfName(index, field, view string, shard uint64) string {
|
||||
return string(txkey.Prefix(index, field, view, shard))
|
||||
}
|
||||
|
||||
func TestIngest_lots_of_views(t *testing.T) {
|
||||
|
||||
// skip unless studying perf because is long (15-30 seconds)
|
||||
//return
|
||||
|
||||
var m0, m1 runtime.MemStats
|
||||
runtime.ReadMemStats(&m0)
|
||||
vv("m0.TotalAlloc = %v", m0.TotalAlloc)
|
||||
defer func() {
|
||||
runtime.ReadMemStats(&m1)
|
||||
vv("m1.TotalAlloc = %v", m1.TotalAlloc)
|
||||
}()
|
||||
// rbtree uses 15% memory and needs half the ingest time
|
||||
// for our 10K view ingest.
|
||||
//
|
||||
// previous master with slice copying instead of rbtree:
|
||||
/*
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:14:09.778839Z m0.TotalAlloc = 728408
|
||||
ingest_test.go:144 2020-11-13T03:14:37.492104Z m1.TotalAlloc = 41,816,617,216
|
||||
--- PASS: TestIngest_lots_of_views (27.71s)
|
||||
*/
|
||||
// lots_views with rbtree
|
||||
/*
|
||||
=== RUN TestIngest_lots_of_views
|
||||
ingest_test.go:141 2020-11-13T03:11:01.540076Z m0.TotalAlloc = 726072
|
||||
ingest_test.go:144 2020-11-13T03:11:15.003591Z m1.TotalAlloc = 35,510,273,184
|
||||
--- PASS: TestIngest_lots_of_views (13.46s)
|
||||
*/
|
||||
|
||||
path, err := ioutil.TempDir("", "rbf_ingest_lots_of_views")
|
||||
panicOn(err)
|
||||
defer os.Remove(path)
|
||||
|
||||
cfg := cfg.NewDefaultConfig()
|
||||
db := NewDB(path, cfg)
|
||||
panicOn(db.Open())
|
||||
|
||||
// setup profiling
|
||||
if false {
|
||||
profile, err := os.Create("./rbf_ingest_put_ct.cpu")
|
||||
panicOn(err)
|
||||
_ = pprof.StartCPUProfile(profile)
|
||||
defer func() {
|
||||
pprof.StopCPUProfile()
|
||||
profile.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
// put containers
|
||||
tx, err := db.Begin(true)
|
||||
panicOn(err)
|
||||
|
||||
index := "i"
|
||||
field := "f"
|
||||
var view string // set below in the loop.
|
||||
|
||||
// put a raw-bitmap container to many views.
|
||||
bits := []uint16{}
|
||||
for i := 0; i < 1<<16; i++ {
|
||||
//for i := 0; i < 100; i++ {
|
||||
if i%2 == 0 {
|
||||
bits = append(bits, uint16(i))
|
||||
}
|
||||
}
|
||||
ct := roaring.NewContainerArray(bits)
|
||||
|
||||
nCt := 10000
|
||||
ckey := uint64(0)
|
||||
shard := ckey / ShardWidth
|
||||
|
||||
for i := 0; i < nCt; i++ {
|
||||
view = fmt.Sprintf("view_%v", i)
|
||||
name := rbfName(index, field, view, shard)
|
||||
err = tx.PutContainer(name, ckey, ct)
|
||||
panicOn(err)
|
||||
ct2, err := tx.Container(name, ckey)
|
||||
panicOn(err)
|
||||
if err := ct2.BitwiseCompare(ct); err != nil {
|
||||
panic("ct2 != ct")
|
||||
}
|
||||
|
||||
// write .dot of it...
|
||||
if false { //ckey == nCt-1 {
|
||||
c, err := tx.cursor(name)
|
||||
if err == ErrBitmapNotFound {
|
||||
panic("not found")
|
||||
} else if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
c.Dump("one.bitmap.dot.dump")
|
||||
}
|
||||
}
|
||||
|
||||
panicOn(tx.Commit())
|
||||
|
||||
sz, err := DiskUse(path, "")
|
||||
panicOn(err)
|
||||
_ = sz
|
||||
vv("sz in bytes= %v", sz)
|
||||
|
||||
db.Close()
|
||||
}
|
||||
|
||||
// func (s *rr) last() (r *RootRecord) {
|
||||
// it := s.tree.Max()
|
||||
// if it == s.tree.NegativeLimit() {
|
||||
// return nil
|
||||
// }
|
||||
// rec := it.Item().(RootRecord)
|
||||
// r = &rec
|
||||
// return
|
||||
// }
|
||||
|
||||
// var _ = (&rr{}).last // happy linter
|
||||
|
||||
func DiskUse(root string, requiredSuffix string) (tot int, err error) {
|
||||
if !DirExists(root) {
|
||||
return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
|
||||
}
|
||||
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if info == nil {
|
||||
panic(fmt.Sprintf("info was nil for path = '%v'", path))
|
||||
}
|
||||
if info.IsDir() {
|
||||
// skip the size of directories themselves, only summing files.
|
||||
} else {
|
||||
sz := info.Size()
|
||||
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
|
||||
tot += int(sz)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
43
rbf/rbf.go
43
rbf/rbf.go
|
|
@ -25,6 +25,7 @@ import (
|
|||
"os"
|
||||
"unsafe"
|
||||
|
||||
"github.com/glycerine/rbtree"
|
||||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
"github.com/pilosa/pilosa/v2/shardwidth"
|
||||
)
|
||||
|
|
@ -166,16 +167,22 @@ func readRootRecords(page []byte) (records []*RootRecord, err error) {
|
|||
}
|
||||
}
|
||||
|
||||
func writeRootRecords(page []byte, records []*RootRecord) (remaining []*RootRecord, err error) {
|
||||
// writeRootRecords is only called by tx.go Tx.writeRootRecordPages().
|
||||
// We can return io.ErrShortBuffer in err. If we still have records
|
||||
// to write that don't fit on page, remain will point to the next
|
||||
// record that hasn't yet been written.
|
||||
func writeRootRecords(page []byte, recit, limit rbtree.Iterator) (remain rbtree.Iterator, err error) {
|
||||
data := page[rootRecordPageHeaderSize:]
|
||||
for i, rec := range records {
|
||||
if data, err = WriteRootRecord(data, rec); err == io.ErrShortBuffer {
|
||||
return records[i:], nil
|
||||
} else if err != nil {
|
||||
return records[i:], err
|
||||
|
||||
for recit != limit {
|
||||
rec := recit.Item().(RootRecord)
|
||||
data, err = WriteRootRecord(data, &rec)
|
||||
if err != nil {
|
||||
return recit, err
|
||||
}
|
||||
recit = recit.Next()
|
||||
}
|
||||
return nil, nil
|
||||
return recit, nil
|
||||
}
|
||||
|
||||
// Branch & leaf page helpers
|
||||
|
|
@ -284,11 +291,11 @@ type leafCell struct {
|
|||
Key uint64
|
||||
Type int // container type
|
||||
|
||||
// N is the number of "things" in Data:
|
||||
// ElemN is the number of "things" in Data:
|
||||
// for an array container the number of integers in the array.
|
||||
// for an RLE, number of intervals.
|
||||
// etc.
|
||||
N int
|
||||
// ElemN is undefined or 0 for ContainerTypeBitmap
|
||||
ElemN int
|
||||
|
||||
BitN int
|
||||
Data []byte
|
||||
|
|
@ -471,14 +478,14 @@ func readLeafCell(page []byte, i int) leafCell {
|
|||
var cell leafCell
|
||||
cell.Key = *(*uint64)(unsafe.Pointer(&buf[0]))
|
||||
cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8])))
|
||||
cell.N = int(*(*uint16)(unsafe.Pointer(&buf[12])))
|
||||
cell.ElemN = int(*(*uint16)(unsafe.Pointer(&buf[12])))
|
||||
cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14])))
|
||||
|
||||
switch cell.Type {
|
||||
case ContainerTypeArray:
|
||||
cell.Data = buf[16 : 16+(cell.N*2)]
|
||||
cell.Data = buf[16 : 16+(cell.ElemN*2)]
|
||||
case ContainerTypeRLE:
|
||||
cell.Data = buf[16 : 16+(cell.N*4)]
|
||||
cell.Data = buf[16 : 16+(cell.ElemN*4)]
|
||||
case ContainerTypeBitmapPtr:
|
||||
cell.Data = buf[16 : 16+4]
|
||||
default:
|
||||
|
|
@ -509,7 +516,7 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) {
|
|||
writeCellOffset(page, i, offset)
|
||||
*(*uint64)(unsafe.Pointer(&page[offset])) = cell.Key
|
||||
*(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type)
|
||||
*(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N)
|
||||
*(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.ElemN)
|
||||
*(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN)
|
||||
assert(offset+16+len(cell.Data) <= PageSize) // leaf cell write extends beyond page
|
||||
copy(page[offset+16:], cell.Data)
|
||||
|
|
@ -611,13 +618,13 @@ func Pagedump(b []byte, indent string, writer io.Writer) {
|
|||
switch cell.Type {
|
||||
case ContainerTypeArray:
|
||||
//fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data))
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=array n=%d \n", indent, i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=array BitN=%d \n", indent, i, cell.Key, cell.BitN)
|
||||
case ContainerTypeRLE:
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=rle n=%d\n", indent, i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=rle BitN=%d\n", indent, i, cell.Key, cell.BitN)
|
||||
case ContainerTypeBitmapPtr:
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=bitmap n=%d\n", indent, i, cell.Key, cell.N)
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=bitmap BitN=%d\n", indent, i, cell.Key, cell.BitN)
|
||||
default:
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=unknown<%d> n=%d\n", indent, i, cell.Key, cell.Type, cell.N)
|
||||
fmt.Fprintf(writer, "%s[%d]: key=%d type=unknown<%d> BitN=%d\n", indent, i, cell.Key, cell.Type, cell.BitN)
|
||||
}
|
||||
}
|
||||
case flags&PageTypeBranch != 0:
|
||||
|
|
|
|||
191
rbf/tx.go
191
rbf/tx.go
|
|
@ -23,6 +23,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/benbjohnson/immutable"
|
||||
"github.com/glycerine/rbtree"
|
||||
"github.com/pilosa/pilosa/v2/hash"
|
||||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
"github.com/pilosa/pilosa/v2/txkey"
|
||||
|
|
@ -37,7 +38,7 @@ type Tx struct {
|
|||
meta [PageSize]byte // copy of current meta page
|
||||
walID int64 // max WAL ID at start of tx
|
||||
walPageN int // wal page count
|
||||
rootRecords []*RootRecord // read-only cache of root records
|
||||
rootRecords *rr // read-only cache of root records
|
||||
|
||||
// pageMap holds WAL pages that have not yet been transferred
|
||||
// into the database pages. So it can be empty, if the whole previous
|
||||
|
|
@ -58,6 +59,70 @@ type Tx struct {
|
|||
DeleteEmptyContainer bool
|
||||
}
|
||||
|
||||
type rr struct {
|
||||
tree rbtree.Tree
|
||||
}
|
||||
|
||||
func newRR() *rr {
|
||||
return &rr{
|
||||
tree: *rbtree.NewTree(
|
||||
func(a, b rbtree.Item) int {
|
||||
an := a.(RootRecord).Name
|
||||
bn := b.(RootRecord).Name
|
||||
if an == bn {
|
||||
return 0
|
||||
}
|
||||
if an < bn {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rr) size() int {
|
||||
return s.tree.Len()
|
||||
}
|
||||
|
||||
func (s *rr) add(r RootRecord) {
|
||||
s.tree.Insert(r)
|
||||
}
|
||||
|
||||
func (s *rr) addAll(recs []*RootRecord) {
|
||||
for _, r := range recs {
|
||||
s.add(*r)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rr) remove(it rbtree.Iterator) {
|
||||
s.tree.DeleteWithIterator(it)
|
||||
}
|
||||
|
||||
func iterToRootRecord(it rbtree.Iterator) RootRecord {
|
||||
return it.Item().(RootRecord)
|
||||
}
|
||||
|
||||
func (s *rr) sliceOfNames() (res []string) {
|
||||
res = make([]string, s.size())
|
||||
|
||||
i := 0
|
||||
for it := s.tree.Min(); it != s.tree.Limit(); it = it.Next() {
|
||||
res[i] = it.Item().(RootRecord).Name
|
||||
i++
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *rr) find(name string) (r RootRecord, iter rbtree.Iterator, exact bool) {
|
||||
iter = s.tree.FindGE(RootRecord{Name: name})
|
||||
if iter.Limit() {
|
||||
return
|
||||
}
|
||||
r = iter.Item().(RootRecord)
|
||||
exact = (r.Name == name)
|
||||
return
|
||||
}
|
||||
|
||||
func (tx *Tx) DBPath() string {
|
||||
return tx.db.Path
|
||||
}
|
||||
|
|
@ -145,11 +210,11 @@ func (tx *Tx) root(name string) (uint32, error) {
|
|||
return 0, err
|
||||
}
|
||||
|
||||
i := sort.Search(len(records), func(i int) bool { return records[i].Name >= name })
|
||||
if i >= len(records) || records[i].Name != name {
|
||||
_, it, exactHit := records.find(name)
|
||||
if !exactHit {
|
||||
return 0, ErrBitmapNotFound
|
||||
}
|
||||
return records[i].Pgno, nil
|
||||
return iterToRootRecord(it).Pgno, nil
|
||||
}
|
||||
|
||||
// BitmapNames returns a list of all bitmap names.
|
||||
|
|
@ -166,13 +231,7 @@ func (tx *Tx) BitmapNames() ([]string, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert to a list of strings.
|
||||
names := make([]string, len(records))
|
||||
for i := range records {
|
||||
names[i] = records[i].Name
|
||||
}
|
||||
return names, nil
|
||||
return records.sliceOfNames(), nil
|
||||
}
|
||||
|
||||
// CreateBitmap creates a new empty bitmap with the given name.
|
||||
|
|
@ -199,14 +258,14 @@ func (tx *Tx) createBitmap(name string) error {
|
|||
}
|
||||
|
||||
// Find btree by name. Exit if already exists.
|
||||
index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name })
|
||||
if index < len(records) && records[index].Name == name {
|
||||
_, _, exact := records.find(name)
|
||||
if exact {
|
||||
return ErrBitmapExists
|
||||
}
|
||||
//fmt.Println("CREATE BITMAP", name, index)
|
||||
|
||||
// Allocate new root page.
|
||||
pgno, err := tx.allocate()
|
||||
pgno, err := tx.allocatePgno()
|
||||
//fmt.Println("CREATE BITMAP @ PGNO", pgno)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -222,9 +281,7 @@ func (tx *Tx) createBitmap(name string) error {
|
|||
}
|
||||
|
||||
// Insert into correct index.
|
||||
records = append(records, nil)
|
||||
copy(records[index+1:], records[index:])
|
||||
records[index] = &RootRecord{Name: name, Pgno: pgno}
|
||||
records.add(RootRecord{Name: name, Pgno: pgno})
|
||||
if err := tx.writeRootRecordPages(records); err != nil {
|
||||
return fmt.Errorf("write bitmaps: %w", err)
|
||||
}
|
||||
|
|
@ -278,19 +335,21 @@ func (tx *Tx) DeleteBitmap(name string) error {
|
|||
}
|
||||
|
||||
// Find btree by name. Exit if it doesn't exist.
|
||||
index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name })
|
||||
if index >= len(records) || records[index].Name != name {
|
||||
record, it, ok := records.find(name)
|
||||
|
||||
if !ok {
|
||||
return fmt.Errorf("bitmap does not exist: %q", name)
|
||||
}
|
||||
pgno := records[index].Pgno
|
||||
pgno := record.Pgno
|
||||
|
||||
// Deallocate all pages in the tree.
|
||||
if err := tx.deallocateTree(pgno); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records.remove(it)
|
||||
|
||||
// Delete from record list & rewrite record pages.
|
||||
records = append(records[:index], records[index+1:]...)
|
||||
if err := tx.writeRootRecordPages(records); err != nil {
|
||||
return fmt.Errorf("write bitmaps: %w", err)
|
||||
}
|
||||
|
|
@ -314,9 +373,8 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < len(records); i++ {
|
||||
record := records[i]
|
||||
for it := records.tree.Min(); it != records.tree.Limit(); {
|
||||
record := it.Item().(RootRecord)
|
||||
|
||||
// Skip bitmaps without matching prefix.
|
||||
if !strings.HasPrefix(record.Name, prefix) {
|
||||
|
|
@ -328,9 +386,11 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Delete from record list.
|
||||
records = append(records[:i], records[i+1:]...)
|
||||
i--
|
||||
// as long we've advanced it past delme, we can
|
||||
// delete delme without affecting it.
|
||||
delme := it
|
||||
it = it.Next()
|
||||
records.remove(delme)
|
||||
}
|
||||
|
||||
// Rewrite record pages.
|
||||
|
|
@ -362,13 +422,16 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error {
|
|||
}
|
||||
|
||||
// Find btree by name. Exit if it doesn't exist.
|
||||
index := sort.Search(len(records), func(i int) bool { return records[i].Name >= oldname })
|
||||
if index >= len(records) || records[index].Name != oldname {
|
||||
rec, it, exactHit := records.find(oldname)
|
||||
if !exactHit {
|
||||
return fmt.Errorf("bitmap does not exist: %q", oldname)
|
||||
}
|
||||
|
||||
// Update record name & rewrite record pages.
|
||||
records[index].Name = newname
|
||||
rec2 := rec
|
||||
rec2.Name = newname
|
||||
records.remove(it)
|
||||
records.add(rec2)
|
||||
if err := tx.writeRootRecordPages(records); err != nil {
|
||||
return fmt.Errorf("write bitmaps: %w", err)
|
||||
}
|
||||
|
|
@ -377,12 +440,12 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error {
|
|||
}
|
||||
|
||||
// RootRecords returns a list of root records.
|
||||
func (tx *Tx) RootRecords() (rr []*RootRecord, err error) {
|
||||
func (tx *Tx) RootRecords() (records *rr, err error) {
|
||||
if tx.rootRecords != nil {
|
||||
return tx.rootRecords, nil
|
||||
}
|
||||
|
||||
var records []*RootRecord
|
||||
records = newRR()
|
||||
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
|
||||
page, err := tx.readPage(pgno)
|
||||
if err != nil {
|
||||
|
|
@ -394,7 +457,7 @@ func (tx *Tx) RootRecords() (rr []*RootRecord, err error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, a...)
|
||||
records.addAll(a)
|
||||
|
||||
// Read next overflow page number.
|
||||
pgno = WalkRootRecordPages(page)
|
||||
|
|
@ -406,7 +469,7 @@ func (tx *Tx) RootRecords() (rr []*RootRecord, err error) {
|
|||
}
|
||||
|
||||
// writeRootRecordPages writes a list of root record pages.
|
||||
func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
|
||||
func (tx *Tx) writeRootRecordPages(records *rr) (err error) {
|
||||
|
||||
// Release all existing root record pages.
|
||||
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
|
||||
|
|
@ -415,7 +478,7 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
err = tx.deallocate(pgno)
|
||||
err = tx.freePgno(pgno)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -423,37 +486,44 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
|
|||
}
|
||||
|
||||
// Exit early if no records exist.
|
||||
if len(records) == 0 {
|
||||
if records.size() == 0 {
|
||||
writeMetaRootRecordPageNo(tx.meta[:], 0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure records are in sorted order.
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name })
|
||||
|
||||
// Allocate initial root record page.
|
||||
pgno, err := tx.allocate()
|
||||
pgno, err := tx.allocatePgno()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
writeMetaRootRecordPageNo(tx.meta[:], pgno)
|
||||
|
||||
// Write new root record pages.
|
||||
for i := 0; len(records) != 0; i++ {
|
||||
limit := records.tree.Limit()
|
||||
it := records.tree.Min()
|
||||
for it != limit {
|
||||
|
||||
// Initialize page & write as many records as will fit.
|
||||
page := make([]byte, PageSize)
|
||||
writePageNo(page, pgno)
|
||||
writeFlags(page, PageTypeRootRecord)
|
||||
if records, err = writeRootRecords(page, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Allocate next and write overflow if we have remaining records.
|
||||
if len(records) != 0 {
|
||||
if pgno, err = tx.allocate(); err != nil {
|
||||
// writeRootRecords does it = it.Next() for us after
|
||||
// each successful write to the page.
|
||||
it, err = writeRootRecords(page, it, limit)
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
// nothing to do, all the rest of the records fit on the page.
|
||||
|
||||
case io.ErrShortBuffer:
|
||||
// Allocate next pgno and write overflow if we have remaining records.
|
||||
if pgno, err = tx.allocatePgno(); err != nil {
|
||||
return err
|
||||
}
|
||||
writeRootRecordOverflowPgno(page, pgno)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
// Write page to disk.
|
||||
|
|
@ -798,7 +868,10 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
|
|||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
for _, record := range records {
|
||||
|
||||
for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() {
|
||||
record := it.Item().(RootRecord)
|
||||
|
||||
if err := tx.walkTree(record.Pgno, 0, func(pgno, parent, typ uint32) error {
|
||||
m[pgno] = struct{}{}
|
||||
return nil
|
||||
|
|
@ -848,10 +921,10 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er
|
|||
}
|
||||
}
|
||||
|
||||
// allocate returns a page number for a new available page. This page may be
|
||||
// allocatePgno returns a page number for a new available page. This page may be
|
||||
// pulled from the free list or, if no free pages are available, it will be
|
||||
// created by extending the file size.
|
||||
func (tx *Tx) allocate() (uint32, error) {
|
||||
func (tx *Tx) allocatePgno() (uint32, error) {
|
||||
// Attempt to find page in freelist.
|
||||
pgno, err := tx.nextFreelistPageNo()
|
||||
|
||||
|
|
@ -863,7 +936,7 @@ func (tx *Tx) allocate() (uint32, error) {
|
|||
if changed, err := c.Remove(uint64(pgno)); err != nil {
|
||||
return 0, err
|
||||
} else if !changed {
|
||||
panic(fmt.Sprintf("tx.Tx.allocate(): double alloc: %d", pgno))
|
||||
panic(fmt.Sprintf("tx.Tx.allocatePgno(): double alloc: %d", pgno))
|
||||
}
|
||||
return pgno, nil
|
||||
}
|
||||
|
|
@ -891,14 +964,14 @@ func (tx *Tx) nextFreelistPageNo() (uint32, error) {
|
|||
}
|
||||
|
||||
// deallocate releases a page number to the freelist.
|
||||
func (tx *Tx) deallocate(pgno uint32) error {
|
||||
func (tx *Tx) freePgno(pgno uint32) error {
|
||||
c := Cursor{tx: tx}
|
||||
c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])}
|
||||
|
||||
if changed, err := c.Add(uint64(pgno)); err != nil {
|
||||
return err
|
||||
} else if !changed {
|
||||
panic(fmt.Sprintf("rbf.Tx.deallocate(): double free: %d", pgno))
|
||||
panic(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", pgno))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -921,7 +994,7 @@ func (tx *Tx) deallocateTree(pgno uint32) error {
|
|||
return nil
|
||||
|
||||
case PageTypeLeaf:
|
||||
return tx.deallocate(pgno)
|
||||
return tx.freePgno(pgno)
|
||||
default:
|
||||
return fmt.Errorf("rbf.Tx.deallocateTree(): invalid page type: pgno=%d type=%d", pgno, typ)
|
||||
}
|
||||
|
|
@ -1333,7 +1406,10 @@ func (tx *Tx) DumpString(short bool, shard uint64) (r string) {
|
|||
records, err := tx.RootRecords()
|
||||
panicOn(err)
|
||||
n := 0
|
||||
for _, rr := range records {
|
||||
|
||||
for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() {
|
||||
rr := it.Item().(RootRecord)
|
||||
|
||||
c, err := tx.cursor(rr.Name)
|
||||
panicOn(err)
|
||||
err = c.First() // First will rewind to beginning.
|
||||
|
|
@ -1730,7 +1806,10 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, record := range records {
|
||||
|
||||
for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() {
|
||||
record := it.Item().(RootRecord)
|
||||
|
||||
if err := tx.walkPageInfo(infos, record.Pgno, record.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3270,7 +3270,10 @@ func (c *Container) bitmapContains(v uint16) bool {
|
|||
// or the index of the next run starting after v, and false, when v is not contained.
|
||||
func BinSearchRuns(v uint16, a []Interval16) (int32, bool) {
|
||||
i := int32(sort.Search(len(a),
|
||||
func(i int) bool { return a[i].Last >= v }))
|
||||
func(i int) bool {
|
||||
return a[i].Last >= v
|
||||
}))
|
||||
|
||||
if i < int32(len(a)) {
|
||||
return i, (v >= a[i].Start) && (v <= a[i].Last)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue