From eae82b72c2b91c0915cc9c53d195ab245f310ef0 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Tue, 10 Nov 2020 18:55:30 +0000 Subject: [PATCH] rbf: use a red-black tree to manage the root records list - 40% faster on ingest_test when putting 10K roots/containers. - 15% fewer bytes allocated total - clarifying renames leafCell.N -> ElemN, allocate -> allocatePgno, deallocate -> freePgno --- go.mod | 1 + go.sum | 2 + rbf.go | 1 - rbf/cursor.go | 61 +++++++++------ rbf/cursorx.go | 9 +-- rbf/db.go | 6 +- rbf/dot.go | 8 +- rbf/ingest_test.go | 173 ++++++++++++++++++++++++++++++++++++++++ rbf/rbf.go | 43 +++++----- rbf/tx.go | 191 ++++++++++++++++++++++++++++++++------------- roaring/roaring.go | 5 +- 11 files changed, 388 insertions(+), 112 deletions(-) create mode 100644 rbf/ingest_test.go diff --git a/go.mod b/go.mod index bfd870b97..9ccdc224f 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/glycerine/lmdb-go v1.9.34 + github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index ea9c803a4..2cb15dd96 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06A github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE= github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= +github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809 h1:wBr8MeUUS+Xi4oweFspffWBlDw8s1rGmRBwM4fUjxrc= +github.com/glycerine/rbtree v0.0.0-20190406191118-ceb71889d809/go.mod h1:tf1G9WLJXoNEQ5TWYvCSkqsOepuCNCJebECwJ/B/64I= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= diff --git a/rbf.go b/rbf.go index 4bf1be5a7..7191639f4 100644 --- a/rbf.go +++ b/rbf.go @@ -460,7 +460,6 @@ func (tx *RBFTx) UseRowCache() bool { // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. func rbfName(index, field, view string, shard uint64) string { - //return fmt.Sprintf("%s\x00%s\x00%s\x00%d", index, field, view, shard) return string(txkey.Prefix(index, field, view, shard)) } diff --git a/rbf/cursor.go b/rbf/cursor.go index bbf6fbe2b..189038c2e 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -69,7 +69,7 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) { if i > 0 && runs[i-1].Last == v-1 { runs[i-1].Last = iv.Last runs = append(runs[:i], runs[i+1:]...) - //TODO check if to big + //TODO check if too big return runs, true } // just before an interval @@ -84,6 +84,8 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) { } return runs, true } + +// checkRun is only called by Cursor.Add() func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell { if len(runs) >= RLEMaxSize { //convertToBitmap @@ -118,14 +120,15 @@ func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell { words[i] = ^uint64(0) } } + // TODO: take this out once we know bitN matches n := uint64(0) for _, v := range bitmap { n += popcount(v) } - return leafCell{Key: key, N: int(n), BitN: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} + return leafCell{Key: key, BitN: int(bitN), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} } - return leafCell{Key: key, N: len(runs), BitN: int(bitN + 1), Type: ContainerTypeRLE, Data: fromInterval16(runs)} + return leafCell{Key: key, ElemN: len(runs), BitN: int(bitN), Type: ContainerTypeRLE, Data: fromInterval16(runs)} } // Add sets a bit on the underlying bitmap. @@ -136,7 +139,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { if exact, err := c.Seek(hi); err != nil { return false, err } else if !exact { - return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, BitN: 1, Data: fromArray16([]uint16{lo})}) + return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, ElemN: 1, BitN: 1, Data: fromArray16([]uint16{lo})}) } // If the container exists and bit is not set then update the page. @@ -155,7 +158,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { copy(other, a[:i]) other[i] = lo copy(other[i+1:], a[i:]) - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)}) + return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, ElemN: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)}) case ContainerTypeRLE: runs := toInterval16(cell.Data) @@ -163,7 +166,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { copy(c.rle[:], runs) run, added := runAdd(c.rle[:len(runs)], lo) if added { - leaf := checkRun(run, cell.BitN, cell.Key) + leaf := checkRun(run, cell.BitN+1, cell.Key) return true, c.putLeafCell(leaf) } return false, nil @@ -219,10 +222,10 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { } // Copy container data and remove new value. - other := make([]uint16, len(a)-1) + other := c.array[:len(a)-1] copy(other[:i], a[:i]) copy(other[i:], a[i+1:]) - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)}) + return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, ElemN: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)}) case ContainerTypeRLE: r := toInterval16(cell.Data) @@ -230,6 +233,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { if !contains { return false, nil } + // INVAR: lo is in run[i] copy(c.rle[:], r) runs := c.rle[:len(r)] @@ -240,16 +244,22 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { } else if lo == c.rle[i].Start { runs[i].Start++ } else if lo > runs[i].Start { + // INVAR: Start < lo < Last. + // We remove lo, so split into two runs: last := runs[i].Last runs[i].Last = lo - 1 + // INVAR: runs[:i] is correct, but still need to insert the new interval at i+1. runs = append(runs, roaring.Interval16{}) + // copy the tail first copy(runs[i+2:], runs[i+1:]) + // overwrite with the new interval. runs[i+1] = roaring.Interval16{Start: lo + 1, Last: last} } if len(runs) == 0 { return true, c.deleteLeafCell(cell.Key) } - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeRLE, N: len(runs), Data: fromInterval16(runs)}) + return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeRLE, ElemN: len(runs), BitN: cell.BitN - 1, Data: fromInterval16(runs)}) + case ContainerTypeBitmapPtr: pgno, bm, err := c.tx.leafCellBitmap(toPgno(cell.Data)) if err != nil { @@ -322,8 +332,7 @@ func toPgno(val []byte) uint32 { return binary.LittleEndian.Uint32(val) } func (c *Cursor) putLeafCell(in leafCell) (err error) { - leafPage := c.leafPage - + leafPage := c.leafPage // the last read leaf page cells := readLeafCells(leafPage, c.leafCells[:]) elem := &c.stack.elems[c.stack.index] cell := in @@ -331,7 +340,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { //new cell if in.Type == ContainerTypeBitmap { //allocated bitmap() - bitmapPgno, err := c.tx.allocate() + bitmapPgno, err := c.tx.allocatePgno() if err != nil { return err } @@ -346,7 +355,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { if in.Type == ContainerTypeBitmap { cell = cells[elem.index] if cell.Type != ContainerTypeBitmapPtr { - bitmapPgno, err := c.tx.allocate() + bitmapPgno, err := c.tx.allocatePgno() if err != nil { return errors.Wrap(err, "cursor.putLeafCell") } @@ -356,7 +365,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { } } - if in.Type == ContainerTypeArray && in.N > ArrayMaxSize { + if in.Type == ContainerTypeArray && in.ElemN > ArrayMaxSize { //convert to bitmap in.Type = ContainerTypeBitmap a := make([]uint64, PageSize/8) @@ -365,7 +374,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { } in.Data = fromArray64(a) cell.Type = ContainerTypeBitmapPtr - bitmapPgno, err := c.tx.allocate() + bitmapPgno, err := c.tx.allocatePgno() if err != nil { return err } @@ -393,7 +402,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { if i == 0 && !newRoot { parent.Pgno = origPgno } else { - if parent.Pgno, err = c.tx.allocate(); err != nil { + if parent.Pgno, err = c.tx.allocatePgno(); err != nil { return fmt.Errorf("cannot allocate leaf: %w", err) } } @@ -450,14 +459,14 @@ func (c *Cursor) deleteLeafCell(key uint64) (err error) { oldPageKey := cells[0].Key cell := c.cell() if cell.Type == ContainerTypeBitmapPtr { - if err := c.tx.deallocate(toPgno(cell.Data)); err != nil { + if err := c.tx.freePgno(toPgno(cell.Data)); err != nil { return err } } // If no more cells exist and we have a parent, remove from parent. if c.stack.index > 0 && len(cells) == 1 { - if err := c.tx.deallocate(elem.pgno); err != nil { + if err := c.tx.freePgno(elem.pgno); err != nil { return err } return c.deleteBranchCell(c.stack.index-1, cells[0].Key) @@ -527,7 +536,7 @@ func (c *Cursor) putBranchCells(stackIndex int, newCells []branchCell) (err erro if i == 0 && !newRoot { parent.Pgno = origPgno } else { - if parent.Pgno, err = c.tx.allocate(); err != nil { + if parent.Pgno, err = c.tx.allocatePgno(); err != nil { return fmt.Errorf("cannot allocate branch: %w", err) } } @@ -634,7 +643,7 @@ func (c *Cursor) deleteBranchCell(stackIndex int, key uint64) (err error) { copy(buf, target) writePageNo(buf[:], elem.pgno) - if err := c.tx.deallocate(cells[0].Pgno); err != nil { + if err := c.tx.freePgno(cells[0].Pgno); err != nil { return err } return c.tx.writePage(buf[:]) @@ -694,7 +703,7 @@ func splitLeafCells(cells []leafCell) [][]leafCell { // half a page then create a new group of cells. if cellN != 0 && (dataOffset(cellN+1)+dataSize+sz) > (PageSize*60)/100 { slices, dataSize = append(slices, nil), 0 - } else if cellN != 0 && cell.Type == ContainerTypeArray && cell.N > ArrayMaxSize { + } else if cellN != 0 && cell.Type == ContainerTypeArray && cell.ElemN > ArrayMaxSize { slices, dataSize = append(slices, nil), 0 sz = PageSize } @@ -1116,7 +1125,6 @@ func (c *Cursor) goNextPage() error { func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) { result.Key = key - result.N = int(c.N()) result.BitN = int(c.N()) result.Type = ContainerTypeNone if c.N() == 0 { @@ -1129,14 +1137,17 @@ func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) { roaring.ConvertArrayToBitmap(c) result.Type = ContainerTypeBitmap result.Data = fromArray64(roaring.AsBitmap(c)) + // result.ElemN is 0 or undefined for bitmap return } result.Type = ContainerTypeArray result.Data = fromArray16(a) + result.ElemN = int(c.N()) return case 2: //bitmap result.Type = ContainerTypeBitmap result.Data = fromArray64(roaring.AsBitmap(c)) + // result.ElemN is 0 or undefined for bitmap return case 3: //run r := roaring.AsRuns(c) @@ -1144,9 +1155,11 @@ func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) { roaring.ConvertRunToBitmap(c) result.Type = ContainerTypeBitmap result.Data = fromArray64(roaring.AsBitmap(c)) + // result.ElemN is 0 or undefined for bitmap + return } - result.N = len(r) //note RBF N is number of containers + result.ElemN = len(r) // ElemN is the number of runs result.Type = ContainerTypeRLE result.Data = fromInterval16(r) return @@ -1186,7 +1199,7 @@ func (c *Cursor) AddRoaring(bm *roaring.Bitmap) (changed bool, err error) { for itr.Next() { hi, cont := itr.Value() leaf := ConvertToLeafArgs(hi, cont) - if leaf.N == 0 { + if leaf.BitN == 0 { continue } // Move cursor to the key of the container. diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 1521b6edc..35dd7593a 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -72,13 +72,10 @@ func (c *Cursor) Rows() ([]uint64, error) { return rows, err } func (tx *Tx) FieldViews() []string { - r, _ := tx.RootRecords() - res := make([]string, len(r)) - for i := range r { - res[i] = r[i].Name - } - return res + rr, _ := tx.RootRecords() + return rr.sliceOfNames() } + func (c *Cursor) DumpKeys() { if err := c.First(); err != nil { //ignoring errors for this debug function diff --git a/rbf/db.go b/rbf/db.go index a570b6f00..9f70b18d3 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -40,7 +40,7 @@ type DB struct { data []byte // database mmap file *os.File // database file descriptor - rootRecords []*RootRecord // cached root records + rootRecords *rr // cached root records pageMap *immutable.Map // pgno-to-WALID mapping txs map[*Tx]struct{} // active transactions opened bool // true if open @@ -329,7 +329,9 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { } // Loop over each bitmap and attempt to move to the first cell. // If we can move to a cell then we have at least one record. - for _, record := range records { + + for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() { + record := it.Item().(RootRecord) // Fetch cursor for bitmap. cur, err := tx.Cursor(record.Name) if err != nil { diff --git a/rbf/dot.go b/rbf/dot.go index 3ad2f9d57..3a4cd9da7 100644 --- a/rbf/dot.go +++ b/rbf/dot.go @@ -38,15 +38,15 @@ func dotCell(b []byte, parent string, writer io.Writer) { switch cell.Type { case ContainerTypeArray: //fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data)) - fmt.Fprintf(writer, "[%d]: key=%d type=array n=%d\n", i, cell.Key, cell.N) + fmt.Fprintf(writer, "[%d]: key=%d type=array n=%d\n", i, cell.Key, cell.BitN) case ContainerTypeRLE: - fmt.Fprintf(writer, "[%d]: key=%d type=rle n=%d\n", i, cell.Key, cell.N) + fmt.Fprintf(writer, "[%d]: key=%d type=rle n=%d\n", i, cell.Key, cell.BitN) case ContainerTypeBitmapPtr: bpn := toPgno(cell.Data) - fmt.Fprintf(writer, "[%d]: key=%d type=bitmap n=%d \n", bpn, i, cell.Key, cell.N) + fmt.Fprintf(writer, "[%d]: key=%d type=bitmap n=%d \n", bpn, i, cell.Key, cell.BitN) links = append(links, fmt.Sprintf("bitmap%d[label=\"bitmap (%d)\"]\n cell%d:%d -> bitmap%d\n", bpn, bpn, pgno, i, bpn)) default: - fmt.Fprintf(writer, "[%d]: key=%d type=unknown<%d> n=%d\n", i, cell.Key, cell.Type, cell.N) + fmt.Fprintf(writer, "[%d]: key=%d type=unknown<%d> n=%d\n", i, cell.Key, cell.Type, cell.BitN) } } fmt.Fprintf(writer, ">]\n") diff --git a/rbf/ingest_test.go b/rbf/ingest_test.go new file mode 100644 index 000000000..be65d7f3a --- /dev/null +++ b/rbf/ingest_test.go @@ -0,0 +1,173 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rbf + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "strings" + "testing" + //"time" + + "github.com/pilosa/pilosa/v2/rbf/cfg" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/txkey" +) + +func rbfName(index, field, view string, shard uint64) string { + return string(txkey.Prefix(index, field, view, shard)) +} + +func TestIngest_lots_of_views(t *testing.T) { + + // skip unless studying perf because is long (15-30 seconds) + //return + + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + vv("m0.TotalAlloc = %v", m0.TotalAlloc) + defer func() { + runtime.ReadMemStats(&m1) + vv("m1.TotalAlloc = %v", m1.TotalAlloc) + }() + // rbtree uses 15% memory and needs half the ingest time + // for our 10K view ingest. + // + // previous master with slice copying instead of rbtree: + /* + === RUN TestIngest_lots_of_views + ingest_test.go:141 2020-11-13T03:14:09.778839Z m0.TotalAlloc = 728408 + ingest_test.go:144 2020-11-13T03:14:37.492104Z m1.TotalAlloc = 41,816,617,216 + --- PASS: TestIngest_lots_of_views (27.71s) + */ + // lots_views with rbtree + /* + === RUN TestIngest_lots_of_views + ingest_test.go:141 2020-11-13T03:11:01.540076Z m0.TotalAlloc = 726072 + ingest_test.go:144 2020-11-13T03:11:15.003591Z m1.TotalAlloc = 35,510,273,184 + --- PASS: TestIngest_lots_of_views (13.46s) + */ + + path, err := ioutil.TempDir("", "rbf_ingest_lots_of_views") + panicOn(err) + defer os.Remove(path) + + cfg := cfg.NewDefaultConfig() + db := NewDB(path, cfg) + panicOn(db.Open()) + + // setup profiling + if false { + profile, err := os.Create("./rbf_ingest_put_ct.cpu") + panicOn(err) + _ = pprof.StartCPUProfile(profile) + defer func() { + pprof.StopCPUProfile() + profile.Close() + }() + } + + // put containers + tx, err := db.Begin(true) + panicOn(err) + + index := "i" + field := "f" + var view string // set below in the loop. + + // put a raw-bitmap container to many views. + bits := []uint16{} + for i := 0; i < 1<<16; i++ { + //for i := 0; i < 100; i++ { + if i%2 == 0 { + bits = append(bits, uint16(i)) + } + } + ct := roaring.NewContainerArray(bits) + + nCt := 10000 + ckey := uint64(0) + shard := ckey / ShardWidth + + for i := 0; i < nCt; i++ { + view = fmt.Sprintf("view_%v", i) + name := rbfName(index, field, view, shard) + err = tx.PutContainer(name, ckey, ct) + panicOn(err) + ct2, err := tx.Container(name, ckey) + panicOn(err) + if err := ct2.BitwiseCompare(ct); err != nil { + panic("ct2 != ct") + } + + // write .dot of it... + if false { //ckey == nCt-1 { + c, err := tx.cursor(name) + if err == ErrBitmapNotFound { + panic("not found") + } else if err != nil { + panic(err) + } + c.Dump("one.bitmap.dot.dump") + } + } + + panicOn(tx.Commit()) + + sz, err := DiskUse(path, "") + panicOn(err) + _ = sz + vv("sz in bytes= %v", sz) + + db.Close() +} + +// func (s *rr) last() (r *RootRecord) { +// it := s.tree.Max() +// if it == s.tree.NegativeLimit() { +// return nil +// } +// rec := it.Item().(RootRecord) +// r = &rec +// return +// } + +// var _ = (&rr{}).last // happy linter + +func DiskUse(root string, requiredSuffix string) (tot int, err error) { + if !DirExists(root) { + return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) + } + + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if info == nil { + panic(fmt.Sprintf("info was nil for path = '%v'", path)) + } + if info.IsDir() { + // skip the size of directories themselves, only summing files. + } else { + sz := info.Size() + if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { + tot += int(sz) + } + } + return nil + }) + return +} diff --git a/rbf/rbf.go b/rbf/rbf.go index e3a293a26..33312e63f 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -25,6 +25,7 @@ import ( "os" "unsafe" + "github.com/glycerine/rbtree" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" ) @@ -166,16 +167,22 @@ func readRootRecords(page []byte) (records []*RootRecord, err error) { } } -func writeRootRecords(page []byte, records []*RootRecord) (remaining []*RootRecord, err error) { +// writeRootRecords is only called by tx.go Tx.writeRootRecordPages(). +// We can return io.ErrShortBuffer in err. If we still have records +// to write that don't fit on page, remain will point to the next +// record that hasn't yet been written. +func writeRootRecords(page []byte, recit, limit rbtree.Iterator) (remain rbtree.Iterator, err error) { data := page[rootRecordPageHeaderSize:] - for i, rec := range records { - if data, err = WriteRootRecord(data, rec); err == io.ErrShortBuffer { - return records[i:], nil - } else if err != nil { - return records[i:], err + + for recit != limit { + rec := recit.Item().(RootRecord) + data, err = WriteRootRecord(data, &rec) + if err != nil { + return recit, err } + recit = recit.Next() } - return nil, nil + return recit, nil } // Branch & leaf page helpers @@ -284,11 +291,11 @@ type leafCell struct { Key uint64 Type int // container type - // N is the number of "things" in Data: + // ElemN is the number of "things" in Data: // for an array container the number of integers in the array. // for an RLE, number of intervals. - // etc. - N int + // ElemN is undefined or 0 for ContainerTypeBitmap + ElemN int BitN int Data []byte @@ -471,14 +478,14 @@ func readLeafCell(page []byte, i int) leafCell { var cell leafCell cell.Key = *(*uint64)(unsafe.Pointer(&buf[0])) cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8]))) - cell.N = int(*(*uint16)(unsafe.Pointer(&buf[12]))) + cell.ElemN = int(*(*uint16)(unsafe.Pointer(&buf[12]))) cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14]))) switch cell.Type { case ContainerTypeArray: - cell.Data = buf[16 : 16+(cell.N*2)] + cell.Data = buf[16 : 16+(cell.ElemN*2)] case ContainerTypeRLE: - cell.Data = buf[16 : 16+(cell.N*4)] + cell.Data = buf[16 : 16+(cell.ElemN*4)] case ContainerTypeBitmapPtr: cell.Data = buf[16 : 16+4] default: @@ -509,7 +516,7 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) { writeCellOffset(page, i, offset) *(*uint64)(unsafe.Pointer(&page[offset])) = cell.Key *(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type) - *(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N) + *(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.ElemN) *(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN) assert(offset+16+len(cell.Data) <= PageSize) // leaf cell write extends beyond page copy(page[offset+16:], cell.Data) @@ -611,13 +618,13 @@ func Pagedump(b []byte, indent string, writer io.Writer) { switch cell.Type { case ContainerTypeArray: //fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data)) - fmt.Fprintf(writer, "%s[%d]: key=%d type=array n=%d \n", indent, i, cell.Key, cell.N) + fmt.Fprintf(writer, "%s[%d]: key=%d type=array BitN=%d \n", indent, i, cell.Key, cell.BitN) case ContainerTypeRLE: - fmt.Fprintf(writer, "%s[%d]: key=%d type=rle n=%d\n", indent, i, cell.Key, cell.N) + fmt.Fprintf(writer, "%s[%d]: key=%d type=rle BitN=%d\n", indent, i, cell.Key, cell.BitN) case ContainerTypeBitmapPtr: - fmt.Fprintf(writer, "%s[%d]: key=%d type=bitmap n=%d\n", indent, i, cell.Key, cell.N) + fmt.Fprintf(writer, "%s[%d]: key=%d type=bitmap BitN=%d\n", indent, i, cell.Key, cell.BitN) default: - fmt.Fprintf(writer, "%s[%d]: key=%d type=unknown<%d> n=%d\n", indent, i, cell.Key, cell.Type, cell.N) + fmt.Fprintf(writer, "%s[%d]: key=%d type=unknown<%d> BitN=%d\n", indent, i, cell.Key, cell.Type, cell.BitN) } } case flags&PageTypeBranch != 0: diff --git a/rbf/tx.go b/rbf/tx.go index 72d041704..13f7690ea 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/benbjohnson/immutable" + "github.com/glycerine/rbtree" "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/txkey" @@ -37,7 +38,7 @@ type Tx struct { meta [PageSize]byte // copy of current meta page walID int64 // max WAL ID at start of tx walPageN int // wal page count - rootRecords []*RootRecord // read-only cache of root records + rootRecords *rr // read-only cache of root records // pageMap holds WAL pages that have not yet been transferred // into the database pages. So it can be empty, if the whole previous @@ -58,6 +59,70 @@ type Tx struct { DeleteEmptyContainer bool } +type rr struct { + tree rbtree.Tree +} + +func newRR() *rr { + return &rr{ + tree: *rbtree.NewTree( + func(a, b rbtree.Item) int { + an := a.(RootRecord).Name + bn := b.(RootRecord).Name + if an == bn { + return 0 + } + if an < bn { + return -1 + } + return 1 + }), + } +} + +func (s *rr) size() int { + return s.tree.Len() +} + +func (s *rr) add(r RootRecord) { + s.tree.Insert(r) +} + +func (s *rr) addAll(recs []*RootRecord) { + for _, r := range recs { + s.add(*r) + } +} + +func (s *rr) remove(it rbtree.Iterator) { + s.tree.DeleteWithIterator(it) +} + +func iterToRootRecord(it rbtree.Iterator) RootRecord { + return it.Item().(RootRecord) +} + +func (s *rr) sliceOfNames() (res []string) { + res = make([]string, s.size()) + + i := 0 + for it := s.tree.Min(); it != s.tree.Limit(); it = it.Next() { + res[i] = it.Item().(RootRecord).Name + i++ + } + return +} + +func (s *rr) find(name string) (r RootRecord, iter rbtree.Iterator, exact bool) { + iter = s.tree.FindGE(RootRecord{Name: name}) + if iter.Limit() { + return + } + r = iter.Item().(RootRecord) + exact = (r.Name == name) + return +} + func (tx *Tx) DBPath() string { return tx.db.Path } @@ -145,11 +210,11 @@ func (tx *Tx) root(name string) (uint32, error) { return 0, err } - i := sort.Search(len(records), func(i int) bool { return records[i].Name >= name }) - if i >= len(records) || records[i].Name != name { + _, it, exactHit := records.find(name) + if !exactHit { return 0, ErrBitmapNotFound } - return records[i].Pgno, nil + return iterToRootRecord(it).Pgno, nil } // BitmapNames returns a list of all bitmap names. @@ -166,13 +231,7 @@ func (tx *Tx) BitmapNames() ([]string, error) { if err != nil { return nil, err } - - // Convert to a list of strings. - names := make([]string, len(records)) - for i := range records { - names[i] = records[i].Name - } - return names, nil + return records.sliceOfNames(), nil } // CreateBitmap creates a new empty bitmap with the given name. @@ -199,14 +258,14 @@ func (tx *Tx) createBitmap(name string) error { } // Find btree by name. Exit if already exists. - index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name }) - if index < len(records) && records[index].Name == name { + _, _, exact := records.find(name) + if exact { return ErrBitmapExists } //fmt.Println("CREATE BITMAP", name, index) // Allocate new root page. - pgno, err := tx.allocate() + pgno, err := tx.allocatePgno() //fmt.Println("CREATE BITMAP @ PGNO", pgno) if err != nil { return err @@ -222,9 +281,7 @@ func (tx *Tx) createBitmap(name string) error { } // Insert into correct index. - records = append(records, nil) - copy(records[index+1:], records[index:]) - records[index] = &RootRecord{Name: name, Pgno: pgno} + records.add(RootRecord{Name: name, Pgno: pgno}) if err := tx.writeRootRecordPages(records); err != nil { return fmt.Errorf("write bitmaps: %w", err) } @@ -278,19 +335,21 @@ func (tx *Tx) DeleteBitmap(name string) error { } // Find btree by name. Exit if it doesn't exist. - index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name }) - if index >= len(records) || records[index].Name != name { + record, it, ok := records.find(name) + + if !ok { return fmt.Errorf("bitmap does not exist: %q", name) } - pgno := records[index].Pgno + pgno := record.Pgno // Deallocate all pages in the tree. if err := tx.deallocateTree(pgno); err != nil { return err } + records.remove(it) + // Delete from record list & rewrite record pages. - records = append(records[:index], records[index+1:]...) if err := tx.writeRootRecordPages(records); err != nil { return fmt.Errorf("write bitmaps: %w", err) } @@ -314,9 +373,8 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { if err != nil { return err } - - for i := 0; i < len(records); i++ { - record := records[i] + for it := records.tree.Min(); it != records.tree.Limit(); { + record := it.Item().(RootRecord) // Skip bitmaps without matching prefix. if !strings.HasPrefix(record.Name, prefix) { @@ -328,9 +386,11 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { return err } - // Delete from record list. - records = append(records[:i], records[i+1:]...) - i-- + // as long we've advanced it past delme, we can + // delete delme without affecting it. + delme := it + it = it.Next() + records.remove(delme) } // Rewrite record pages. @@ -362,13 +422,16 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error { } // Find btree by name. Exit if it doesn't exist. - index := sort.Search(len(records), func(i int) bool { return records[i].Name >= oldname }) - if index >= len(records) || records[index].Name != oldname { + rec, it, exactHit := records.find(oldname) + if !exactHit { return fmt.Errorf("bitmap does not exist: %q", oldname) } // Update record name & rewrite record pages. - records[index].Name = newname + rec2 := rec + rec2.Name = newname + records.remove(it) + records.add(rec2) if err := tx.writeRootRecordPages(records); err != nil { return fmt.Errorf("write bitmaps: %w", err) } @@ -377,12 +440,12 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error { } // RootRecords returns a list of root records. -func (tx *Tx) RootRecords() (rr []*RootRecord, err error) { +func (tx *Tx) RootRecords() (records *rr, err error) { if tx.rootRecords != nil { return tx.rootRecords, nil } - var records []*RootRecord + records = newRR() for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; { page, err := tx.readPage(pgno) if err != nil { @@ -394,7 +457,7 @@ func (tx *Tx) RootRecords() (rr []*RootRecord, err error) { if err != nil { return nil, err } - records = append(records, a...) + records.addAll(a) // Read next overflow page number. pgno = WalkRootRecordPages(page) @@ -406,7 +469,7 @@ func (tx *Tx) RootRecords() (rr []*RootRecord, err error) { } // writeRootRecordPages writes a list of root record pages. -func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) { +func (tx *Tx) writeRootRecordPages(records *rr) (err error) { // Release all existing root record pages. for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; { @@ -415,7 +478,7 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) { return err } - err = tx.deallocate(pgno) + err = tx.freePgno(pgno) if err != nil { return err } @@ -423,37 +486,44 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) { } // Exit early if no records exist. - if len(records) == 0 { + if records.size() == 0 { writeMetaRootRecordPageNo(tx.meta[:], 0) return nil } - // Ensure records are in sorted order. - sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name }) - // Allocate initial root record page. - pgno, err := tx.allocate() + pgno, err := tx.allocatePgno() if err != nil { return err } writeMetaRootRecordPageNo(tx.meta[:], pgno) // Write new root record pages. - for i := 0; len(records) != 0; i++ { + limit := records.tree.Limit() + it := records.tree.Min() + for it != limit { + // Initialize page & write as many records as will fit. page := make([]byte, PageSize) writePageNo(page, pgno) writeFlags(page, PageTypeRootRecord) - if records, err = writeRootRecords(page, records); err != nil { - return err - } - // Allocate next and write overflow if we have remaining records. - if len(records) != 0 { - if pgno, err = tx.allocate(); err != nil { + // writeRootRecords does it = it.Next() for us after + // each successful write to the page. + it, err = writeRootRecords(page, it, limit) + + switch err { + case nil: + // nothing to do, all the rest of the records fit on the page. + + case io.ErrShortBuffer: + // Allocate next pgno and write overflow if we have remaining records. + if pgno, err = tx.allocatePgno(); err != nil { return err } writeRootRecordOverflowPgno(page, pgno) + default: + return err } // Write page to disk. @@ -798,7 +868,10 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { if err != nil { return m, err } - for _, record := range records { + + for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() { + record := it.Item().(RootRecord) + if err := tx.walkTree(record.Pgno, 0, func(pgno, parent, typ uint32) error { m[pgno] = struct{}{} return nil @@ -848,10 +921,10 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } -// allocate returns a page number for a new available page. This page may be +// allocatePgno returns a page number for a new available page. This page may be // pulled from the free list or, if no free pages are available, it will be // created by extending the file size. -func (tx *Tx) allocate() (uint32, error) { +func (tx *Tx) allocatePgno() (uint32, error) { // Attempt to find page in freelist. pgno, err := tx.nextFreelistPageNo() @@ -863,7 +936,7 @@ func (tx *Tx) allocate() (uint32, error) { if changed, err := c.Remove(uint64(pgno)); err != nil { return 0, err } else if !changed { - panic(fmt.Sprintf("tx.Tx.allocate(): double alloc: %d", pgno)) + panic(fmt.Sprintf("tx.Tx.allocatePgno(): double alloc: %d", pgno)) } return pgno, nil } @@ -891,14 +964,14 @@ func (tx *Tx) nextFreelistPageNo() (uint32, error) { } // deallocate releases a page number to the freelist. -func (tx *Tx) deallocate(pgno uint32) error { +func (tx *Tx) freePgno(pgno uint32) error { c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} if changed, err := c.Add(uint64(pgno)); err != nil { return err } else if !changed { - panic(fmt.Sprintf("rbf.Tx.deallocate(): double free: %d", pgno)) + panic(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", pgno)) } return nil } @@ -921,7 +994,7 @@ func (tx *Tx) deallocateTree(pgno uint32) error { return nil case PageTypeLeaf: - return tx.deallocate(pgno) + return tx.freePgno(pgno) default: return fmt.Errorf("rbf.Tx.deallocateTree(): invalid page type: pgno=%d type=%d", pgno, typ) } @@ -1333,7 +1406,10 @@ func (tx *Tx) DumpString(short bool, shard uint64) (r string) { records, err := tx.RootRecords() panicOn(err) n := 0 - for _, rr := range records { + + for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() { + rr := it.Item().(RootRecord) + c, err := tx.cursor(rr.Name) panicOn(err) err = c.First() // First will rewind to beginning. @@ -1730,7 +1806,10 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { if err != nil { return nil, err } - for _, record := range records { + + for it := records.tree.Min(); it != records.tree.Limit(); it = it.Next() { + record := it.Item().(RootRecord) + if err := tx.walkPageInfo(infos, record.Pgno, record.Name); err != nil { return nil, err } diff --git a/roaring/roaring.go b/roaring/roaring.go index 526aeb7bc..793c7b93c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3270,7 +3270,10 @@ func (c *Container) bitmapContains(v uint16) bool { // or the index of the next run starting after v, and false, when v is not contained. func BinSearchRuns(v uint16, a []Interval16) (int32, bool) { i := int32(sort.Search(len(a), - func(i int) bool { return a[i].Last >= v })) + func(i int) bool { + return a[i].Last >= v + })) + if i < int32(len(a)) { return i, (v >= a[i].Start) && (v <= a[i].Last) }