scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

upgrade to immutable 0.4.0(generics) (#2317)

Browse source
This commit is contained in:
tgruben 2022-11-28 12:12:03 -06:00 • committed by GitHub
parent df829c592f
commit a98b9144cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 50 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -14,7 +14,7 @@ require (
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2
github.com/aws/aws-sdk-go v1.42.39
github.com/beevik/ntp v0.3.0
github.com/benbjohnson/immutable v0.3.0
github.com/benbjohnson/immutable v0.4.0
github.com/cespare/xxhash v1.1.0
github.com/chzyer/readline v1.5.1
github.com/confluentinc/confluent-kafka-go v1.9.1

4
go.sum
View file

@ -139,8 +139,8 @@ github.com/aws/smithy-go v1.0.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB
github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw=
github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA=
github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

12
rbf/cursorx.go
View file

@ -15,10 +15,10 @@ import (
// probably should just implement the container interface
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
shardVsContainerExponent := uint(4) //needs constant exported from roaring package
shardVsContainerExponent := uint(4) // needs constant exported from roaring package
rows := make([]uint64, 0)
if err := c.First(); err != nil {
if err == io.EOF { //root leaf with no elements
if err == io.EOF { // root leaf with no elements
return rows, nil
}
return nil, errors.Wrap(err, "rows")
@ -52,15 +52,15 @@ func (tx *Tx) FieldViews() []string {
records, _ := tx.RootRecords()
a := make([]string, 0, records.Len())
for itr := records.Iterator(); !itr.Done(); {
name, _ := itr.Next()
a = append(a, name.(string))
name, _, _ := itr.Next()
a = append(a, name)
}
return a
}
func (c *Cursor) DumpKeys() {
if err := c.First(); err != nil {
//ignoring errors for this debug function
// ignoring errors for this debug function
return
}
for {
@ -77,6 +77,7 @@ func (c *Cursor) DumpKeys() {
fmt.Println("key", cell.Key)
}
}
func (c *Cursor) DumpStack() {
fmt.Println("STACK")
for i := c.stack.top; i >= 0; i-- {
@ -84,6 +85,7 @@ func (c *Cursor) DumpStack() {
}
fmt.Println()
}
func (c *Cursor) Dump(name string) {
writer, _ := os.Create(name)
defer writer.Close()

18
rbf/db.go
View file

@ -44,13 +44,13 @@ type txWaiter struct {
type DB struct {
cfg rbfcfg.Config
data []byte // database mmap
file *os.File // database file descriptor
rootRecords *immutable.SortedMap // cached root records
pageMap *PageMap // pgno-to-WALID mapping
txs map[*Tx]struct{} // active transactions
opened bool // true if open
logger logger.Logger // for diagnostics from async things
data []byte // database mmap
file *os.File // database file descriptor
rootRecords *immutable.SortedMap[string, uint32] // cached root records
pageMap *PageMap // pgno-to-WALID mapping
txs map[*Tx]struct{} // active transactions
opened bool // true if open
logger logger.Logger // for diagnostics from async things
wal []byte // wal mmap
walFile *os.File // wal file descriptor
@ -514,9 +514,9 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) {
// If we can move to a cell then we have at least one record.
for itr := records.Iterator(); !itr.Done(); {
name, _ := itr.Next()
name, _, _ := itr.Next()
// Fetch cursor for bitmap.
cur, err := tx.Cursor(name.(string))
cur, err := tx.Cursor(name)
if err != nil {
return false, err
}

9
rbf/rbf.go
View file

@ -149,6 +149,7 @@ func writeMetaChecksum(page []byte, chksum uint32) {
// Root record page helpers
func WalkRootRecordPages(page []byte) uint32 { return binary.BigEndian.Uint32(page[8:]) }
func writeRootRecordOverflowPgno(page []byte, pgno uint32) {
binary.BigEndian.PutUint32(page[8:], pgno)
}
@ -169,13 +170,13 @@ func readRootRecords(page []byte) (records []*RootRecord, err error) {
// We can return io.ErrShortBuffer in err. If we still have records
// to write that don't fit on page, remain will point to the next
// record that hasn't yet been written.
func writeRootRecords(page []byte, itr *immutable.SortedMapIterator) (err error) {
func writeRootRecords(page []byte, itr *immutable.SortedMapIterator[string, uint32]) (err error) {
data := page[rootRecordPageHeaderSize:]
for !itr.Done() {
name, pgno := itr.Next()
name, pgno, _ := itr.Next()
data, err = WriteRootRecord(data, &RootRecord{Name: name.(string), Pgno: pgno.(uint32)})
data, err = WriteRootRecord(data, &RootRecord{Name: name, Pgno: pgno})
if err != nil {
itr.Seek(name)
return err
@ -685,7 +686,7 @@ func Pagedump(b []byte, indent string, writer io.Writer) {
cell := readLeafCell(b, i)
switch cell.Type {
case ContainerTypeArray:
//fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data))
// fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data))
fmt.Fprintf(writer, "%s[%d]: key=%d type=array BitN=%d \n", indent, i, cell.Key, cell.BitN)
case ContainerTypeRLE:
fmt.Fprintf(writer, "%s[%d]: key=%d type=rle BitN=%d\n", indent, i, cell.Key, cell.BitN)

53
rbf/tx.go
View file

@ -24,11 +24,11 @@ var _ = txkey.ToString
// view at the point-in-time they are started.
type Tx struct {
mu sync.RWMutex
db *DB // parent db
meta [PageSize]byte // copy of current meta page
walID int64 // max WAL ID at start of tx
walPageN int // wal page count
rootRecords *immutable.SortedMap // read-only cache of root records
db *DB // parent db
meta [PageSize]byte // copy of current meta page
walID int64 // max WAL ID at start of tx
walPageN int // wal page count
rootRecords *immutable.SortedMap[string, uint32] // read-only cache of root records
// pageMap holds WAL pages that have not yet been transferred
// into the database pages. So it can be empty, if the whole previous
@ -249,7 +249,7 @@ func (tx *Tx) root(name string) (uint32, error) {
if !ok {
return 0, ErrBitmapNotFound
}
return pgno.(uint32), nil
return pgno, nil
}
// BitmapNames returns a list of all bitmap names.
@ -269,8 +269,8 @@ func (tx *Tx) BitmapNames() ([]string, error) {
a := make([]string, 0, records.Len())
for itr := records.Iterator(); !itr.Done(); {
k, _ := itr.Next()
a = append(a, k.(string))
k, _, _ := itr.Next()
a = append(a, k)
}
return a, nil
}
@ -393,7 +393,7 @@ func (tx *Tx) DeleteBitmap(name string) error {
}
// Deallocate all pages in the tree.
if err := tx.deallocateTree(pgno.(uint32)); err != nil {
if err := tx.deallocateTree(pgno); err != nil {
return err
}
@ -424,18 +424,18 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
}
for itr := records.Iterator(); !itr.Done(); {
name, pgno := itr.Next()
name, pgno, _ := itr.Next()
// Skip bitmaps without matching prefix.
if !strings.HasPrefix(name.(string), prefix) {
if !strings.HasPrefix(name, prefix) {
continue
}
// Deallocate all pages in the tree.
if err := tx.deallocateTree(pgno.(uint32)); err != nil {
if err := tx.deallocateTree(pgno); err != nil {
return err
}
records = records.Delete(name.(string))
records = records.Delete(name)
}
// Rewrite record pages.
@ -483,12 +483,12 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error {
}
// RootRecords returns a list of root records.
func (tx *Tx) RootRecords() (records *immutable.SortedMap, err error) {
func (tx *Tx) RootRecords() (records *immutable.SortedMap[string, uint32], err error) {
if tx.rootRecords != nil {
return tx.rootRecords, nil
}
records = immutable.NewSortedMap(nil)
records = immutable.NewSortedMap[string, uint32](nil)
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, _, err := tx.readPage(pgno)
if err != nil {
@ -514,8 +514,7 @@ func (tx *Tx) RootRecords() (records *immutable.SortedMap, err error) {
}
// writeRootRecordPages writes a list of root record pages.
func (tx *Tx) writeRootRecordPages(records *immutable.SortedMap) (err error) {
func (tx *Tx) writeRootRecordPages(records *immutable.SortedMap[string, uint32]) (err error) {
// Release all existing root record pages.
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, _, err := tx.readPage(pgno)
@ -998,9 +997,9 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
errorList.Append(err)
} else {
for itr := records.Iterator(); !itr.Done(); {
_, pgno := itr.Next()
_, pgno, _ := itr.Next()
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
if err := tx.walkTree(pgno, 0, func(pgno, parent, typ uint32, err error) error {
if err != nil {
errorList.Append(err)
}
@ -1028,15 +1027,15 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
// Loop over each bitmap in the database.
for itr := records.Iterator(); !itr.Done(); {
name, pgno := itr.Next()
name, pgno, _ := itr.Next()
// Skip over any bitmaps that don't have a matching prefix.
if !strings.HasPrefix(name.(string), prefix) {
if !strings.HasPrefix(name, prefix) {
continue
}
// Traverse the bitmap's b-tree and count the bytes for each page.
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
if err := tx.walkTree(pgno, 0, func(pgno, parent, typ uint32, err error) error {
n += PageSize
return err
}); err != nil {
@ -1801,13 +1800,13 @@ func (si *emptyContainerIterator) Close() {}
func (si *emptyContainerIterator) Next() bool {
return false
}
func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
vprint.PanicOn("emptyContainerIterator never has any Values")
return 0, nil
}
func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
// begin write boilerplate
if tx.db == nil {
err = ErrTxClosed
@ -2134,9 +2133,9 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
errorList.Append(err)
} else {
for itr := records.Iterator(); !itr.Done(); {
name, pgno := itr.Next()
name, pgno, _ := itr.Next()
if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
if err := tx.walkPageInfo(infos, pgno, name); err != nil {
errorList.Append(err)
}
}
@ -2246,8 +2245,8 @@ func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) {
}
it := records.Iterator()
for !it.Done() {
k, _ := it.Next()
root := k.(string)
k, _, _ := it.Next()
root := k
fv := txkey.FieldViewFromPrefix([]byte(root))
fvs = append(fvs, fv)
}