From a98b9144cb59e560ca0969b811facadb7c5709ee Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 28 Nov 2022 12:12:03 -0600 Subject: [PATCH] upgrade to immutable 0.4.0(generics) (#2317) --- go.mod | 2 +- go.sum | 4 ++-- rbf/cursorx.go | 12 +++++++----- rbf/db.go | 18 ++++++++--------- rbf/rbf.go | 9 +++++---- rbf/tx.go | 53 +++++++++++++++++++++++++------------------------- 6 files changed, 50 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index c0350259c..50fac68d1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 github.com/aws/aws-sdk-go v1.42.39 github.com/beevik/ntp v0.3.0 - github.com/benbjohnson/immutable v0.3.0 + github.com/benbjohnson/immutable v0.4.0 github.com/cespare/xxhash v1.1.0 github.com/chzyer/readline v1.5.1 github.com/confluentinc/confluent-kafka-go v1.9.1 diff --git a/go.sum b/go.sum index 1634de0d4..a546005d7 100644 --- a/go.sum +++ b/go.sum @@ -139,8 +139,8 @@ github.com/aws/smithy-go v1.0.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= -github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw= -github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= +github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA= +github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 80e7afbc1..4a5fdc454 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -15,10 +15,10 @@ import ( // probably should just implement the container interface // but for now i'll do it func (c *Cursor) Rows() ([]uint64, error) { - shardVsContainerExponent := uint(4) //needs constant exported from roaring package + shardVsContainerExponent := uint(4) // needs constant exported from roaring package rows := make([]uint64, 0) if err := c.First(); err != nil { - if err == io.EOF { //root leaf with no elements + if err == io.EOF { // root leaf with no elements return rows, nil } return nil, errors.Wrap(err, "rows") @@ -52,15 +52,15 @@ func (tx *Tx) FieldViews() []string { records, _ := tx.RootRecords() a := make([]string, 0, records.Len()) for itr := records.Iterator(); !itr.Done(); { - name, _ := itr.Next() - a = append(a, name.(string)) + name, _, _ := itr.Next() + a = append(a, name) } return a } func (c *Cursor) DumpKeys() { if err := c.First(); err != nil { - //ignoring errors for this debug function + // ignoring errors for this debug function return } for { @@ -77,6 +77,7 @@ func (c *Cursor) DumpKeys() { fmt.Println("key", cell.Key) } } + func (c *Cursor) DumpStack() { fmt.Println("STACK") for i := c.stack.top; i >= 0; i-- { @@ -84,6 +85,7 @@ func (c *Cursor) DumpStack() { } fmt.Println() } + func (c *Cursor) Dump(name string) { writer, _ := os.Create(name) defer writer.Close() diff --git a/rbf/db.go b/rbf/db.go index 6453cb43d..f18f0fce0 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -44,13 +44,13 @@ type txWaiter struct { type DB struct { cfg rbfcfg.Config - data []byte // database mmap - file *os.File // database file descriptor - rootRecords *immutable.SortedMap // cached root records - pageMap *PageMap // pgno-to-WALID mapping - txs map[*Tx]struct{} // active transactions - opened bool // true if open - logger logger.Logger // for diagnostics from async things + data []byte // database mmap + file *os.File // database file descriptor + rootRecords *immutable.SortedMap[string, uint32] // cached root records + pageMap *PageMap // pgno-to-WALID mapping + txs map[*Tx]struct{} // active transactions + opened bool // true if open + logger logger.Logger // for diagnostics from async things wal []byte // wal mmap walFile *os.File // wal file descriptor @@ -514,9 +514,9 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { // If we can move to a cell then we have at least one record. for itr := records.Iterator(); !itr.Done(); { - name, _ := itr.Next() + name, _, _ := itr.Next() // Fetch cursor for bitmap. - cur, err := tx.Cursor(name.(string)) + cur, err := tx.Cursor(name) if err != nil { return false, err } diff --git a/rbf/rbf.go b/rbf/rbf.go index 2778a6592..12b1198a2 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -149,6 +149,7 @@ func writeMetaChecksum(page []byte, chksum uint32) { // Root record page helpers func WalkRootRecordPages(page []byte) uint32 { return binary.BigEndian.Uint32(page[8:]) } + func writeRootRecordOverflowPgno(page []byte, pgno uint32) { binary.BigEndian.PutUint32(page[8:], pgno) } @@ -169,13 +170,13 @@ func readRootRecords(page []byte) (records []*RootRecord, err error) { // We can return io.ErrShortBuffer in err. If we still have records // to write that don't fit on page, remain will point to the next // record that hasn't yet been written. -func writeRootRecords(page []byte, itr *immutable.SortedMapIterator) (err error) { +func writeRootRecords(page []byte, itr *immutable.SortedMapIterator[string, uint32]) (err error) { data := page[rootRecordPageHeaderSize:] for !itr.Done() { - name, pgno := itr.Next() + name, pgno, _ := itr.Next() - data, err = WriteRootRecord(data, &RootRecord{Name: name.(string), Pgno: pgno.(uint32)}) + data, err = WriteRootRecord(data, &RootRecord{Name: name, Pgno: pgno}) if err != nil { itr.Seek(name) return err @@ -685,7 +686,7 @@ func Pagedump(b []byte, indent string, writer io.Writer) { cell := readLeafCell(b, i) switch cell.Type { case ContainerTypeArray: - //fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data)) + // fmt.Fprintf(os.Stderr, "[%d]: key=%d type=array n=%d elems=%v\n", i, cell.Key, cell.N, toArray16(cell.Data)) fmt.Fprintf(writer, "%s[%d]: key=%d type=array BitN=%d \n", indent, i, cell.Key, cell.BitN) case ContainerTypeRLE: fmt.Fprintf(writer, "%s[%d]: key=%d type=rle BitN=%d\n", indent, i, cell.Key, cell.BitN) diff --git a/rbf/tx.go b/rbf/tx.go index a3252df86..600495044 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -24,11 +24,11 @@ var _ = txkey.ToString // view at the point-in-time they are started. type Tx struct { mu sync.RWMutex - db *DB // parent db - meta [PageSize]byte // copy of current meta page - walID int64 // max WAL ID at start of tx - walPageN int // wal page count - rootRecords *immutable.SortedMap // read-only cache of root records + db *DB // parent db + meta [PageSize]byte // copy of current meta page + walID int64 // max WAL ID at start of tx + walPageN int // wal page count + rootRecords *immutable.SortedMap[string, uint32] // read-only cache of root records // pageMap holds WAL pages that have not yet been transferred // into the database pages. So it can be empty, if the whole previous @@ -249,7 +249,7 @@ func (tx *Tx) root(name string) (uint32, error) { if !ok { return 0, ErrBitmapNotFound } - return pgno.(uint32), nil + return pgno, nil } // BitmapNames returns a list of all bitmap names. @@ -269,8 +269,8 @@ func (tx *Tx) BitmapNames() ([]string, error) { a := make([]string, 0, records.Len()) for itr := records.Iterator(); !itr.Done(); { - k, _ := itr.Next() - a = append(a, k.(string)) + k, _, _ := itr.Next() + a = append(a, k) } return a, nil } @@ -393,7 +393,7 @@ func (tx *Tx) DeleteBitmap(name string) error { } // Deallocate all pages in the tree. - if err := tx.deallocateTree(pgno.(uint32)); err != nil { + if err := tx.deallocateTree(pgno); err != nil { return err } @@ -424,18 +424,18 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { } for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() + name, pgno, _ := itr.Next() // Skip bitmaps without matching prefix. - if !strings.HasPrefix(name.(string), prefix) { + if !strings.HasPrefix(name, prefix) { continue } // Deallocate all pages in the tree. - if err := tx.deallocateTree(pgno.(uint32)); err != nil { + if err := tx.deallocateTree(pgno); err != nil { return err } - records = records.Delete(name.(string)) + records = records.Delete(name) } // Rewrite record pages. @@ -483,12 +483,12 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error { } // RootRecords returns a list of root records. -func (tx *Tx) RootRecords() (records *immutable.SortedMap, err error) { +func (tx *Tx) RootRecords() (records *immutable.SortedMap[string, uint32], err error) { if tx.rootRecords != nil { return tx.rootRecords, nil } - records = immutable.NewSortedMap(nil) + records = immutable.NewSortedMap[string, uint32](nil) for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; { page, _, err := tx.readPage(pgno) if err != nil { @@ -514,8 +514,7 @@ func (tx *Tx) RootRecords() (records *immutable.SortedMap, err error) { } // writeRootRecordPages writes a list of root record pages. -func (tx *Tx) writeRootRecordPages(records *immutable.SortedMap) (err error) { - +func (tx *Tx) writeRootRecordPages(records *immutable.SortedMap[string, uint32]) (err error) { // Release all existing root record pages. for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; { page, _, err := tx.readPage(pgno) @@ -998,9 +997,9 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { errorList.Append(err) } else { for itr := records.Iterator(); !itr.Done(); { - _, pgno := itr.Next() + _, pgno, _ := itr.Next() - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { + if err := tx.walkTree(pgno, 0, func(pgno, parent, typ uint32, err error) error { if err != nil { errorList.Append(err) } @@ -1028,15 +1027,15 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { // Loop over each bitmap in the database. for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() + name, pgno, _ := itr.Next() // Skip over any bitmaps that don't have a matching prefix. - if !strings.HasPrefix(name.(string), prefix) { + if !strings.HasPrefix(name, prefix) { continue } // Traverse the bitmap's b-tree and count the bytes for each page. - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { + if err := tx.walkTree(pgno, 0, func(pgno, parent, typ uint32, err error) error { n += PageSize return err }); err != nil { @@ -1801,13 +1800,13 @@ func (si *emptyContainerIterator) Close() {} func (si *emptyContainerIterator) Next() bool { return false } + func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { vprint.PanicOn("emptyContainerIterator never has any Values") return 0, nil } func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - // begin write boilerplate if tx.db == nil { err = ErrTxClosed @@ -2134,9 +2133,9 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { errorList.Append(err) } else { for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() + name, pgno, _ := itr.Next() - if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { + if err := tx.walkPageInfo(infos, pgno, name); err != nil { errorList.Append(err) } } @@ -2246,8 +2245,8 @@ func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) { } it := records.Iterator() for !it.Done() { - k, _ := it.Next() - root := k.(string) + k, _, _ := it.Next() + root := k fv := txkey.FieldViewFromPrefix([]byte(root)) fvs = append(fvs, fv) }