Avoid panics in RBF debug tooling

ctl/rbf_check.go

@@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm
 	}
 }
 
-// Run executes the export.
+// Run executes a consistency check of an RBF database.
 func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
 	// Open database.
 	db := rbf.NewDB(cmd.Path, nil)
@@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
 
 	// Run check on the database.
 	if err := db.Check(); err != nil {
-		return err
+		switch err := err.(type) {
+		case rbf.ErrorList:
+			for i := range err {
+				fmt.Fprintln(cmd.Stdout, err[i])
+			}
+		default:
+			fmt.Fprintln(cmd.Stdout, err)
+		}
+		return fmt.Errorf("check failed")
 	}
 
 	// If successful, print a success message.

ctl/rbf_check_test.go

@@ -0,0 +1,33 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package ctl
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"testing"
+)
+
+func TestRBFCheckCommand_Run(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-check", "ok")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), `ok`+"\n"; got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+
+	t.Run("ErrInvalidPageType", func(t *testing.T) {
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type")
+		if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+}

ctl/rbf_pages.go

@@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
 	// Iterate over each page and grab info.
 	infos, err := tx.PageInfos()
 	if err != nil {
-		return err
+		fmt.Fprintln(cmd.Stdout, "ERRORS:")
+		switch err := err.(type) {
+		case rbf.ErrorList:
+			for i := range err {
+				fmt.Fprintln(cmd.Stdout, err[i])
+			}
+		default:
+			fmt.Fprintln(cmd.Stdout, err)
+		}
+		fmt.Fprintln(cmd.Stdout, "")
 	}
 
 	// Write header.

ctl/rbf_pages_test.go

@@ -0,0 +1,52 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package ctl
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"testing"
+)
+
+func TestRBFPagesCommand_Run(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		want := `
+ID       TYPE       EXTRA
+======== ========== ====================
+0        meta       pageN=4,walid=4,rootrec=1,freelist=2
+1        rootrec    next=0
+2        leaf       flags=x2,celln=0
+3        leaf       flags=x2,celln=1
+`[1:]
+
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-pages", "ok")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := stdout.String(); got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+
+	t.Run("ErrInvalidPageType", func(t *testing.T) {
+		want := `
+ID       TYPE       EXTRA
+======== ========== ====================
+0        meta       pageN=5,walid=4,rootrec=1,freelist=2
+1        rootrec    next=0
+2        leaf       flags=x2,celln=0
+3        leaf       flags=x2,celln=1
+4        unknown [<nil>]
+`[1:]
+
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := stdout.String(); got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+}

rbf/cursor.go

@@ -932,6 +932,10 @@ func (c *Cursor) First() error {
 		case PageTypeBranch:
 			elem.index = 0
 
+			if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count
+				return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n)
+			}
+
 			// Read cell pgno into the next stack level.
 			cell := readBranchCell(buf, elem.index)
 

rbf/rbf.go

@@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) {
 		fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n)
 	}
 }
+
+// ErrorList represents a list of errors.
+type ErrorList []error
+
+// Err returns the list if it contains errors. Otherwise returns nil.
+func (a ErrorList) Err() error {
+	if len(a) > 0 {
+		return a
+	}
+	return nil
+}
+
+func (a ErrorList) Error() string {
+	switch len(a) {
+	case 0:
+		return "no errors"
+	case 1:
+		return a[0].Error()
+	}
+	return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1)
+}
+
+func (a ErrorList) FullError() string {
+	if len(a) == 0 {
+		return ""
+	}
+
+	var buf bytes.Buffer
+	for _, err := range a {
+		fmt.Fprintln(&buf, err)
+	}
+	return buf.String()
+}
+
+// Append appends an error to the list. If err is an ErrorList then all errors are appended.
+func (a *ErrorList) Append(err error) {
+	switch err := err.(type) {
+	case ErrorList:
+		*a = append(*a, err...)
+	default:
+		*a = append(*a, err)
+	}
+}

rbf/rbf_test.go

@@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
 	if err != nil {
 		panic(err)
 	}
+	return NewDBAt(tb, path, cfg...)
+}
 
+// NewDBAt returns a new instance of DB with a given path.
+func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
 	var cfg0 *rbfcfg.Config
 	if len(cfg) > 0 {
 		cfg0 = cfg[0]
 	}
-	db := rbf.NewDB(path, cfg0)
-	return db
+	return rbf.NewDB(path, cfg0)
 }
 
 // MustOpenDB returns a db opened on a temporary file. On error, fail test.
 func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
+	tb.Helper()
+	path, err := testhook.TempDir(tb, "rbfdb")
+	if err != nil {
+		panic(err)
+	}
+	return MustOpenDBAt(tb, path, cfg...)
+}
+
+// MustOpenDBAt returns a db opened on an existing file. On error, fail test.
+func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
 	tb.Helper()
 	if len(cfg) == 0 || cfg[0] == nil {
 		newconf := rbfcfg.NewDefaultConfig()
@@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
 	} else if cfg[0].Logger == nil {
 		cfg[0].Logger = logger.NewLogfLogger(tb)
 	}
-	db := NewDB(tb, cfg...)
+	db := NewDBAt(tb, path, cfg...)
 	if err := db.Open(); err != nil {
 		tb.Fatal(err)
 	}

rbf/tx.go

@@ -738,10 +738,11 @@ func (tx *Tx) Check() error {
 		return ErrTxClosed
 	}
 
+	var errorList ErrorList
 	if err := tx.checkPageAllocations(); err != nil {
-		return fmt.Errorf("page allocations: %w", err)
+		errorList.Append(err)
 	}
-	return nil
+	return errorList.Err()
 }
 
 func (tx *Tx) checkPage(pgno, parent, typ uint32) error {
@@ -767,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error {
 
 // checkPageAllocations ensures that all pages are either in-use or on the freelist.
 func (tx *Tx) checkPageAllocations() error {
+	var errorList ErrorList
 	freePageSet, err := tx.freePageSet()
 	if err != nil {
-		return err
+		errorList.Append(err)
 	}
 
 	inusePageSet, err := tx.inusePageSet()
 	if err != nil {
-		return err
+		errorList.Append(err)
 	}
 
 	// Iterate over all pages and ensure they are either in-use or free.
@@ -785,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error {
 		_, isFree := freePageSet[pgno]
 
 		if isInuse && isFree {
-			return fmt.Errorf("page in-use & free: pgno=%d", pgno)
-		} else if !isInuse && !isFree {
-			page, _, err := tx.readPage(pgno)
-			if err != nil {
-				return err
-			}
-			flags := readFlags(page)
-			if flags == PageTypeBranch || flags == PageTypeLeaf {
-				return fmt.Errorf("page not in-use & not free: pgno=%d", pgno)
-			}
-			//assuming its a bitmap so its ok TODO ben?
-			return nil
+			errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno))
+			continue
+		}
+
+		if !isInuse && !isFree {
+			errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno))
+			continue
 		}
 	}
 
-	return nil
+	return errorList.Err()
 }
 
 // freePageSet returns the set of pages in the freelist.
 func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
+	var errorList ErrorList
+
 	m := make(map[uint32]struct{})
 	c := Cursor{tx: tx}
 	c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])}
@@ -816,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
 
 	for {
 		if err := c.Next(); err == io.EOF {
-			return m, nil
+			return m, errorList.Err()
 		} else if err != nil {
-			return m, err
+			errorList.Append(err)
+			return m, errorList.Err()
 		}
 
 		elem := &c.stack.elems[c.stack.top]
 		leafPage, _, err := c.tx.readPage(elem.pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err))
+			continue
 		}
-		cell := readLeafCell(leafPage, elem.index)
 
+		cell := readLeafCell(leafPage, elem.index)
 		for _, v := range cell.Values(tx) {
 			pgno := uint32((cell.Key << 16) | uint64(v))
 			m[pgno] = struct{}{}
@@ -837,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
 
 // inusePageSet returns the set of pages in use by the root records or b-trees.
 func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
+	var errorList ErrorList
 	m := make(map[uint32]struct{})
 	m[0] = struct{}{} // meta page
 
@@ -846,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
 
 		page, _, err := tx.readPage(pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(err)
+			break
 		}
 		pgno = WalkRootRecordPages(page)
 	}
 
 	// Traverse freelist and mark pages as in-use.
-	if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error {
+	if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error {
+		if err != nil {
+			errorList.Append(err)
+			return nil
+		}
+
 		m[pgno] = struct{}{}
-		return tx.checkPage(pgno, parent, typ)
+		if err := tx.checkPage(pgno, parent, typ); err != nil {
+			errorList.Append(err)
+		}
+		return nil
 	}); err != nil {
 		return m, err
 	}
@@ -862,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
 	// Traverse every b-tree and mark pages as in-use.
 	records, err := tx.RootRecords()
 	if err != nil {
-		return m, err
-	}
+		errorList.Append(err)
+	} else {
+		for itr := records.Iterator(); !itr.Done(); {
+			_, pgno := itr.Next()
 
-	for itr := records.Iterator(); !itr.Done(); {
-		_, pgno := itr.Next()
+			if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
+				if err != nil {
+					errorList.Append(err)
+				}
 
-		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
-			m[pgno] = struct{}{}
-
-			return tx.checkPage(pgno, parent, typ)
-		}); err != nil {
-			return m, err
+				m[pgno] = struct{}{}
+				if err := tx.checkPage(pgno, parent, typ); err != nil {
+					errorList.Append(err)
+				}
+				return nil
+			}); err != nil {
+				return m, err
+			}
 		}
 	}
 
-	return m, nil
+	return m, errorList.Err()
 }
 
 // GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix.
@@ -897,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
 		}
 
 		// Traverse the bitmap's b-tree and count the bytes for each page.
-		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
+		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
 			n += PageSize
-			return nil
+			return err
 		}); err != nil {
 			return 0, err
 		}
@@ -908,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
 }
 
 // walkTree recursively iterates over a page and all its children.
-func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error {
+func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error {
 	// Read page and iterate over children.
 	page, _, err := tx.readPage(pgno)
 	if err != nil {
-		return err
+		return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err))
 	}
 
-	// Execute callback.
-	typ := readFlags(page)
-	if err := fn(pgno, parent, typ); err != nil {
-		return err
-	}
-
-	switch typ {
+	switch typ := readFlags(page); typ {
 	case PageTypeBranch:
+		if err := fn(pgno, parent, typ, nil); err != nil {
+			return err
+		}
+
 		for i, n := 0, readCellN(page); i < n; i++ {
 			cell := readBranchCell(page, i)
 			if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil {
@@ -930,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er
 			}
 		}
 		return nil
+
 	case PageTypeLeaf:
+		if err := fn(pgno, parent, typ, nil); err != nil {
+			return err
+		}
+
 		// Execute callback only for bitmap pages pointed to by this leaf.
 		for i, n := 0, readCellN(page); i < n; i++ {
 			if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr {
-				if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil {
+				if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil {
 					return err
 				}
 			}
 		}
 		return nil
+
 	default:
-		return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ)
+		return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ))
 	}
 }
 
@@ -1906,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) {
 
 // PageInfos returns meta data about all pages in the database.
 func (tx *Tx) PageInfos() ([]PageInfo, error) {
+	var errorList ErrorList
 	infos := make([]PageInfo, tx.PageN())
 
 	// Read meta page info.
@@ -1919,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
 	for pgno := metaInfo.RootRecordPageNo; pgno != 0; {
 		info, err := tx.rootRecordPageInfo(pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(err)
+			break
 		}
 		infos[pgno] = info
 		pgno = info.Next
@@ -1927,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
 
 	// Traverse freelist and mark pages as in-use.
 	if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil {
-		return nil, err
+		errorList.Append(err)
 	}
 
 	// Traverse every b-tree and mark pages as in-use.
 	records, err := tx.RootRecords()
 	if err != nil {
-		return nil, err
-	}
+		errorList.Append(err)
+	} else {
+		for itr := records.Iterator(); !itr.Done(); {
+			name, pgno := itr.Next()
 
-	for itr := records.Iterator(); !itr.Done(); {
-		name, pgno := itr.Next()
-
-		if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
-			return nil, err
+			if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
+				errorList.Append(err)
+			}
 		}
 	}
 
 	// Build page info objects for each free page.
 	freePageSet, err := tx.freePageSet()
 	if err != nil {
-		return nil, err
-	}
-	for pgno := range freePageSet {
-		infos[pgno] = &FreePageInfo{Pgno: pgno}
+		errorList.Append(err)
+	} else {
+		for pgno := range freePageSet {
+			infos[pgno] = &FreePageInfo{Pgno: pgno}
+		}
 	}
 
-	return infos, nil
+	return infos, errorList.Err()
 }
 
 // metaPageInfo returns page metadata for the meta page.
@@ -1987,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) {
 }
 
 func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
-	return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error {
+	var errorList ErrorList
+
+	if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error {
+		if err != nil {
+			errorList.Append(err)
+			return nil
+		}
+
 		buf, _, err := tx.readPage(pgno)
 		if err != nil {
-			return err
+			errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err))
+			return nil
 		}
 
 		switch typ {
@@ -2016,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
 				Parent: parent,
 				Tree:   name,
 			}
-		default:
-			vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno))
 		}
 
 		return nil
-	})
+	}); err != nil {
+		errorList.Append(err)
+	}
+
+	return errorList.Err()
 }
 
 // PageData returns the raw page data for a single page.

rbf/tx_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) {
 			t.Fatalf("unexpected error: %#v", err)
 		}
 	})
+
+	t.Run("ErrBadFreelist", func(t *testing.T) {
+		t.Parallel()
+
+		db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist"))
+		defer db.Close()
+		tx := MustBegin(t, db, false)
+		defer tx.Rollback()
+
+		if err, ok := tx.Check().(rbf.ErrorList); !ok {
+			t.Fatal("expected error list")
+		} else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) {
+			t.Fatalf("unexpected error:\n%s", s)
+		}
+	})
+
+	t.Run("ErrBadBitmap", func(t *testing.T) {
+		t.Parallel()
+
+		db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap"))
+		defer db.Close()
+		tx := MustBegin(t, db, false)
+		defer tx.Rollback()
+
+		if err, ok := tx.Check().(rbf.ErrorList); !ok {
+			t.Fatal("expected error list")
+		} else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) {
+			t.Fatalf("unexpected error:\n%s", s)
+		}
+	})
 }
 
 func mustReadPage(tb testing.TB, path string, pgno uint32) []byte {