diff --git a/ctl/rbf_check.go b/ctl/rbf_check.go index 00594b861..f3d40912d 100644 --- a/ctl/rbf_check.go +++ b/ctl/rbf_check.go @@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm } } -// Run executes the export. +// Run executes a consistency check of an RBF database. func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Open database. db := rbf.NewDB(cmd.Path, nil) @@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Run check on the database. if err := db.Check(); err != nil { - return err + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + return fmt.Errorf("check failed") } // If successful, print a success message. diff --git a/ctl/rbf_check_test.go b/ctl/rbf_check_test.go new file mode 100644 index 000000000..c11cd30dc --- /dev/null +++ b/ctl/rbf_check_test.go @@ -0,0 +1,33 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFCheckCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), `ok`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` { + t.Fatal(err) + } else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/rbf_pages.go b/ctl/rbf_pages.go index ca3484f31..b774175ae 100644 --- a/ctl/rbf_pages.go +++ b/ctl/rbf_pages.go @@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error { // Iterate over each page and grab info. infos, err := tx.PageInfos() if err != nil { - return err + fmt.Fprintln(cmd.Stdout, "ERRORS:") + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + fmt.Fprintln(cmd.Stdout, "") } // Write header. diff --git a/ctl/rbf_pages_test.go b/ctl/rbf_pages_test.go new file mode 100644 index 000000000..73c395e1b --- /dev/null +++ b/ctl/rbf_pages_test.go @@ -0,0 +1,52 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFPagesCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=4,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=5,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +4 unknown [] +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/data b/ctl/testdata/rbf-check/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-check/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/wal b/ctl/testdata/rbf-check/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-check/ok/data b/ctl/testdata/rbf-check/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-check/ok/data differ diff --git a/ctl/testdata/rbf-check/ok/wal b/ctl/testdata/rbf-check/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/data b/ctl/testdata/rbf-pages/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-pages/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/wal b/ctl/testdata/rbf-pages/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/ok/data b/ctl/testdata/rbf-pages/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-pages/ok/data differ diff --git a/ctl/testdata/rbf-pages/ok/wal b/ctl/testdata/rbf-pages/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/cursor.go b/rbf/cursor.go index eda6bec1d..2419bc5f8 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -932,6 +932,10 @@ func (c *Cursor) First() error { case PageTypeBranch: elem.index = 0 + if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count + return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n) + } + // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) diff --git a/rbf/rbf.go b/rbf/rbf.go index 74a5135eb..4a66ba309 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) { fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n) } } + +// ErrorList represents a list of errors. +type ErrorList []error + +// Err returns the list if it contains errors. Otherwise returns nil. +func (a ErrorList) Err() error { + if len(a) > 0 { + return a + } + return nil +} + +func (a ErrorList) Error() string { + switch len(a) { + case 0: + return "no errors" + case 1: + return a[0].Error() + } + return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1) +} + +func (a ErrorList) FullError() string { + if len(a) == 0 { + return "" + } + + var buf bytes.Buffer + for _, err := range a { + fmt.Fprintln(&buf, err) + } + return buf.String() +} + +// Append appends an error to the list. If err is an ErrorList then all errors are appended. +func (a *ErrorList) Append(err error) { + switch err := err.(type) { + case ErrorList: + *a = append(*a, err...) + default: + *a = append(*a, err) + } +} diff --git a/rbf/rbf/testdata/check/bad-freelist/data b/rbf/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..8b03c7b02 Binary files /dev/null and b/rbf/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/rbf/testdata/check/bad-freelist/wal b/rbf/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 4071a1a95..17605a6fc 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { if err != nil { panic(err) } + return NewDBAt(tb, path, cfg...) +} +// NewDBAt returns a new instance of DB with a given path. +func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { var cfg0 *rbfcfg.Config if len(cfg) > 0 { cfg0 = cfg[0] } - db := rbf.NewDB(path, cfg0) - return db + return rbf.NewDB(path, cfg0) } // MustOpenDB returns a db opened on a temporary file. On error, fail test. func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { + tb.Helper() + path, err := testhook.TempDir(tb, "rbfdb") + if err != nil { + panic(err) + } + return MustOpenDBAt(tb, path, cfg...) +} + +// MustOpenDBAt returns a db opened on an existing file. On error, fail test. +func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { tb.Helper() if len(cfg) == 0 || cfg[0] == nil { newconf := rbfcfg.NewDefaultConfig() @@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { } else if cfg[0].Logger == nil { cfg[0].Logger = logger.NewLogfLogger(tb) } - db := NewDB(tb, cfg...) + db := NewDBAt(tb, path, cfg...) if err := db.Open(); err != nil { tb.Fatal(err) } diff --git a/rbf/testdata/check/bad-bitmap/data b/rbf/testdata/check/bad-bitmap/data new file mode 100644 index 000000000..6cc32c0a1 Binary files /dev/null and b/rbf/testdata/check/bad-bitmap/data differ diff --git a/rbf/testdata/check/bad-bitmap/wal b/rbf/testdata/check/bad-bitmap/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/testdata/check/bad-freelist/data b/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..a762ee9db Binary files /dev/null and b/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/testdata/check/bad-freelist/wal b/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/tx.go b/rbf/tx.go index 5d731fe4f..c45380469 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -738,10 +738,11 @@ func (tx *Tx) Check() error { return ErrTxClosed } + var errorList ErrorList if err := tx.checkPageAllocations(); err != nil { - return fmt.Errorf("page allocations: %w", err) + errorList.Append(err) } - return nil + return errorList.Err() } func (tx *Tx) checkPage(pgno, parent, typ uint32) error { @@ -767,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error { // checkPageAllocations ensures that all pages are either in-use or on the freelist. func (tx *Tx) checkPageAllocations() error { + var errorList ErrorList freePageSet, err := tx.freePageSet() if err != nil { - return err + errorList.Append(err) } inusePageSet, err := tx.inusePageSet() if err != nil { - return err + errorList.Append(err) } // Iterate over all pages and ensure they are either in-use or free. @@ -785,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error { _, isFree := freePageSet[pgno] if isInuse && isFree { - return fmt.Errorf("page in-use & free: pgno=%d", pgno) - } else if !isInuse && !isFree { - page, _, err := tx.readPage(pgno) - if err != nil { - return err - } - flags := readFlags(page) - if flags == PageTypeBranch || flags == PageTypeLeaf { - return fmt.Errorf("page not in-use & not free: pgno=%d", pgno) - } - //assuming its a bitmap so its ok TODO ben? - return nil + errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno)) + continue + } + + if !isInuse && !isFree { + errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno)) + continue } } - return nil + return errorList.Err() } // freePageSet returns the set of pages in the freelist. func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList + m := make(map[uint32]struct{}) c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} @@ -816,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { for { if err := c.Next(); err == io.EOF { - return m, nil + return m, errorList.Err() } else if err != nil { - return m, err + errorList.Append(err) + return m, errorList.Err() } elem := &c.stack.elems[c.stack.top] leafPage, _, err := c.tx.readPage(elem.pgno) if err != nil { - return nil, err + errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err)) + continue } - cell := readLeafCell(leafPage, elem.index) + cell := readLeafCell(leafPage, elem.index) for _, v := range cell.Values(tx) { pgno := uint32((cell.Key << 16) | uint64(v)) m[pgno] = struct{}{} @@ -837,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { // inusePageSet returns the set of pages in use by the root records or b-trees. func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList m := make(map[uint32]struct{}) m[0] = struct{}{} // meta page @@ -846,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { page, _, err := tx.readPage(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } pgno = WalkRootRecordPages(page) } // Traverse freelist and mark pages as in-use. - if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + m[pgno] = struct{}{} - return tx.checkPage(pgno, parent, typ) + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil }); err != nil { return m, err } @@ -862,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return m, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + _, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - _, pgno := itr.Next() + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + } - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { - m[pgno] = struct{}{} - - return tx.checkPage(pgno, parent, typ) - }); err != nil { - return m, err + m[pgno] = struct{}{} + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil + }); err != nil { + return m, err + } } } - return m, nil + return m, errorList.Err() } // GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix. @@ -897,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // Traverse the bitmap's b-tree and count the bytes for each page. - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { n += PageSize - return nil + return err }); err != nil { return 0, err } @@ -908,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // walkTree recursively iterates over a page and all its children. -func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error { +func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error { // Read page and iterate over children. page, _, err := tx.readPage(pgno) if err != nil { - return err + return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err)) } - // Execute callback. - typ := readFlags(page) - if err := fn(pgno, parent, typ); err != nil { - return err - } - - switch typ { + switch typ := readFlags(page); typ { case PageTypeBranch: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + for i, n := 0, readCellN(page); i < n; i++ { cell := readBranchCell(page, i) if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil { @@ -930,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } return nil + case PageTypeLeaf: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + // Execute callback only for bitmap pages pointed to by this leaf. for i, n := 0, readCellN(page); i < n; i++ { if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr { - if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil { + if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil { return err } } } return nil + default: - return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ) + return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ)) } } @@ -1906,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) { // PageInfos returns meta data about all pages in the database. func (tx *Tx) PageInfos() ([]PageInfo, error) { + var errorList ErrorList infos := make([]PageInfo, tx.PageN()) // Read meta page info. @@ -1919,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { for pgno := metaInfo.RootRecordPageNo; pgno != 0; { info, err := tx.rootRecordPageInfo(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } infos[pgno] = info pgno = info.Next @@ -1927,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { // Traverse freelist and mark pages as in-use. if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil { - return nil, err + errorList.Append(err) } // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return nil, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + name, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() - - if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { - return nil, err + if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { + errorList.Append(err) + } } } // Build page info objects for each free page. freePageSet, err := tx.freePageSet() if err != nil { - return nil, err - } - for pgno := range freePageSet { - infos[pgno] = &FreePageInfo{Pgno: pgno} + errorList.Append(err) + } else { + for pgno := range freePageSet { + infos[pgno] = &FreePageInfo{Pgno: pgno} + } } - return infos, nil + return infos, errorList.Err() } // metaPageInfo returns page metadata for the meta page. @@ -1987,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) { } func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { - return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error { + var errorList ErrorList + + if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + buf, _, err := tx.readPage(pgno) if err != nil { - return err + errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err)) + return nil } switch typ { @@ -2016,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { Parent: parent, Tree: name, } - default: - vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno)) } return nil - }) + }); err != nil { + errorList.Append(err) + } + + return errorList.Err() } // PageData returns the raw page data for a single page. diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 5be3a50ae..7a726c2ed 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "os" + "path/filepath" "strings" "sync" "testing" @@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) { t.Fatalf("unexpected error: %#v", err) } }) + + t.Run("ErrBadFreelist", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) + + t.Run("ErrBadBitmap", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) } func mustReadPage(tb testing.TB, path string, pgno uint32) []byte {