From 7fa1a5edd27e9e6fba82e9076290dd41dbfe244d Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 23 Oct 2020 08:22:15 -0600 Subject: [PATCH] Add RBF CLI commands --- cmd/rbf.go | 126 +++++++++++++++++ cmd/root.go | 1 + ctl/rbf_check.go | 60 ++++++++ ctl/rbf_dump.go | 111 +++++++++++++++ ctl/rbf_page.go | 140 +++++++++++++++++++ ctl/rbf_pages.go | 99 +++++++++++++ rbf/cursor_test.go | 6 +- rbf/cursorx.go | 4 +- rbf/rbf.go | 43 ++++-- rbf/tx.go | 336 +++++++++++++++++++++++++++++++++++++++++++-- 10 files changed, 901 insertions(+), 25 deletions(-) create mode 100644 cmd/rbf.go create mode 100644 ctl/rbf_check.go create mode 100644 ctl/rbf_dump.go create mode 100644 ctl/rbf_page.go create mode 100644 ctl/rbf_pages.go diff --git a/cmd/rbf.go b/cmd/rbf.go new file mode 100644 index 000000000..f8de09fb1 --- /dev/null +++ b/cmd/rbf.go @@ -0,0 +1,126 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "errors" + "io" + "strconv" + + "github.com/pilosa/pilosa/v2/ctl" + "github.com/spf13/cobra" +) + +func newRBFCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "rbf", + Short: "Inspect RBF data files.", + Long: ` +Provides a set of commands for inspecting RBF data files. +`, + } + cmd.AddCommand(newRBFCheckCommand(stdin, stdout, stderr)) + cmd.AddCommand(newRBFDumpCommand(stdin, stdout, stderr)) + cmd.AddCommand(newRBFPagesCommand(stdin, stdout, stderr)) + cmd.AddCommand(newRBFPageCommand(stdin, stdout, stderr)) + return cmd +} + +func newRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + c := ctl.NewRBFCheckCommand(stdin, stdout, stderr) + cmd := &cobra.Command{ + Use: "check", + Short: "Run consistency check on RBF data.", + Long: ` +Executes a consistency check on an RBF data directory. +`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c.Path = args[0] + return c.Run(context.Background()) + }, + } + return cmd +} + +func newRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + c := ctl.NewRBFDumpCommand(stdin, stdout, stderr) + cmd := &cobra.Command{ + Use: "dump", + Short: "Prints RBF raw page data", + Long: ` +Dumps the raw hex data for one or more RBF pages. +`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c.Path = args[0] + + for _, arg := range args[1:] { + pgno, err := strconv.Atoi(arg) + if err != nil { + return errors.New("invalid page number") + } + c.Pgnos = append(c.Pgnos, uint32(pgno)) + } + + return c.Run(context.Background()) + }, + } + return cmd +} + +func newRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + c := ctl.NewRBFPagesCommand(stdin, stdout, stderr) + cmd := &cobra.Command{ + Use: "pages", + Short: "Prints metadata for the list of all pages", + Long: ` +Prints a line for every page in the database with its type/status. +`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c.Path = args[0] + return c.Run(context.Background()) + }, + } + return cmd +} + +func newRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + c := ctl.NewRBFPageCommand(stdin, stdout, stderr) + cmd := &cobra.Command{ + Use: "page", + Short: "Prints data for a single page", + Long: ` +Prints the header & cell data for a single page. +`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c.Path = args[0] + + for _, arg := range args[1:] { + pgno, err := strconv.Atoi(arg) + if err != nil { + return errors.New("invalid page number") + } + c.Pgnos = append(c.Pgnos, uint32(pgno)) + } + + return c.Run(context.Background()) + }, + } + return cmd +} diff --git a/cmd/root.go b/cmd/root.go index e3c25ef50..89f59bcfb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -69,6 +69,7 @@ at https://www.pilosa.com/docs/. rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr)) rc.AddCommand(newImportCommand(stdin, stdout, stderr)) rc.AddCommand(newInspectCommand(stdin, stdout, stderr)) + rc.AddCommand(newRBFCommand(stdin, stdout, stderr)) rc.AddCommand(newServeCmd(stdin, stdout, stderr)) rc.AddCommand(newHolderCmd(stdin, stdout, stderr)) diff --git a/ctl/rbf_check.go b/ctl/rbf_check.go new file mode 100644 index 000000000..ded98604d --- /dev/null +++ b/ctl/rbf_check.go @@ -0,0 +1,60 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "context" + "fmt" + "io" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/rbf" +) + +// RBFCheckCommand represents a command for running a consistency check on RBF. +type RBFCheckCommand struct { + // Filepath to the RBF database. + Path string + + // Standard input/output + *pilosa.CmdIO +} + +// NewRBFCheckCommand returns a new instance of RBFCheckCommand. +func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckCommand { + return &RBFCheckCommand{ + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + } +} + +// Run executes the export. +func (cmd *RBFCheckCommand) Run(ctx context.Context) error { + // Open database. + db := rbf.NewDB(cmd.Path) + if err := db.Open(); err != nil { + return err + } + defer db.Close() + + // Run check on the database. + if err := db.Check(); err != nil { + return err + } + + // If successful, print a success message. + fmt.Fprintln(cmd.Stdout, "ok") + + return nil +} diff --git a/ctl/rbf_dump.go b/ctl/rbf_dump.go new file mode 100644 index 000000000..3af88ef20 --- /dev/null +++ b/ctl/rbf_dump.go @@ -0,0 +1,111 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "strings" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/rbf" +) + +// RBFDumpCommand represents a command for dumping raw data for an RBF page. +type RBFDumpCommand struct { + // Filepath to the RBF database. + Path string + + // Page numbers to print. + Pgnos []uint32 + + // Standard input/output + *pilosa.CmdIO +} + +// NewRBFDumpCommand returns a new instance of RBFDumpCommand. +func NewRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFDumpCommand { + return &RBFDumpCommand{ + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + } +} + +// Run executes the export. +func (cmd *RBFDumpCommand) Run(ctx context.Context) error { + // Open database. + db := rbf.NewDB(cmd.Path) + if err := db.Open(); err != nil { + return err + } + defer db.Close() + + // Execute with a transaction. + tx, err := db.Begin(false) + if err != nil { + return err + } + defer tx.Rollback() + + // Fetch each page & dump. + for _, pgno := range cmd.Pgnos { + buf, err := tx.PageData(pgno) + if err != nil { + return err + } + + fmt.Fprintf(cmd.Stdout, "## PAGE %d\n", pgno) + fmt.Fprintln(cmd.Stdout, compressedHexDump(buf)) + fmt.Fprintln(cmd.Stdout, "") + } + + return nil +} + +func compressedHexDump(b []byte) string { + const prefixN = len("00000000") + + var output []string + var prev string + var ellipsis bool + + lines := strings.Split(strings.TrimSpace(hex.Dump(b)), "\n") + for i, line := range lines { + // Add line to output if it is not repeating or the last line. + if i == 0 || i == len(lines)-1 || trimPrefixN(line, prefixN) != trimPrefixN(prev, prefixN) { + output = append(output, line) + prev, ellipsis = line, false + continue + } + + // Add an ellipsis for the first duplicate line. + if !ellipsis { + output = append(output, "...") + ellipsis = true + continue + } + } + + return strings.Join(output, "\n") +} + +// trimPrefixN trims n bytes from the beginning of a string. +func trimPrefixN(s string, n int) string { + if len(s) < n { + return "" + } + return s[n:] +} diff --git a/ctl/rbf_page.go b/ctl/rbf_page.go new file mode 100644 index 000000000..d392131ae --- /dev/null +++ b/ctl/rbf_page.go @@ -0,0 +1,140 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "context" + "fmt" + "io" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/rbf" +) + +// RBFPageCommand represents a command for printing data for a single RBF page. +type RBFPageCommand struct { + // Filepath to the RBF database. + Path string + + // Page numbers to print. + Pgnos []uint32 + + // Standard input/output + *pilosa.CmdIO +} + +// NewRBFPageCommand returns a new instance of RBFPageCommand. +func NewRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPageCommand { + return &RBFPageCommand{ + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + } +} + +// Run executes the export. +func (cmd *RBFPageCommand) Run(ctx context.Context) error { + // Open database. + db := rbf.NewDB(cmd.Path) + if err := db.Open(); err != nil { + return err + } + defer db.Close() + + // Execute with a transaction. + tx, err := db.Begin(false) + if err != nil { + return err + } + defer tx.Rollback() + + // Fetch the page. + pages, err := tx.Pages(cmd.Pgnos) + if err != nil { + return err + } + + for _, page := range pages { + switch page := page.(type) { + case *rbf.MetaPage: + cmd.printMetaPage(page) + case *rbf.RootRecordPage: + cmd.printRootRecordPage(page) + case *rbf.LeafPage: + cmd.printLeafPage(page) + case *rbf.BranchPage: + cmd.printBranchPage(page) + case *rbf.BitmapPage: + cmd.printBitmapPage(page) + case *rbf.FreePage: + cmd.printFreePage(page) + default: + return fmt.Errorf("unexpected page type %T", page) + } + fmt.Fprintln(cmd.Stdout, "") + } + + return nil +} + +func (cmd *RBFPageCommand) printMetaPage(page *rbf.MetaPage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: meta\n") + fmt.Fprintf(cmd.Stdout, "PageN: %d\n", page.PageN) + fmt.Fprintf(cmd.Stdout, "WALID: %d\n", page.WALID) + fmt.Fprintf(cmd.Stdout, "Root Record Pgno: %d\n", page.RootRecordPageNo) + fmt.Fprintf(cmd.Stdout, "Freelist Pgno: %d\n", page.FreelistPageNo) +} + +func (cmd *RBFPageCommand) printRootRecordPage(page *rbf.RootRecordPage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: root record\n") + fmt.Fprintf(cmd.Stdout, "Next: %d\n", page.Next) + fmt.Fprintf(cmd.Stdout, "Records: n=%d\n", len(page.Records)) + for i, rec := range page.Records { + fmt.Fprintf(cmd.Stdout, "[%d]: name=%q pgno=%d\n", i, rec.Name, rec.Pgno) + } +} + +func (cmd *RBFPageCommand) printLeafPage(page *rbf.LeafPage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: leaf\n") + fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells)) + for i, cell := range page.Cells { + if cell.Type == "bitmap-ptr" { + fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s pgno=%d\n", i, cell.Key, cell.Type, cell.Pgno) + } else { + fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s values=%v\n", i, cell.Key, cell.Type, cell.Values) + } + } +} + +func (cmd *RBFPageCommand) printBranchPage(page *rbf.BranchPage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: branch\n") + fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells)) + for i, cell := range page.Cells { + fmt.Fprintf(cmd.Stdout, "[%d]: key=%d flags=%d pgno=%d\n", i, cell.Key, cell.Flags, cell.Pgno) + } +} + +func (cmd *RBFPageCommand) printBitmapPage(page *rbf.BitmapPage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: bitmap\n") + fmt.Fprintf(cmd.Stdout, "Values: %v\n", page.Values) +} + +func (cmd *RBFPageCommand) printFreePage(page *rbf.FreePage) { + fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno) + fmt.Fprintf(cmd.Stdout, "Type: free\n") +} diff --git a/ctl/rbf_pages.go b/ctl/rbf_pages.go new file mode 100644 index 000000000..6bd4d9627 --- /dev/null +++ b/ctl/rbf_pages.go @@ -0,0 +1,99 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "context" + "fmt" + "io" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/rbf" + "github.com/pilosa/pilosa/v2/txkey" +) + +// RBFPagesCommand represents a command for printing a list of RBF page metadata. +type RBFPagesCommand struct { + // Filepath to the RBF database. + Path string + + // Standard input/output + *pilosa.CmdIO +} + +// NewRBFPagesCommand returns a new instance of RBFPagesCommand. +func NewRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPagesCommand { + return &RBFPagesCommand{ + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + } +} + +// Run executes the export. +func (cmd *RBFPagesCommand) Run(ctx context.Context) error { + // Open database. + db := rbf.NewDB(cmd.Path) + if err := db.Open(); err != nil { + return err + } + defer db.Close() + + // Execute with a transaction. + tx, err := db.Begin(false) + if err != nil { + return err + } + defer tx.Rollback() + + // Iterate over each page and grab info. + infos, err := tx.PageInfos() + if err != nil { + return err + } + + // Write header. + fmt.Fprintln(cmd.Stdout, "ID TYPE TREE EXTRA") + fmt.Fprintln(cmd.Stdout, "======== ========== ============================== ====================") + + // Print one line for each page. + for pgno, info := range infos { + switch info := info.(type) { + case *rbf.MetaPageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", pgno, "meta", "", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) + case *rbf.RootRecordPageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q next=%d\n", pgno, "rootrec", "", info.Next) + case *rbf.LeafPageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q flags=x%x,celln=%d\n", pgno, "leaf", txkeyString(info.Tree), info.Flags, info.CellN) + case *rbf.BranchPageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q flags=x%x,celln=%d\n", pgno, "branch", txkeyString(info.Tree), info.Flags, info.CellN) + case *rbf.BitmapPageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q -\n", pgno, "bitmap", info.Tree) + case *rbf.FreePageInfo: + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-30q -\n", pgno, "free", "") + default: + panic(fmt.Sprintf("unexpected page info type %T", info)) + } + } + + return nil +} + +func txkeyString(s string) (ret string) { + defer func() { + if err := recover(); err != nil { + ret = s + } + }() + return txkey.ToString([]byte(s)) +} diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 9f134a92e..0b4edf718 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -807,7 +807,7 @@ type EasyWalker struct { func (e *EasyWalker) Visitor(pgno uint32, records []*rbf.RootRecord) { for _, record := range records { e.VisitRoot(record.Pgno, record.Name) - rbf.Page(e.tx, record.Pgno, e) + rbf.WalkPage(e.tx, record.Pgno, e) } } func (e *EasyWalker) VisitRoot(pgno uint32, name string) { @@ -927,7 +927,7 @@ func TestCursor_SplitBranchCells(t *testing.T) { } } before := &EasyWalker{tx: tx} - rbf.Page(tx, 0, before) + rbf.WalkPage(tx, 0, before) if before.String() != "RL" { t.Fatalf("Expecting RL (one branch) got %v", before.String()) @@ -939,7 +939,7 @@ func TestCursor_SplitBranchCells(t *testing.T) { } after := &EasyWalker{tx: tx} - rbf.Page(tx, 0, after) + rbf.WalkPage(tx, 0, after) if after.String() != "RBLL" { t.Fatalf("Expecting RBLL (a branch split) got %v", after.String()) diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 3c4b16352..107aad915 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -215,7 +215,7 @@ type Walker interface { Visit(pgno uint32, n Nodetype) } -func Page(tx *Tx, pgno uint32, walker Walker) { +func WalkPage(tx *Tx, pgno uint32, walker Walker) { page, err := tx.readPage(pgno) if err != nil { panic(err) @@ -232,7 +232,7 @@ func Page(tx *Tx, pgno uint32, walker Walker) { walker.Visit(pgno, Branch) for i, n := 0, readCellN(page); i < n; i++ { cell := readBranchCell(page, i) - Page(tx, cell.Pgno, walker) + WalkPage(tx, cell.Pgno, walker) } case PageTypeLeaf: walker.Visit(pgno, Leaf) diff --git a/rbf/rbf.go b/rbf/rbf.go index d1f2fe4ca..69d9078b5 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -55,7 +55,8 @@ const ( PageTypeRootRecord = 1 PageTypeLeaf = 2 PageTypeBranch = 4 - PageTypeBitmapHeader = 8 // Only used by the WAL for marking next page + PageTypeBitmapHeader = 8 // Only used by the WAL for marking next page + PageTypeBitmap = 16 // Only used internally when walking the b-tree ) // Meta commit/rollback flags. @@ -73,6 +74,24 @@ const ( ContainerTypeBitmapPtr ) +// ContainerTypeString returns a string representation of the container type. +func ContainerTypeString(typ int) string { + switch typ { + case ContainerTypeNone: + return "none" + case ContainerTypeArray: + return "array" + case ContainerTypeRLE: + return "rle" + case ContainerTypeBitmap: + return "bitmap" + case ContainerTypeBitmapPtr: + return "bitmap-ptr" + default: + return fmt.Sprintf("unknown<%d>", typ) + } +} + const ( rootRecordPageHeaderSize = 12 rootRecordHeaderSize = 4 + 2 // pgno, len(name) @@ -333,16 +352,8 @@ func (c *leafCell) Values(tx *Tx) []uint16 { a = a[:n] return a case ContainerTypeBitmapPtr: - a := make([]uint16, 0, BitmapN*64) _, bm, _ := tx.leafCellBitmap(toPgno(c.Data)) - for i, v := range bm { - for j := uint(0); j < 64; j++ { - if v&(1<