From e20b9075e26d675812588e2e6bb0168350b3b519 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 11:17:48 -0500 Subject: [PATCH 1/4] #413 set standard view as default for export command --- client.go | 9 ++++++--- cmd/export.go | 1 + ctl/export.go | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index 96e725876..22ea7d59c 100644 --- a/client.go +++ b/client.go @@ -389,11 +389,13 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired + } else if view == "" { + view = ViewStandard } // Retrieve a list of nodes that own the slice. @@ -407,7 +409,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) continue } else { @@ -419,7 +421,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 } // exportNode copies a CSV export from a node to w. -func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { // Create URL. u := url.URL{ Scheme: "http", @@ -428,6 +430,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame str RawQuery: url.Values{ "index": {index}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } diff --git a/cmd/export.go b/cmd/export.go index 310e3f05c..3572ffb7c 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -53,6 +53,7 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") + flags.StringVarP(&Exporter.View, "view", "v", "", "View to export into - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") return exportCmd diff --git a/ctl/export.go b/ctl/export.go index 8f67b7896..3ad665551 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -31,7 +31,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - + View string // Filename to export to. Path string @@ -55,6 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired + } else if cmd.View == "" { + cmd.View = pilosa.ViewStandard } // Use output file, if specified. @@ -85,7 +87,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { return err } } From b635bdf970431a0f375bbcc96a508c1fe066ebea Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 12:54:49 -0500 Subject: [PATCH 2/4] #413 add tests and remove redundant word --- client.go | 4 ++-- cmd/export.go | 6 +++--- cmd/export_test.go | 8 ++++++++ ctl/export.go | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/client.go b/client.go index 22ea7d59c..323eef9c3 100644 --- a/client.go +++ b/client.go @@ -394,8 +394,8 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if view == "" { - view = ViewStandard + } else if view != ViewInverse || view != ViewStandard { + return ErrInvalidView } // Retrieve a list of nodes that own the slice. diff --git a/cmd/export.go b/cmd/export.go index 3572ffb7c..e55f84e59 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -51,9 +51,9 @@ The file does not contain any headers. flags := exportCmd.Flags() flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") - flags.StringVarP(&Exporter.View, "view", "v", "", "View to export into - default standard") + flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") + flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") return exportCmd diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..2d01cf4d6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -44,6 +44,7 @@ frame = "f1" v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.View, "standard") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, @@ -51,3 +52,10 @@ frame = "f1" } executeDry(t, tests) } + +func TestExportInvalidView(t *testing.T) { + output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test") + if !strings.Contains(err.Error(), "invalid view") { + t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output) + } +} diff --git a/ctl/export.go b/ctl/export.go index 3ad665551..8b2d6cd35 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -55,8 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if cmd.View == "" { - cmd.View = pilosa.ViewStandard + } else if cmd.View != pilosa.ViewStandard || cmd.View != pilosa.ViewInverse { + return pilosa.ErrInvalidView } // Use output file, if specified. From 384329bc911850d156428368dc22e2edb1df773b Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 13:33:44 -0500 Subject: [PATCH 3/4] fix view always return error --- client.go | 10 +++++++++- ctl/export.go | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/client.go b/client.go index 323eef9c3..ef7a28a72 100644 --- a/client.go +++ b/client.go @@ -394,7 +394,15 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if view != ViewInverse || view != ViewStandard { + } + foundView := false + for _, v := range []string{ViewInverse, ViewStandard} { + if view == v { + foundView = true + break + } + } + if foundView == false { return ErrInvalidView } diff --git a/ctl/export.go b/ctl/export.go index 8b2d6cd35..63266a8d6 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -49,13 +49,22 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand // Run executes the export. func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) + foundView := false // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if cmd.View != pilosa.ViewStandard || cmd.View != pilosa.ViewInverse { + } + + for _, v := range []string{pilosa.ViewInverse, pilosa.ViewStandard} { + if cmd.View == v { + foundView = true + break + } + } + if foundView == false { return pilosa.ErrInvalidView } From 5de9a7d0d0264667689c34039d69ddf52204a6b8 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 14:56:47 -0500 Subject: [PATCH 4/4] refactor view validator --- client.go | 10 +--------- ctl/export.go | 11 +---------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/client.go b/client.go index ef7a28a72..058d6d836 100644 --- a/client.go +++ b/client.go @@ -394,15 +394,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } - foundView := false - for _, v := range []string{ViewInverse, ViewStandard} { - if view == v { - foundView = true - break - } - } - if foundView == false { + } else if !(view == ViewStandard || view == ViewInverse) { return ErrInvalidView } diff --git a/ctl/export.go b/ctl/export.go index 63266a8d6..7da310623 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -49,22 +49,13 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand // Run executes the export. func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - foundView := false // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } - - for _, v := range []string{pilosa.ViewInverse, pilosa.ViewStandard} { - if cmd.View == v { - foundView = true - break - } - } - if foundView == false { + } else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) { return pilosa.ErrInvalidView }