diff --git a/client.go b/client.go index d6f1f959f..c13d44560 100644 --- a/client.go +++ b/client.go @@ -41,10 +41,12 @@ type InternalClient interface { ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error + EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error ImportValueK(ctx context.Context, index, field string, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error + CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) @@ -116,6 +118,9 @@ func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } +func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } @@ -126,6 +131,9 @@ func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, s return nil } func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { return nil, nil } diff --git a/cmd/check.go b/cmd/check.go index d7bd33afd..19e0519a8 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "os" "github.com/spf13/cobra" @@ -27,8 +26,8 @@ import ( var checker *ctl.CheckCommand -func newCheckCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) +func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command { + checker = ctl.NewCheckCommand(stdin, stdout, stderr) checkCmd := &cobra.Command{ Use: "check [path2]...", Short: "Do a consistency check on a pilosa data file.", diff --git a/cmd/config.go b/cmd/config.go index a877d12ef..73464ba65 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -28,7 +27,7 @@ import ( var conf *ctl.ConfigCommand func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) + conf = ctl.NewConfigCommand(stdin, stdout, stderr) Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "config", diff --git a/cmd/export.go b/cmd/export.go index 217e9f09c..6cfeb0d9c 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -26,8 +25,8 @@ import ( var Exporter *ctl.ExportCommand -func newExportCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) +func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + Exporter = ctl.NewExportCommand(stdin, stdout, stderr) exportCmd := &cobra.Command{ Use: "export", Short: "Export data from pilosa.", diff --git a/cmd/generate_config.go b/cmd/generate_config.go index 6421a1f4f..8c8553aa4 100644 --- a/cmd/generate_config.go +++ b/cmd/generate_config.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -26,8 +25,8 @@ import ( var generateConf *ctl.GenerateConfigCommand -func newGenerateConfigCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) +func newGenerateConfigCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command { + generateConf = ctl.NewGenerateConfigCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "generate-config", Short: "Print the default configuration.", diff --git a/cmd/import.go b/cmd/import.go index 7e82bfd46..3d63586ce 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -51,7 +51,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") + flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") + flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/cmd/inspect.go b/cmd/inspect.go index 4f5ff9301..570e7aae3 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "os" "github.com/spf13/cobra" @@ -27,8 +26,8 @@ import ( var inspector *ctl.InspectCommand -func newInspectCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) +func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + inspector = ctl.NewInspectCommand(stdin, stdout, stderr) inspectCmd := &cobra.Command{ Use: "inspect", diff --git a/ctl/import.go b/ctl/import.go index 2969b581b..1c000dd73 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -40,15 +40,20 @@ type ImportCommand struct { // nolint: maligned Index string `json:"index"` Field string `json:"field"` - // Options for index & field to be created if they don't exist + // Options for the index to be created if it doesn't exist indexOptions pilosa.IndexOptions + // Options for the field to be created if it doesn't exist + fieldOptions pilosa.FieldOptions + // CreateSchema ensures the schema exists before import CreateSchema bool - // REMOVED: Indicates that the payload should be treated as string keys. - // TODO: remove this in a future release - StringKeys bool `json:"StringKeys"` + // IndexKeys makes the import command use keys=true when creating an index + IndexKeys bool `json:"indexKeys"` + + // FieldKeys makes the import command use keys=true when creating a field + FieldKeys bool `json:"fieldKeys"` // Filenames to import from. Paths []string `json:"paths"` @@ -80,11 +85,6 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // REMOVED: warning that --string-keys flag has been deprecated. - if cmd.StringKeys { - logger.Printf("REMOVED: The string-keys flag is no longer used.") - } - // Validate arguments. // Index and field are validated early before the files are parsed. if cmd.Index == "" { @@ -102,6 +102,12 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { + cmd.indexOptions = pilosa.IndexOptions{ + Keys: cmd.IndexKeys, + } + cmd.fieldOptions = pilosa.FieldOptions{ + Keys: cmd.FieldKeys, + } err := cmd.ensureSchema(ctx) if err != nil { return errors.Wrap(err, "ensuring schema") @@ -146,7 +152,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) + err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } diff --git a/http/client.go b/http/client.go index 7cc941c16..b3fd0ea83 100644 --- a/http/client.go +++ b/http/client.go @@ -372,7 +372,11 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p } func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - err := c.CreateField(ctx, indexName, fieldName) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) +} + +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { + err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { return nil } @@ -666,16 +670,35 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin return resp.Body, nil } -// CreateField creates a new field on the server. func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { if index == "" { return pilosa.ErrIndexRequired } + // convert pilosa.FieldOptions to fieldOptions + fieldOpt := fieldOptions{ + Type: opt.Type, + Keys: &opt.Keys, + } + if fieldOpt.Type == "set" { + fieldOpt.CacheType = &opt.CacheType + fieldOpt.CacheSize = &opt.CacheSize + } else if fieldOpt.Type == "int" { + fieldOpt.Min = &opt.Min + fieldOpt.Max = &opt.Max + } else if fieldOpt.Type == "time" { + fieldOpt.TimeQuantum = &opt.TimeQuantum + } + // TODO: remove buf completely? (depends on whether importer needs to create specific field types) // Encode query request. buf, err := json.Marshal(&postFieldRequest{ - //Options: opt, + Options: fieldOpt, }) if err != nil { return errors.Wrap(err, "marshaling") diff --git a/test/pilosa.go b/test/pilosa.go index f6ed907ef..e7111ad92 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -129,40 +129,40 @@ func (m *Command) Reopen() error { // MustCreateIndex uses this command's API to create an index and fails the test // if there is an error. -func (m *Command) MustCreateIndex(t *testing.T, name string, opts pilosa.IndexOptions) *pilosa.Index { +func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexOptions) *pilosa.Index { idx, err := m.API.CreateIndex(context.Background(), name, opts) if err != nil { - t.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err) + tb.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err) } return idx } // MustCreateField uses this command's API to create the field. The index must // already exist - it fails the test if there is an error. -func (m *Command) MustCreateField(t *testing.T, index, field string, opts ...pilosa.FieldOption) *pilosa.Field { +func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pilosa.FieldOption) *pilosa.Field { f, err := m.API.CreateField(context.Background(), index, field, opts...) if err != nil { - t.Fatalf("creating field: %s in index: %s err: %v", field, index, err) + tb.Fatalf("creating field: %s in index: %s err: %v", field, index, err) } return f } // MustQuery uses this command's API to execute the given query request, failing // if Query returns a non-nil error, otherwise returning the QueryResponse. -func (m *Command) MustQuery(t *testing.T, req *pilosa.QueryRequest) pilosa.QueryResponse { +func (m *Command) MustQuery(tb testing.TB, req *pilosa.QueryRequest) pilosa.QueryResponse { resp, err := m.API.Query(context.Background(), req) if err != nil { - t.Fatalf("making query: %v, err: %v", req, err) + tb.Fatalf("making query: %v, err: %v", req, err) } return resp } // MustRecalculateCaches calls RecalculateCaches on the command's API, and fails // if there is an error. -func (m *Command) MustRecalculateCaches(t *testing.T) { +func (m *Command) MustRecalculateCaches(tb testing.TB) { err := m.API.RecalculateCaches(context.Background()) if err != nil { - t.Fatalf("recalcluating caches: %v", err) + tb.Fatalf("recalcluating caches: %v", err) } } @@ -224,10 +224,10 @@ func (c Cluster) Close() error { } // MustNewCluster creates a new cluster -func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { +func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { c, err := newCluster(size, opts...) if err != nil { - t.Fatalf("new cluster: %v", err) + tb.Fatalf("new cluster: %v", err) } return c } @@ -271,10 +271,10 @@ func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { } // MustRunCluster creates and starts a new cluster -func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { +func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { c, err := runCluster(size, opts...) if err != nil { - t.Fatalf("run cluster: %v", err) + tb.Fatalf("run cluster: %v", err) } return c }