diff --git a/client.go b/client.go index a0b89f604..1b1b44e31 100644 --- a/client.go +++ b/client.go @@ -41,10 +41,12 @@ type InternalClient interface { ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error + EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error ImportValueK(ctx context.Context, index, field string, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error + CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) @@ -112,6 +114,9 @@ func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } +func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } @@ -122,6 +127,9 @@ func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, s return nil } func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { return nil, nil } diff --git a/cmd/import.go b/cmd/import.go index 7e82bfd46..3d63586ce 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -51,7 +51,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") + flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") + flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/ctl/import.go b/ctl/import.go index 2969b581b..1c000dd73 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -40,15 +40,20 @@ type ImportCommand struct { // nolint: maligned Index string `json:"index"` Field string `json:"field"` - // Options for index & field to be created if they don't exist + // Options for the index to be created if it doesn't exist indexOptions pilosa.IndexOptions + // Options for the field to be created if it doesn't exist + fieldOptions pilosa.FieldOptions + // CreateSchema ensures the schema exists before import CreateSchema bool - // REMOVED: Indicates that the payload should be treated as string keys. - // TODO: remove this in a future release - StringKeys bool `json:"StringKeys"` + // IndexKeys makes the import command use keys=true when creating an index + IndexKeys bool `json:"indexKeys"` + + // FieldKeys makes the import command use keys=true when creating a field + FieldKeys bool `json:"fieldKeys"` // Filenames to import from. Paths []string `json:"paths"` @@ -80,11 +85,6 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // REMOVED: warning that --string-keys flag has been deprecated. - if cmd.StringKeys { - logger.Printf("REMOVED: The string-keys flag is no longer used.") - } - // Validate arguments. // Index and field are validated early before the files are parsed. if cmd.Index == "" { @@ -102,6 +102,12 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { + cmd.indexOptions = pilosa.IndexOptions{ + Keys: cmd.IndexKeys, + } + cmd.fieldOptions = pilosa.FieldOptions{ + Keys: cmd.FieldKeys, + } err := cmd.ensureSchema(ctx) if err != nil { return errors.Wrap(err, "ensuring schema") @@ -146,7 +152,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) + err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } diff --git a/http/client.go b/http/client.go index d996b9e57..64a6675e0 100644 --- a/http/client.go +++ b/http/client.go @@ -372,7 +372,11 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p } func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - err := c.CreateField(ctx, indexName, fieldName) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) +} + +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { + err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { return nil } @@ -636,16 +640,35 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin return resp.Body, nil } -// CreateField creates a new field on the server. func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { if index == "" { return pilosa.ErrIndexRequired } + // convert pilosa.FieldOptions to fieldOptions + fieldOpt := fieldOptions{ + Type: opt.Type, + Keys: &opt.Keys, + } + if fieldOpt.Type == "set" { + fieldOpt.CacheType = &opt.CacheType + fieldOpt.CacheSize = &opt.CacheSize + } else if fieldOpt.Type == "int" { + fieldOpt.Min = &opt.Min + fieldOpt.Max = &opt.Max + } else if fieldOpt.Type == "time" { + fieldOpt.TimeQuantum = &opt.TimeQuantum + } + // TODO: remove buf completely? (depends on whether importer needs to create specific field types) // Encode query request. buf, err := json.Marshal(&postFieldRequest{ - //Options: opt, + Options: fieldOpt, }) if err != nil { return errors.Wrap(err, "marshaling")