From 7b872e00cb46128224e83df7a090a8c3e71b3d23 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 17:10:56 +0300 Subject: [PATCH 1/2] Implements #1570 --- client.go | 8 ++++++++ cmd/import.go | 2 ++ ctl/import.go | 19 +++++++++++++++++-- http/client.go | 29 ++++++++++++++++++++++++++--- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index a0b89f604..1b1b44e31 100644 --- a/client.go +++ b/client.go @@ -41,10 +41,12 @@ type InternalClient interface { ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error + EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error ImportValueK(ctx context.Context, index, field string, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error + CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) @@ -112,6 +114,9 @@ func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } +func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } @@ -122,6 +127,9 @@ func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, s return nil } func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { return nil, nil } diff --git a/cmd/import.go b/cmd/import.go index 7e82bfd46..5c05949f9 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -52,6 +52,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") + flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") + flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/ctl/import.go b/ctl/import.go index 2969b581b..83c95be84 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -40,9 +40,12 @@ type ImportCommand struct { // nolint: maligned Index string `json:"index"` Field string `json:"field"` - // Options for index & field to be created if they don't exist + // Options for the index to be created if it doesn't exist indexOptions pilosa.IndexOptions + // Options for the field to be created if it doesn't exist + fieldOptions pilosa.FieldOptions + // CreateSchema ensures the schema exists before import CreateSchema bool @@ -50,6 +53,12 @@ type ImportCommand struct { // nolint: maligned // TODO: remove this in a future release StringKeys bool `json:"StringKeys"` + // IndexKeys makes the import command use keys=true when creating an index + IndexKeys bool `json:"indexKeys"` + + // FieldKeys makes the import command use keys=true when creating a field + FieldKeys bool `json:"fieldKeys"` + // Filenames to import from. Paths []string `json:"paths"` @@ -102,6 +111,12 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { + cmd.indexOptions = pilosa.IndexOptions{ + Keys: cmd.IndexKeys, + } + cmd.fieldOptions = pilosa.FieldOptions{ + Keys: cmd.FieldKeys, + } err := cmd.ensureSchema(ctx) if err != nil { return errors.Wrap(err, "ensuring schema") @@ -146,7 +161,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) + err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } diff --git a/http/client.go b/http/client.go index d996b9e57..64a6675e0 100644 --- a/http/client.go +++ b/http/client.go @@ -372,7 +372,11 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p } func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - err := c.CreateField(ctx, indexName, fieldName) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) +} + +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { + err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { return nil } @@ -636,16 +640,35 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin return resp.Body, nil } -// CreateField creates a new field on the server. func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { if index == "" { return pilosa.ErrIndexRequired } + // convert pilosa.FieldOptions to fieldOptions + fieldOpt := fieldOptions{ + Type: opt.Type, + Keys: &opt.Keys, + } + if fieldOpt.Type == "set" { + fieldOpt.CacheType = &opt.CacheType + fieldOpt.CacheSize = &opt.CacheSize + } else if fieldOpt.Type == "int" { + fieldOpt.Min = &opt.Min + fieldOpt.Max = &opt.Max + } else if fieldOpt.Type == "time" { + fieldOpt.TimeQuantum = &opt.TimeQuantum + } + // TODO: remove buf completely? (depends on whether importer needs to create specific field types) // Encode query request. buf, err := json.Marshal(&postFieldRequest{ - //Options: opt, + Options: fieldOpt, }) if err != nil { return errors.Wrap(err, "marshaling") From 48bd76b967e7893d4ccc81e18de54892177f0cfc Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 22:58:23 +0300 Subject: [PATCH 2/2] Removed --string-keys option from pilosa import --- cmd/import.go | 1 - ctl/import.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 5c05949f9..3d63586ce 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -51,7 +51,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") diff --git a/ctl/import.go b/ctl/import.go index 83c95be84..1c000dd73 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -49,10 +49,6 @@ type ImportCommand struct { // nolint: maligned // CreateSchema ensures the schema exists before import CreateSchema bool - // REMOVED: Indicates that the payload should be treated as string keys. - // TODO: remove this in a future release - StringKeys bool `json:"StringKeys"` - // IndexKeys makes the import command use keys=true when creating an index IndexKeys bool `json:"indexKeys"` @@ -89,11 +85,6 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // REMOVED: warning that --string-keys flag has been deprecated. - if cmd.StringKeys { - logger.Printf("REMOVED: The string-keys flag is no longer used.") - } - // Validate arguments. // Index and field are validated early before the files are parsed. if cmd.Index == "" {