scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 04:47:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1621 from yuce/1570-import-cmd-keys-options

Browse source 
Adds --field-keys and --index-keys options to pilosa import
This commit is contained in:
Yuce Tekol 2018-09-06 05:56:04 +03:00 • committed by GitHub
commit 0ac1e25a07
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 52 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
client.go
View file

@ -41,10 +41,12 @@ type InternalClient interface {
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureField(ctx context.Context, indexName string, fieldName string) error
EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error
ImportValueK(ctx context.Context, index, field string, vals []FieldValue) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
@ -112,6 +114,9 @@ func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options
func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error {
return nil
}
@ -122,6 +127,9 @@ func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, s
return nil
}
func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}

3
cmd/import.go
View file

@ -51,7 +51,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.")
flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index")
flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")

26
ctl/import.go
View file

@ -40,15 +40,20 @@ type ImportCommand struct { // nolint: maligned
Index string `json:"index"`
Field string `json:"field"`
// Options for index & field to be created if they don't exist
// Options for the index to be created if it doesn't exist
indexOptions pilosa.IndexOptions
// Options for the field to be created if it doesn't exist
fieldOptions pilosa.FieldOptions
// CreateSchema ensures the schema exists before import
CreateSchema bool
// REMOVED: Indicates that the payload should be treated as string keys.
// TODO: remove this in a future release
StringKeys bool `json:"StringKeys"`
// IndexKeys makes the import command use keys=true when creating an index
IndexKeys bool `json:"indexKeys"`
// FieldKeys makes the import command use keys=true when creating a field
FieldKeys bool `json:"fieldKeys"`
// Filenames to import from.
Paths []string `json:"paths"`
@ -80,11 +85,6 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand
func (cmd *ImportCommand) Run(ctx context.Context) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// REMOVED: warning that --string-keys flag has been deprecated.
if cmd.StringKeys {
logger.Printf("REMOVED: The string-keys flag is no longer used.")
}
// Validate arguments.
// Index and field are validated early before the files are parsed.
if cmd.Index == "" {
@ -102,6 +102,12 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
cmd.client = client
if cmd.CreateSchema {
cmd.indexOptions = pilosa.IndexOptions{
Keys: cmd.IndexKeys,
}
cmd.fieldOptions = pilosa.FieldOptions{
Keys: cmd.FieldKeys,
}
err := cmd.ensureSchema(ctx)
if err != nil {
return errors.Wrap(err, "ensuring schema")
@ -146,7 +152,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
if err != nil {
return fmt.Errorf("Error Creating Index: %s", err)
}
err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field)
err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions)
if err != nil {
return fmt.Errorf("Error Creating Field: %s", err)
}

29
http/client.go
View file

@ -372,7 +372,11 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p
}
func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
err := c.CreateField(ctx, indexName, fieldName)
return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{})
}
func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error {
err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt)
if err == nil || errors.Cause(err) == pilosa.ErrFieldExists {
return nil
}
@ -636,16 +640,35 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin
return resp.Body, nil
}
// CreateField creates a new field on the server.
func (c *InternalClient) CreateField(ctx context.Context, index, field string) error {
return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{})
}
// CreateField creates a new field on the server.
func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error {
if index == "" {
return pilosa.ErrIndexRequired
}
// convert pilosa.FieldOptions to fieldOptions
fieldOpt := fieldOptions{
Type: opt.Type,
Keys: &opt.Keys,
}
if fieldOpt.Type == "set" {
fieldOpt.CacheType = &opt.CacheType
fieldOpt.CacheSize = &opt.CacheSize
} else if fieldOpt.Type == "int" {
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
} else if fieldOpt.Type == "time" {
fieldOpt.TimeQuantum = &opt.TimeQuantum
}
// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
// Encode query request.
buf, err := json.Marshal(&postFieldRequest{
//Options: opt,
Options: fieldOpt,
})
if err != nil {
return errors.Wrap(err, "marshaling")