Merge pull request #1625 from yuce/1623-pilosa-import-field-options

Added field options to pilosa import

cmd/import.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"io"
 
+	"github.com/pilosa/pilosa"
+
 	"github.com/spf13/cobra"
 
 	"github.com/pilosa/pilosa/ctl"
@@ -51,14 +53,16 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
 	flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
-	flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index")
-	flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field")
+	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
+	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
+	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
+	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
+	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
 	flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
-	//flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field")
-	//flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked")
-	//flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field")
 	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)
 
 	return importCmd

cmd/import_test.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/pilosa/pilosa"
+
 	"github.com/pilosa/pilosa/cmd"
 )
 
@@ -47,6 +49,52 @@ field = "f1"
 				return v.Error()
 			},
 		},
+		{
+			args: []string{"import", "--index", "i1", "--field", "f1", "--field-keys", "--field-min", "-10", "--field-max", "100"},
+			env:  map[string]string{},
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Importer.Index, "i1")
+				v.Check(cmd.Importer.Field, "f1")
+				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
+					Keys:      true,
+					Max:       100,
+					Min:       -10,
+					CacheType: pilosa.CacheTypeRanked,
+					CacheSize: 50000,
+				})
+				return v.Error()
+			},
+		},
+		{
+			args: []string{"import", "--index", "i1", "--field", "f1", "--field-time-quantum", "YMD"},
+			env:  map[string]string{},
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Importer.Index, "i1")
+				v.Check(cmd.Importer.Field, "f1")
+				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
+					TimeQuantum: "YMD",
+					CacheType:   pilosa.CacheTypeRanked,
+					CacheSize:   50000,
+				})
+				return v.Error()
+			},
+		},
+		{
+			args: []string{"import", "--index", "i1", "--field", "f1", "--field-cache-type", "lru", "--field-cache-size", "100"},
+			env:  map[string]string{},
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Importer.Index, "i1")
+				v.Check(cmd.Importer.Field, "f1")
+				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
+					CacheType: "lru",
+					CacheSize: 100,
+				})
+				return v.Error()
+			},
+		},
 	}
 	executeDry(t, tests)
 }

ctl/import.go

@@ -41,20 +41,14 @@ type ImportCommand struct { // nolint: maligned
 	Field string `json:"field"`
 
 	// Options for the index to be created if it doesn't exist
-	indexOptions pilosa.IndexOptions
+	IndexOptions pilosa.IndexOptions
 
 	// Options for the field to be created if it doesn't exist
-	fieldOptions pilosa.FieldOptions
+	FieldOptions pilosa.FieldOptions
 
 	// CreateSchema ensures the schema exists before import
 	CreateSchema bool
 
-	// IndexKeys makes the import command use keys=true when creating an index
-	IndexKeys bool `json:"indexKeys"`
-
-	// FieldKeys makes the import command use keys=true when creating a field
-	FieldKeys bool `json:"fieldKeys"`
-
 	// Filenames to import from.
 	Paths []string `json:"paths"`
 
@@ -102,11 +96,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	cmd.client = client
 
 	if cmd.CreateSchema {
-		cmd.indexOptions = pilosa.IndexOptions{
-			Keys: cmd.IndexKeys,
-		}
-		cmd.fieldOptions = pilosa.FieldOptions{
-			Keys: cmd.FieldKeys,
+		// set the correct type for the field
+		if cmd.FieldOptions.TimeQuantum != "" {
+			cmd.FieldOptions.Type = "time"
+		} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
+			cmd.FieldOptions.Type = "int"
+		} else {
+			cmd.FieldOptions.Type = "set"
 		}
 		err := cmd.ensureSchema(ctx)
 		if err != nil {
@@ -148,11 +144,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 }
 
 func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
-	err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions)
+	err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions)
 	if err != nil {
 		return fmt.Errorf("Error Creating Index: %s", err)
 	}
-	err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions)
+	err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.FieldOptions)
 	if err != nil {
 		return fmt.Errorf("Error Creating Field: %s", err)
 	}