mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
5dcabfcc7f
This commit adds a Decimal field type which is implemented mostly with the Int field. It adds an optional "Scale" value to the Int field which means that the values stored in that field are actually meant to be divided by 10^Scale before being interpreted. In order to make use of this functionality, we extend the importValue request to allow a slice of floats rather than just int64. If the slice of floats is present, each float in the slice is multiplied by 10^Scale and converted to an int64 before being imported. If a slice of int64 is imported to a Decimal field, it is treated normally, and scale is ignored. This allows the conversion to be handled at the client side if desired. Currently there are Field level methods for querying Float values out of a decimal field, but no support in PQL or the executor for getting float values. Going to wait until I can use the generic result type before doing that, so for now, any values queried will be the scaled integer values. needed to add client support for importing float values, and did this by adding a more general and simplified client method for value imports. rewrote api.ImportValue to use the new method which should be more performant and efficient. allow floats to be "pilosa import"ed into decimal fields
69 lines
3.5 KiB
Go
69 lines
3.5 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
|
|
"github.com/pilosa/pilosa/v2"
|
|
"github.com/pilosa/pilosa/v2/ctl"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
var Importer *ctl.ImportCommand
|
|
|
|
// newImportCommand runs the Pilosa import subcommand for ingesting bulk data.
|
|
func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
|
|
Importer = ctl.NewImportCommand(stdin, stdout, stderr)
|
|
importCmd := &cobra.Command{
|
|
Use: "import",
|
|
Short: "Bulk load data into pilosa.",
|
|
Long: `Bulk imports one or more CSV files to a host's index and field. The data
|
|
of the CSV file are grouped by shard for the most efficient import.
|
|
|
|
The format of the CSV file is:
|
|
|
|
ROWID,COLUMNID,[TIME]
|
|
|
|
The file should contain no headers. The TIME column is optional and can be
|
|
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
|
|
`,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
Importer.Paths = args
|
|
return Importer.Run(context.Background())
|
|
},
|
|
}
|
|
|
|
flags := importCmd.Flags()
|
|
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
|
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
|
|
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
|
|
flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
|
|
flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
|
|
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
|
|
flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
|
|
flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
|
|
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
|
|
flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
|
|
flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
|
|
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
|
|
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
|
|
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
|
|
flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
|
|
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
|
|
|
|
return importCmd
|
|
}
|