scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 07:01:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

support for decimal fields

Browse source 
This commit adds a Decimal field type which is implemented mostly with
the Int field. It adds an optional "Scale" value to the Int field
which means that the values stored in that field are actually meant to
be divided by 10^Scale before being interpreted.

In order to make use of this functionality, we extend the importValue
request to allow a slice of floats rather than just int64. If the
slice of floats is present, each float in the slice is multiplied by
10^Scale and converted to an int64 before being imported. If a slice
of int64 is imported to a Decimal field, it is treated normally, and
scale is ignored. This allows the conversion to be handled at the
client side if desired.

Currently there are Field level methods for querying Float values out
of a decimal field, but no support in PQL or the executor for getting
float values. Going to wait until I can use the generic result type
before doing that, so for now, any values queried will be the scaled
integer values.

needed to add client support for importing float values, and did this
by adding a more general and simplified client method for value
imports.

rewrote api.ImportValue to use the new method which should be more
performant and efficient.

allow floats to be "pilosa import"ed into decimal fields
This commit is contained in:
Matt Jaffee 2019-10-25 17:56:06 -05:00
parent 4ef7f7e26b
commit 5dcabfcc7f
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
18 changed files with 3468 additions and 715 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
api.go
View file

@ -23,7 +23,9 @@ import (
"fmt"
"io"
"io/ioutil"
"math"
"net/url"
"sort"
"strconv"
"strings"
"sync"
@ -890,10 +892,17 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
return attrs, nil
}
// ImportOptions holds the options for the API.Import method.
// ImportOptions holds the options for the API.Import
// method.
//
// TODO(2.0) we have entirely missed the point of functional options
// by exporting this structure. If it needs to be exported for some
// reason, we should consider not using functional options here which
// just adds complexity.
type ImportOptions struct {
Clear bool
IgnoreKeyCheck bool
Presorted bool
}
// ImportOption is a functional option type for API.Import.
@ -917,6 +926,13 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption {
}
}
func OptImportOptionsPresorted(b bool) ImportOption {
return func(o *ImportOptions) error {
o.Presorted = b
return nil
}
}
// Import bulk imports data into a particular index,field,shard.
func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
@ -1037,6 +1053,10 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
return errors.Wrap(err, "validating api method")
}
if err := req.Validate(); err != nil {
return errors.Wrap(err, "validating import value request")
}
// Set up import options.
options, err := setUpImportOptions(opts...)
if err != nil {
@ -1060,57 +1080,93 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
return errors.Wrap(err, "translating columns")
}
// For translated data, map the columnIDs to shards. If
// this node does not own the shard, forward to the node that does.
m := make(map[uint64][]FieldValue)
for i, colID := range req.ColumnIDs {
shard := colID / ShardWidth
if _, ok := m[shard]; !ok {
m[shard] = make([]FieldValue, 0)
}
m[shard] = append(m[shard], FieldValue{
Value: req.Values[i],
ColumnID: colID,
})
}
// Signal to the receiving nodes to ignore checking for key translation.
opts = append(opts, OptImportOptionsIgnoreKeyCheck(true))
var eg errgroup.Group
for shard, vals := range m {
// TODO: if local node owns this shard we don't need to go through the client
shard := shard
vals := vals
eg.Go(func() error {
return api.server.defaultClient.ImportValue(ctx, req.Index, req.Field, shard, vals, opts...)
})
}
return eg.Wait()
req.Shard = math.MaxUint64
}
}
// Validate shard ownership.
if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
return errors.Wrap(err, "validating shard ownership")
if !options.Presorted {
sort.Sort(req)
}
// Import columnIDs into existence field.
if !options.Clear {
if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
return errors.Wrap(err, "importing existence columns")
// if we're importing into a specific shard
if req.Shard != math.MaxUint64 {
// Check that column IDs match the stated shard.
if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard {
return errors.Errorf("shard %d specified, but import spans shards %d to %d", req.Shard, s1, s2)
}
// Validate shard ownership. TODO - we should forward to the
// correct node rather than barfing here.
if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
return errors.Wrap(err, "validating shard ownership")
}
// Import columnIDs into existence field.
if !options.Clear {
if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
return errors.Wrap(err, "importing existence columns")
}
}
// Import into fragment.
if len(req.Values) > 0 {
err = field.importValue(req.ColumnIDs, req.Values, options)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
} else if len(req.FloatValues) > 0 {
err = field.importFloatValue(req.ColumnIDs, req.FloatValues, options)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
}
return errors.Wrap(err, "importing")
}
options.IgnoreKeyCheck = true
start := 0
shard := req.ColumnIDs[0] / ShardWidth
var eg errgroup.Group // TODO make this a pooled errgroup
for i, colID := range req.ColumnIDs {
if colID/ShardWidth != shard {
subreq := &ImportValueRequest{
Index: req.Index,
Field: req.Field,
Shard: shard,
ColumnIDs: req.ColumnIDs[start:i],
}
if req.Values != nil {
subreq.Values = req.Values[start:i]
} else if req.FloatValues != nil {
subreq.FloatValues = req.FloatValues[start:i]
}
eg.Go(func() error {
return api.server.defaultClient.ImportValue2(ctx, subreq, options)
})
start = i
shard = colID / ShardWidth
}
}
// Import into fragment.
err = field.importValue(req.ColumnIDs, req.Values, options)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
subreq := &ImportValueRequest{
Index: req.Index,
Field: req.Field,
Shard: shard,
ColumnIDs: req.ColumnIDs[start:],
}
return errors.Wrap(err, "importing")
if req.Values != nil {
subreq.Values = req.Values[start:]
} else if req.FloatValues != nil {
subreq.FloatValues = req.FloatValues[start:]
}
eg.Go(func() error {
// TODO we should elevate the logic for figuring out which
// node(s) to send to into API instead of having those details
// in the client implementation.
return api.server.defaultClient.ImportValue2(ctx, subreq, options)
})
return eg.Wait()
}
func importExistenceColumns(index *Index, columnIDs []uint64) error {

141
api_test.go
View file

@ -258,6 +258,147 @@ func TestAPI_ImportValue(t *testing.T) {
t.Fatal(err)
}
})
t.Run("ValDecimalField", func(t *testing.T) {
ctx := context.Background()
index := "valdec"
field := "fdec"
_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
fld, err := m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
if err != nil {
t.Fatalf("creating field: %v", err)
}
// Generate some keyed records.
values := []float64{}
colIDs := []uint64{}
for i := 0; i < 10; i++ {
values = append(values, float64(i)+0.1)
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
ColumnIDs: colIDs,
FloatValues: values,
}
if err := m1.API.ImportValue(ctx, req); err != nil {
t.Fatal(err)
}
pql := fmt.Sprintf("Row(%s>60)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
t.Fatalf("unexpected column keys: %+v", ids)
}
sum, count, err := fld.FloatSum(nil, field)
if err != nil {
t.Fatalf("getting floatsum: %v", err)
} else if sum != 0.1+1.1+2.1+3.1+4.1+5.1+6.1+7.1+8.1+9.1 {
t.Fatalf("unexpected sum: %f", sum)
} else if count != 10 {
t.Fatalf("unexpected count: %d", count)
}
min, count, err := fld.FloatMin(nil, field)
if err != nil {
t.Fatalf("getting floatmin: %v", err)
} else if min != 0.1 {
t.Fatalf("unexpected min: %f", min)
} else if count != 1 {
t.Fatalf("unexpected count: %d", count)
}
max, count, err := fld.FloatMax(nil, field)
if err != nil {
t.Fatalf("getting floatmax: %v", err)
} else if max != 9.1 {
t.Fatalf("unexpected max: %f", max)
} else if count != 1 {
t.Fatalf("unexpected count: %d", count)
}
val, exists, err := fld.FloatValue(1)
if err != nil {
t.Fatalf("unepxected err getting floatvalue")
} else if !exists {
t.Fatalf("column 1 should exist")
} else if val != 1.1 {
t.Fatalf("unexpected floatvalue %f", val)
}
changed, err := fld.SetFloatValue(11, 11.1)
if err != nil {
t.Fatalf("setting float value: %v", err)
} else if !changed {
t.Fatalf("expected change")
}
val, exists, err = fld.FloatValue(11)
if err != nil {
t.Fatalf("getting float val: %v", err)
} else if !exists {
t.Fatalf("should exist")
} else if val != 11.1 {
t.Fatalf("unexpected val: %f", 11.1)
}
})
t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
ctx := context.Background()
index := "valdecneg"
field := "fdecneg"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
if err != nil {
t.Fatalf("creating field: %v", err)
}
// Generate some keyed records.
values := []float64{}
colIDs := []uint64{}
for i := 0; i < 10; i++ {
values = append(values, float64(i)*100+10)
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
ColumnIDs: colIDs,
FloatValues: values,
}
if err := m1.API.ImportValue(ctx, req); err != nil {
t.Fatal(err)
}
pql := fmt.Sprintf("Row(%s>60)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
t.Fatalf("unexpected column keys: %+v", ids)
}
})
}
// offsetModHasher represents a simple, mod-based hashing offset by 1.

5
client.go
View file

@ -59,6 +59,7 @@ type InternalClient interface {
EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@ -129,6 +130,10 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar
func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}

2
cmd/import.go
View file

@ -53,7 +53,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")

101
ctl/import.go
View file

@ -20,6 +20,7 @@ import (
"fmt"
"io"
"log"
"math"
"os"
"sort"
"strconv"
@ -102,11 +103,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
if cmd.FieldOptions.Type == "" {
// set the correct type for the field
if cmd.FieldOptions.TimeQuantum != "" {
cmd.FieldOptions.Type = "time"
cmd.FieldOptions.Type = pilosa.FieldTypeTime
} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
cmd.FieldOptions.Type = "int"
cmd.FieldOptions.Type = pilosa.FieldTypeInt
} else {
cmd.FieldOptions.Type = "set"
cmd.FieldOptions.Type = pilosa.FieldTypeSet
}
}
err := cmd.ensureSchema(ctx)
@ -163,8 +164,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
// If fieldType is `int`, treat the import data as values to be range-encoded.
if fieldType == pilosa.FieldTypeInt {
return cmd.bufferValues(ctx, useColumnKeys, path)
if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
}
return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
}
@ -285,9 +286,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
return nil
}
// bufferValues buffers slices of FieldValues to be imported as a batch.
func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
// bufferValues buffers slices of record identifiers and values to be imported as a batch.
func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
req := &pilosa.ImportValueRequest{
Index: cmd.Index,
Field: cmd.Field,
Shard: math.MaxUint64,
}
var r *csv.Reader
@ -307,6 +312,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
r.FieldsPerRecord = -1
rnum := 0
for {
rnum++
@ -325,69 +331,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
}
var val pilosa.FieldValue
// Parse column id.
if useColumnKeys {
val.ColumnKey = record[0]
req.ColumnKeys = append(req.ColumnKeys, record[0])
} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
req.ColumnIDs = append(req.ColumnIDs, columnID)
} else if err != nil {
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
}
// Parse value.
if parseAsFloat {
value, err := strconv.ParseFloat(record[1], 64)
if err != nil {
return errors.Wrapf(err, "parseing value '%s' as float", record[1])
}
req.FloatValues = append(req.FloatValues, value)
} else {
if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
value, err := strconv.ParseInt(record[1], 10, 64)
if err != nil {
return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
}
req.Values = append(req.Values, value)
}
// Parse FieldValue.
value, err := strconv.ParseInt(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
}
val.Value = value
a = append(a, val)
// If we've reached the buffer size then import FieldValues.
if len(a) == cmd.BufferSize {
if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
return err
// If we've reached the buffer size then import the batch.
if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
return errors.Wrap(err, "importing values")
}
a = a[:0]
req.ColumnIDs = req.ColumnIDs[:0]
req.ColumnKeys = req.ColumnKeys[:0]
req.Values = req.Values[:0]
req.FloatValues = req.FloatValues[:0]
}
}
// If there are still values in the buffer then flush them.
return cmd.importValues(ctx, useColumnKeys, a)
}
// importValues sends batches of FieldValues to the server.
func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
if useColumnKeys {
logger.Printf("importing keyed values: n=%d", len(vals))
if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
return errors.Wrap(err, "importing keys")
}
return nil
}
// Group vals by shard.
logger.Printf("grouping %d vals", len(vals))
valsByShard := http.FieldValues(vals).GroupByShard()
// Parse path into FieldValues.
for shard, vals := range valsByShard {
if cmd.Sort {
sort.Sort(http.FieldValues(vals))
}
logger.Printf("importing shard: %d, n=%d", shard, len(vals))
if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
return errors.Wrap(err, "importing values")
}
}
return nil
return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
}
func (cmd *ImportCommand) TLSHost() string {

2
diagnostics.go
View file

@ -233,7 +233,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
numIndexes++
for _, field := range index.Fields() {
numFields++
if field.Type() == FieldTypeInt {
if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
bsiFieldCount++
}
if field.TimeQuantum() != "" {

16
encoding/proto/proto.go
View file

@ -369,12 +369,13 @@ func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
return &internal.ImportValueRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
}
}
@ -538,6 +539,7 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
Min: o.Min,
Max: o.Max,
Base: o.Base,
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
@ -808,6 +810,7 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions)
m.Min = options.Min
m.Max = options.Max
m.Base = options.Base
m.Scale = options.Scale
m.BitDepth = uint(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
m.Keys = options.Keys
@ -983,6 +986,7 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV
m.ColumnIDs = pb.ColumnIDs
m.ColumnKeys = pb.ColumnKeys
m.Values = pb.Values
m.FloatValues = pb.FloatValues
}
func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {

4
executor.go
View file

@ -944,7 +944,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNShard: %v", err)
} else if f := e.Holder.Field(index, fieldName); f != nil && f.Type() == FieldTypeInt {
} else if f := e.Holder.Field(index, fieldName); f != nil && (f.Type() == FieldTypeInt || f.Type() == FieldTypeDecimal) {
return nil, fmt.Errorf("cannot compute TopN() on integer field: %q", fieldName)
}
@ -2109,7 +2109,7 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
}
// Int field.
if f.Type() == FieldTypeInt {
if f.Type() == FieldTypeInt || f.Type() == FieldTypeDecimal {
// Read row value.
rowVal, ok, err := c.IntArg(fieldName)
if err != nil {

138
field.go
View file

@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"math"
"os"
"path/filepath"
"sort"
@ -54,11 +55,12 @@ const (
// Field types.
const (
FieldTypeSet = "set"
FieldTypeInt = "int"
FieldTypeTime = "time"
FieldTypeMutex = "mutex"
FieldTypeBool = "bool"
FieldTypeSet = "set"
FieldTypeInt = "int"
FieldTypeTime = "time"
FieldTypeMutex = "mutex"
FieldTypeBool = "bool"
FieldTypeDecimal = "decimal"
)
// Field represents a container for views.
@ -155,6 +157,32 @@ func OptFieldTypeInt(min, max int64) FieldOption {
}
}
func OptFieldTypeDecimal(scale int64, minmax ...int64) FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("can't set field type to 'decimal', already set to: %s", fo.Type)
}
fo.Min = math.MinInt64
fo.Max = math.MaxInt64
if len(minmax) == 2 {
min, max := minmax[0], minmax[1]
if min > max {
return errors.Errorf("decimal field min cannot be greater than max, got %d, %d", min, max)
}
fo.Min = min
fo.Max = max
} else if len(minmax) > 2 {
return errors.Errorf("unknown extra parameters beyond min and max: %v", minmax)
} else if len(minmax) == 1 {
fo.Min = minmax[0]
}
fo.Type = FieldTypeDecimal
fo.Base = bsiBase(fo.Min, fo.Max)
fo.Scale = scale
return nil
}
}
// OptFieldTypeTime is a functional option on FieldOptions
// used to specify the field as being type `time` and to
// provide any respective configuration values.
@ -550,6 +578,7 @@ func (f *Field) loadMeta() error {
f.options.Min = pb.Min
f.options.Max = pb.Max
f.options.Base = pb.Base
f.options.Scale = pb.Scale
f.options.BitDepth = uint(pb.BitDepth)
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
@ -609,13 +638,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = 0
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
case FieldTypeInt:
case FieldTypeInt, FieldTypeDecimal:
f.options.Type = opt.Type
f.options.CacheType = CacheTypeNone
f.options.CacheSize = 0
f.options.Min = opt.Min
f.options.Max = opt.Max
f.options.Base = opt.Base
f.options.Scale = opt.Scale
f.options.BitDepth = opt.BitDepth
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
@ -627,6 +657,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
Min: opt.Min,
Max: opt.Max,
Base: opt.Base,
Scale: opt.Scale,
BitDepth: opt.BitDepth,
}
// Validate bsiGroup.
@ -1051,6 +1082,21 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) {
return me
}
// FloatValue reads an integer field value for a column, and converts
// it to a float based on the configured scale.
func (f *Field) FloatValue(columnID uint64) (value float64, exists bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return 0, false, ErrBSIGroupNotFound
}
val, exists, err := f.Value(columnID)
if exists {
value = float64(val) / math.Pow10(int(bsig.Scale))
}
return value, exists, err
}
// Value reads a field value for a column.
func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
bsig := f.bsiGroup(f.name)
@ -1073,6 +1119,18 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
return int64(v) + bsig.Base, true, nil
}
// SetFloatValue takes a floating point value, and converts it to an
// integer based on the field's configured scale, before setting that
// integer via SetValue.
func (f *Field) SetFloatValue(columnID uint64, value float64) (changed bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return false, ErrBSIGroupNotFound
}
val := int64(float64(value) * math.Pow10(int(bsig.Scale)))
return f.SetValue(columnID, val)
}
// SetValue sets a field value for a column.
func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) {
// Fetch bsiGroup & validate min/max.
@ -1118,6 +1176,22 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error)
return view.setValue(columnID, bsig.BitDepth, baseValue)
}
// FloatSum performs a Sum query and converts the result to a float
// based on the field's configured scale.
func (f *Field) FloatSum(filter *Row, name string) (sum float64, count int64, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return 0, 0, ErrBSIGroupNotFound
}
sumI, count, err := f.Sum(filter, name)
if err == nil {
sum = float64(sumI) / math.Pow10(int(bsig.Scale))
}
return sum, count, err
}
// Sum returns the sum and count for a field.
// An optional filtering row can be provided.
func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) {
@ -1138,6 +1212,21 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) {
return int64(vsum) + (int64(vcount) * bsig.Base), int64(vcount), nil
}
// FloatMin performs a Min query and converts the result to a float
// based on the field's configured scale.
func (f *Field) FloatMin(filter *Row, name string) (min float64, count int64, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return 0, 0, ErrBSIGroupNotFound
}
minI, count, err := f.Min(filter, name)
if err == nil {
min = float64(minI) / math.Pow10(int(bsig.Scale))
}
return min, count, err
}
// Min returns the min for a field.
// An optional filtering row can be provided.
func (f *Field) Min(filter *Row, name string) (min, count int64, err error) {
@ -1158,6 +1247,21 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) {
return int64(vmin) + bsig.Base, int64(vcount), nil
}
// FloatMax performs a max query and converts the result to a float
// based on the field's configured scale.
func (f *Field) FloatMax(filter *Row, name string) (max float64, count int64, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return 0, 0, ErrBSIGroupNotFound
}
maxI, count, err := f.Max(filter, name)
if err == nil {
max = float64(maxI) / math.Pow10(int(bsig.Scale))
}
return max, count, err
}
// Max returns the max for a field.
// An optional filtering row can be provided.
func (f *Field) Max(filter *Row, name string) (max, count int64, err error) {
@ -1283,6 +1387,21 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts
return nil
}
func (f *Field) importFloatValue(columnIDs []uint64, values []float64, options *ImportOptions) error {
// convert values to int64 values based on scale
ivalues := make([]int64, len(values))
bsig := f.bsiGroup(f.name)
if bsig == nil {
return errors.Wrap(ErrBSIGroupNotFound, f.name)
}
mult := math.Pow10(int(bsig.Scale))
for i, fval := range values {
ivalues[i] = int64(fval * mult)
}
// then call importValue
return f.importValue(columnIDs, ivalues, options)
}
// importValue bulk imports range-encoded value data.
func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportOptions) error {
viewName := viewBSIGroupPrefix + f.name
@ -1419,6 +1538,7 @@ type FieldOptions struct {
BitDepth uint `json:"bitDepth,omitempty"`
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
Scale int64 `json:"scale,omitempty"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
@ -1454,6 +1574,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Base: o.Base,
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
Min: o.Min,
Max: o.Max,
@ -1480,10 +1601,11 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
o.CacheSize,
o.Keys,
})
case FieldTypeInt:
case FieldTypeInt, FieldTypeDecimal:
return json.Marshal(struct {
Type string `json:"type"`
Base int64 `json:"base"`
Scale int64 `json:"scale"`
BitDepth uint `json:"bitDepth"`
Min int64 `json:"min"`
Max int64 `json:"max"`
@ -1491,6 +1613,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
}{
o.Type,
o.Base,
o.Scale,
o.BitDepth,
o.Min,
o.Max,
@ -1563,6 +1686,7 @@ type bsiGroup struct {
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
Base int64 `json:"base,omitempty"`
Scale int64 `json:"scale,omitempty"`
BitDepth uint `json:"bitDepth,omitempty"`
}

40
handler.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
)
// QueryRequest represent a request to process a query.
@ -107,12 +108,39 @@ var NopHandler Handler = nopHandler{}
// ImportValueRequest describes the import request structure
// for a value (BSI) import.
type ImportValueRequest struct {
Index string
Field string
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
Index string
Field string
// if Shard is MaxUint64 (an impossible shard value), this
// indicates that the column IDs may come from multiple shards.
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
FloatValues []float64
}
func (ivr *ImportValueRequest) Len() int { return len(ivr.ColumnIDs) }
func (ivr *ImportValueRequest) Less(i, j int) bool { return ivr.ColumnIDs[i] < ivr.ColumnIDs[j] }
func (ivr *ImportValueRequest) Swap(i, j int) {
ivr.ColumnIDs[i], ivr.ColumnIDs[j] = ivr.ColumnIDs[j], ivr.ColumnIDs[i]
if len(ivr.Values) > 0 {
ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
} else if len(ivr.FloatValues) > 0 {
ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
}
}
func (i *ImportValueRequest) Validate() error {
if i.Index == "" || i.Field == "" {
return errors.Errorf("index and field required, but got '%s' and '%s'", i.Index, i.Field)
}
if len(i.ColumnIDs) != 0 && len(i.ColumnKeys) != 0 {
return errors.Errorf("must pass either column ids or keys, but not both")
}
if len(i.Values) != 0 && len(i.FloatValues) != 0 {
return errors.Errorf("must pass ints or floats but not both")
}
return nil
}
// ImportRequest describes the import request structure

45
http/client.go
View file

@ -553,6 +553,35 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
return nil
}
// ImportValue2 is a simplified ImportValue method which just uses the
// ImportValueRequest instead of splitting up ImportValue and
// ImportValueK... it also supports importing float values. The idea
// being that (assuming it works) this will become the default (and be
// renamed) for 2.0, and we can deprecate the other methods.
func (c *InternalClient) ImportValue2(ctx context.Context, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.NewImportValue")
defer span.Finish()
buf, err := c.serializer.Marshal(req)
if err != nil {
return errors.Errorf("marshal import request: %s", err)
}
// Retrieve a list of nodes that own the shard.
nodes, err := c.FragmentNodes(ctx, req.Index, req.Shard)
if err != nil {
return errors.Errorf("shard nodes: %s", err)
}
// Import to each node.
for _, node := range nodes {
if err := c.importNode(ctx, node, req.Index, req.Field, buf, options); err != nil {
return errors.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
return nil
}
// ImportValueK bulk imports keyed field values to a host.
func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
@ -791,18 +820,28 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
}
// convert pilosa.FieldOptions to fieldOptions
//
// TODO this kind of sucks because it's one more place that needs
// changes when we change anything with field options (and there
// are a lot of places already). It's not clear to me that this is
// providing a lot of value, but I think this kind of validation
// should probably happen in the field anyway??
fieldOpt := fieldOptions{
Type: opt.Type,
Keys: &opt.Keys,
}
if fieldOpt.Type == "set" {
if fieldOpt.Type == pilosa.FieldTypeSet {
fieldOpt.CacheType = &opt.CacheType
fieldOpt.CacheSize = &opt.CacheSize
} else if fieldOpt.Type == "int" {
} else if fieldOpt.Type == pilosa.FieldTypeInt {
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
} else if fieldOpt.Type == "time" {
} else if fieldOpt.Type == pilosa.FieldTypeTime {
fieldOpt.TimeQuantum = &opt.TimeQuantum
} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
fieldOpt.Scale = &opt.Scale
}
// TODO: remove buf completely? (depends on whether importer needs to create specific field types)

41
http/client_test.go
View file

@ -998,6 +998,47 @@ func TestClient_FragmentBlocks(t *testing.T) {
}
}
func TestClient_CreateDecimalField(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster[0]
c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
index := "cdf"
err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
field := "dfield"
err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeDecimal, Scale: 1, Min: -1000, Max: 1000})
if err != nil {
t.Fatalf("creating field: %v", err)
}
fld, err := cmd.API.Field(context.Background(), index, field)
if err != nil {
t.Fatalf("getting field: %v", err)
}
if fld.Options().Scale != 1 {
t.Fatalf("expected Scale 1, got: %+v", fld.Options())
}
err = c.ImportValue2(context.Background(), &pilosa.ImportValueRequest{Index: index, Field: field, ColumnIDs: []uint64{1, 2, 3}, Shard: 0, FloatValues: []float64{1.1, 2.2, 3.3}}, &pilosa.ImportOptions{})
if err != nil {
t.Fatalf("importing float values: %v", err)
}
resp, err := c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>21)"})
if err != nil {
t.Fatalf("querying: %v", err)
}
if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
}
}
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*http.InternalClient

17
http/handler.go
View file

@ -776,7 +776,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
switch req.Options.Type {
case pilosa.FieldTypeSet:
fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize))
case pilosa.FieldTypeInt:
case pilosa.FieldTypeInt, pilosa.FieldTypeDecimal:
if req.Options.Min == nil {
min := int64(math.MinInt64)
req.Options.Min = &min
@ -785,7 +785,15 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
max := int64(math.MaxInt64)
req.Options.Max = &max
}
fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
if req.Options.Type == pilosa.FieldTypeDecimal {
scale := int64(0)
if req.Options.Scale != nil {
scale = *req.Options.Scale
}
fos = append(fos, pilosa.OptFieldTypeDecimal(scale, *req.Options.Min, *req.Options.Max))
} else {
fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
}
case pilosa.FieldTypeTime:
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView))
case pilosa.FieldTypeMutex:
@ -819,6 +827,7 @@ type fieldOptions struct {
CacheSize *uint32 `json:"cacheSize,omitempty"`
Min *int64 `json:"min,omitempty"`
Max *int64 `json:"max,omitempty"`
Scale *int64 `json:"scale,omitempty"`
TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
Keys *bool `json:"keys,omitempty"`
NoStandardView bool `json:"noStandardView,omitempty"`
@ -850,7 +859,7 @@ func (o *fieldOptions) validate() error {
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
}
case pilosa.FieldTypeInt:
case pilosa.FieldTypeInt, pilosa.FieldTypeDecimal:
if o.CacheType != nil {
return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int"))
} else if o.CacheSize != nil {
@ -1112,7 +1121,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Unmarshal request based on field type.
if field.Type() == pilosa.FieldTypeInt {
if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
// Field type: Int
// Marshal into request object.
req := &pilosa.ImportValueRequest{}

2124
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

1
internal/private.proto
View file

@ -18,6 +18,7 @@ message FieldOptions {
bool NoStandardView = 12;
int64 Base = 13;
uint64 BitDepth = 14;
int64 Scale = 15;
}
message ImportResponse {

1359
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

1
internal/public.proto
View file

@ -99,6 +99,7 @@ message ImportValueRequest {
repeated uint64 ColumnIDs = 5;
repeated string ColumnKeys = 7;
repeated int64 Values = 6;
repeated double FloatValues = 8;
}
message TranslateKeysRequest {

2
view.go
View file

@ -210,7 +210,7 @@ fragLoop:
// flags returns a set of flags for the underlying fragments.
func (v *view) flags() byte {
var flag byte
if v.fieldType == FieldTypeInt {
if v.fieldType == FieldTypeInt || v.fieldType == FieldTypeDecimal {
flag |= roaringFlagBSIv2
}
return flag