From 742135dc10210c9dcb5fbfe278262b9f9fdb6d27 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 8 Jan 2020 15:23:53 -0600 Subject: [PATCH] Get ForeignIndex string value when reading BSI field. In the `Inspect` function in `server/grpc.go`, getting the value of an `int` field with a foreign index to an index with `Keys()`, we need to return the string key value instead of the BSI int value for the field. This commit also changes the method `Field.keys()` to be exported as `Field.Keys()` so that it's accessible in the server package. --- api.go | 8 +++--- executor.go | 12 ++++----- field.go | 40 ++++++++++++++++++++++++++--- index.go | 4 +++ server/grpc.go | 69 +++++++++++++++++++++++++++++++++++--------------- 5 files changed, 100 insertions(+), 33 deletions(-) diff --git a/api.go b/api.go index 978f169f7..0fbc7ae7f 100644 --- a/api.go +++ b/api.go @@ -544,7 +544,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin var colStr string var err error - if field.keys() { + if field.Keys() { if rowStr, err = field.translateStore.TranslateID(rowID); err != nil { return errors.Wrap(err, "translating row") } @@ -959,7 +959,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // check to see if keys need translation. if !options.IgnoreKeyCheck { // Translate row keys. - if field.keys() { + if field.Keys() { if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } @@ -980,7 +980,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // For translated data, map the columnIDs to shards. If // this node does not own the shard, forward to the node that does. - if index.Keys() || field.keys() { + if index.Keys() || field.Keys() { m := make(map[uint64][]Bit) for i, colID := range req.ColumnIDs { @@ -1086,7 +1086,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // Translate values when the field uses keys (for example, when // the field has a ForeignIndex with keys). - if field.keys() { + if field.Keys() { uints, err := field.translateStore.TranslateKeys(req.StringValues) if err != nil { return errors.Wrap(err, "translating string values") diff --git a/executor.go b/executor.go index 0b4d7d8e9..b6d62ca03 100644 --- a/executor.go +++ b/executor.go @@ -3619,7 +3619,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { } c.Args[rowKey] = rowID } - } else if field.keys() { + } else if field.Keys() { if err := e.translateRowKey(c, field.translateStore, rowKey); err != nil { return errors.Wrap(err, "translating rowkey") } @@ -3744,7 +3744,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e for i, field := range fields { prev := previous[i] - if field.keys() { + if field.Keys() { prevStr, ok := prev.(string) if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") @@ -3841,7 +3841,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } - if field.keys() { + if field.Keys() { key, err := field.translateStore.TranslateID(result.Pair.ID) if err != nil { return nil, err @@ -3863,7 +3863,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } - if field.keys() { + if field.Keys() { other := make([]Pair, len(result.Pairs)) for i := range result.Pairs { key, err := field.translateStore.TranslateID(result.Pairs[i].ID) @@ -3892,7 +3892,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field == nil { return nil, ErrFieldNotFound } - if field.keys() { + if field.Keys() { key, err := field.translateStore.TranslateID(g.RowID) if err != nil { return nil, errors.Wrap(err, "translating row ID in Group") @@ -3921,7 +3921,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field := idx.Field(fieldName); field == nil { return nil, ErrFieldNotFound - } else if field.keys() { + } else if field.Keys() { other.Keys = make([]string, len(result)) for i, id := range result { key, err := field.translateStore.TranslateID(id) diff --git a/field.go b/field.go index 24249ee80..800c998a0 100644 --- a/field.go +++ b/field.go @@ -798,8 +798,8 @@ func (f *Field) Close() error { return nil } -// keys returns true if the field uses string keys. -func (f *Field) keys() bool { +// Keys returns true if the field uses string keys. +func (f *Field) Keys() bool { f.mu.RLock() defer f.mu.RUnlock() return f.usesKeys @@ -1156,6 +1156,21 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { return me } +// StringValue reads an integer field value for a column, and converts +// it to a string based on a foreign index string key. +func (f *Field) StringValue(columnID uint64) (value string, exists bool, err error) { + bsig := f.bsiGroup(f.name) + if bsig == nil { + return value, false, ErrBSIGroupNotFound + } + + val, exists, err := f.Value(columnID) + if exists { + value, err = f.translateStore.TranslateID(uint64(val)) + } + return value, exists, err +} + // FloatValue reads an integer field value for a column, and converts // it to a float based on the configured scale. func (f *Field) FloatValue(columnID uint64) (value float64, exists bool, err error) { @@ -1678,6 +1693,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { TimeQuantum: string(o.TimeQuantum), Keys: o.Keys, NoStandardView: o.NoStandardView, + ForeignIndex: o.ForeignIndex, } } @@ -1698,7 +1714,25 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.CacheSize, o.Keys, }) - case FieldTypeInt, FieldTypeDecimal: + case FieldTypeInt: + return json.Marshal(struct { + Type string `json:"type"` + Base int64 `json:"base"` + BitDepth uint `json:"bitDepth"` + Min int64 `json:"min"` + Max int64 `json:"max"` + Keys bool `json:"keys"` + ForeignIndex string `json:"foreignIndex"` + }{ + o.Type, + o.Base, + o.BitDepth, + o.Min, + o.Max, + o.Keys, + o.ForeignIndex, + }) + case FieldTypeDecimal: return json.Marshal(struct { Type string `json:"type"` Base int64 `json:"base"` diff --git a/index.go b/index.go index ed05ca6bc..c0659f733 100644 --- a/index.go +++ b/index.go @@ -198,6 +198,10 @@ fileLoop: return errors.Wrapf(ErrName, "'%s'", fi.Name()) } + // Pass holder through to the field for use in looking + // up a foreign index. + fld.holder = i.holder + if err := fld.Open(); err != nil { return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } diff --git a/server/grpc.go b/server/grpc.go index a0030d6f5..6ffedfff3 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -80,12 +80,14 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL func fieldDataType(f *pilosa.Field) string { switch f.Type() { case "set", "mutex": - if f.Options().Keys { + if f.Keys() { return "[]string" - } else { - return "[]uint64" } + return "[]uint64" case "int": + if f.Keys() { + return "string" + } return "int64" case "decimal": return "float64" @@ -128,7 +130,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } offset := req.Offset - if !index.Options().Keys { + if !index.Keys() { ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids) if !ok { return errors.New("invalid int columns") @@ -243,15 +245,28 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } case "int": - value, exists, err := field.Value(col) - if err != nil { - return errors.Wrap(err, "getting int field value for column") - } else if exists { - rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}}) + if field.Keys() { + value, exists, err := field.StringValue(col) + if err != nil { + return errors.Wrap(err, "getting string field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } } else { - rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: nil}) + value, exists, err := field.Value(col) + if err != nil { + return errors.Wrap(err, "getting int field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } } case "decimal": @@ -426,16 +441,30 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer return errors.Wrap(err, "translating column key") } - value, exists, err := field.Value(id) - if err != nil { - return errors.Wrap(err, "getting int field value for column") - } else if exists { - rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}}) + if field.Keys() { + value, exists, err := field.StringValue(id) + if err != nil { + return errors.Wrap(err, "getting string field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } } else { - rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: nil}) + value, exists, err := field.Value(id) + if err != nil { + return errors.Wrap(err, "getting int field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } } + case "decimal": // Translate column key. id, err := index.TranslateStore().TranslateKey(col)