scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'enterprise' into bug-q2-double-delete

Browse source
This commit is contained in:
tgruben 2020-01-12 16:09:12 -06:00 • committed by GitHub
commit 3261ec4dc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 3923 additions and 2336 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
api.go
View file

@ -544,7 +544,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
var colStr string
var err error
if field.keys() {
if field.Keys() {
if rowStr, err = field.translateStore.TranslateID(rowID); err != nil {
return errors.Wrap(err, "translating row")
}
@ -959,7 +959,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
// check to see if keys need translation.
if !options.IgnoreKeyCheck {
// Translate row keys.
if field.keys() {
if field.Keys() {
if len(req.RowIDs) != 0 {
return errors.New("row ids cannot be used because field uses string keys")
}
@ -980,7 +980,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
// For translated data, map the columnIDs to shards. If
// this node does not own the shard, forward to the node that does.
if index.Keys() || field.keys() {
if index.Keys() || field.Keys() {
m := make(map[uint64][]Bit)
for i, colID := range req.ColumnIDs {
@ -1083,6 +1083,22 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
}
req.Shard = math.MaxUint64
}
// Translate values when the field uses keys (for example, when
// the field has a ForeignIndex with keys).
if field.Keys() {
uints, err := field.translateStore.TranslateKeys(req.StringValues)
if err != nil {
return errors.Wrap(err, "translating string values")
}
// Because the BSI field supports negative value, we have to
// convert the slice of uint64 keys to a slice of int64.
ints := make([]int64, len(uints))
for i := range uints {
ints[i] = int64(uints[i])
}
req.Values = ints
}
}
if !options.Presorted {

55
api_test.go
View file

@ -536,6 +536,61 @@ func TestAPI_ImportValue(t *testing.T) {
}
})
t.Run("ValStringField", func(t *testing.T) {
ctx := context.Background()
index := "valstr"
field := "fstr"
fgnIndex := "fgnvalstr"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating foreign index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field,
pilosa.OptFieldTypeInt(0, math.MaxInt64),
pilosa.OptFieldForeignIndex(fgnIndex),
)
if err != nil {
t.Fatalf("creating field: %v", err)
}
// Generate some keyed records.
values := []string{}
colIDs := []uint64{}
for i := 0; i < 10; i++ {
value := fmt.Sprintf("strval-%d", (i)*100+10)
values = append(values, value)
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
ColumnIDs: colIDs,
StringValues: values,
}
if err := m0.API.ImportValue(ctx, req); err != nil {
t.Fatal(err)
}
pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected columns: %+v", ids)
}
})
}
// offsetModHasher represents a simple, mod-based hashing offset by 1.

2
ctl/import.go
View file

@ -108,6 +108,8 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
cmd.FieldOptions.Type = pilosa.FieldTypeInt
} else {
cmd.FieldOptions.Type = pilosa.FieldTypeSet
cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
}
}
err := cmd.ensureSchema(ctx)

8
docs/getting-started.md
View file

@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
``` response
{"success":true}
```
The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
``` request
@ -325,7 +325,7 @@ Next, let's create the `repository` index:
repository := schema.Index("repository")
```
The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```
@ -615,7 +615,7 @@ Next, let's create the `repository` index:
```
Index repository = schema.index("repository");
```
The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```
@ -818,7 +818,7 @@ Next, let's create the `repository` index:
```
repository = schema.index("repository")
```
The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```

2
docs/query-language.md
View file

@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
#### Arguments and Types
* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
* `UINT` An unsigned integer (e.g. 42839).
* `BOOL` A boolean value, `true` or `false`.

38
encoding/proto/proto.go
View file

@ -379,13 +379,14 @@ func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
return &internal.ImportValueRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
StringValues: m.StringValues,
}
}
@ -567,16 +568,17 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
return nil
}
return &internal.FieldOptions{
Type: o.Type,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Min: o.Min,
Max: o.Max,
Base: o.Base,
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
Type: o.Type,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Min: o.Min,
Max: o.Max,
Base: o.Base,
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
ForeignIndex: o.ForeignIndex,
}
}
@ -848,6 +850,7 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions)
m.BitDepth = uint(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
m.Keys = options.Keys
m.ForeignIndex = options.ForeignIndex
}
func decodeNodes(a []*internal.Node, m []*pilosa.Node) {
@ -1025,6 +1028,7 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV
m.ColumnKeys = pb.ColumnKeys
m.Values = pb.Values
m.FloatValues = pb.FloatValues
m.StringValues = pb.StringValues
}
func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {

100
executor.go
View file

@ -3619,18 +3619,9 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
}
c.Args[rowKey] = rowID
}
} else if field.keys() {
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
// allow passing row id directly (this can come in handy, but make sure it is a valid row id)
if !isValidID(c.Args[rowKey]) {
return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey])
}
} else if value := callArgString(c, rowKey); value != "" {
id, err := field.translateStore.TranslateKey(value)
if err != nil {
return err
}
c.Args[rowKey] = id
} else if field.Keys() {
if err := e.translateRowKey(c, field.translateStore, rowKey); err != nil {
return errors.Wrap(err, "translating rowkey")
}
} else {
if isString(c.Args[rowKey]) {
@ -3662,6 +3653,42 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
return nil
}
func (e *executor) translateRowKey(c *pql.Call, store TranslateStore, rowKey string) error {
if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) {
// In the case where a field has a foreign index with keys,
// allow `== "key"` or `!= "key"` to be used against the BSI
// field.
cond := c.Args[rowKey].(*pql.Condition)
if isString(cond.Value) {
switch cond.Op {
case pql.EQ, pql.NEQ:
id, err := store.TranslateKey(cond.Value.(string))
if err != nil {
return errors.Wrap(err, "translating key")
}
c.Args[rowKey] = &pql.Condition{
Op: cond.Op,
Value: id,
}
default:
return errors.Errorf("conditional is not supported with string predicates: %s", cond.Op)
}
}
} else if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
// allow passing row id directly (this can come in handy, but make sure it is a valid row id)
if !isValidID(c.Args[rowKey]) {
return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey])
}
} else if value := callArgString(c, rowKey); value != "" {
id, err := store.TranslateKey(value)
if err != nil {
return errors.Wrap(err, "translating key")
}
c.Args[rowKey] = id
}
return nil
}
func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) error {
if c.Name != "GroupBy" {
panic("translateGroupByCall called with '" + c.Name + "'")
@ -3717,7 +3744,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e
for i, field := range fields {
prev := previous[i]
if field.keys() {
if field.Keys() {
prevStr, ok := prev.(string)
if !ok {
return errors.New("prev value must be a string when field 'keys' option enabled")
@ -3767,13 +3794,47 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
return other, nil
}
// TODO: instead of supporting SignedRow here, we may be able to
// make the return type for an int field with a ForeignIndex be
// a *Row instead (because it should always be positive).
case SignedRow:
var store TranslateStore
if fieldName := callArgString(call, "field"); fieldName != "" {
field := idx.Field(fieldName)
if field != nil && field.Keys() {
store = field.TranslateStore()
}
}
// In the case where a field/foreignIndex doesn't exist,
// fall back to using the index translateStore.
if store == nil && idx.Keys() {
store = idx.translateStore
}
if store != nil {
rslt := result.Pos
other := &Row{Attrs: rslt.Attrs}
for _, segment := range rslt.Segments() {
for _, col := range segment.Columns() {
key, err := store.TranslateID(col)
if err != nil {
return nil, err
}
other.Keys = append(other.Keys, key)
}
}
return SignedRow{Pos: other}, nil
}
case PairField:
if fieldName := callArgString(call, "field"); fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return nil, fmt.Errorf("field %q not found", fieldName)
}
if field.keys() {
if field.Keys() {
key, err := field.translateStore.TranslateID(result.Pair.ID)
if err != nil {
return nil, err
@ -3795,7 +3856,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
if field == nil {
return nil, fmt.Errorf("field %q not found", fieldName)
}
if field.keys() {
if field.Keys() {
other := make([]Pair, len(result.Pairs))
for i := range result.Pairs {
key, err := field.translateStore.TranslateID(result.Pairs[i].ID)
@ -3824,7 +3885,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
if field == nil {
return nil, ErrFieldNotFound
}
if field.keys() {
if field.Keys() {
key, err := field.translateStore.TranslateID(g.RowID)
if err != nil {
return nil, errors.Wrap(err, "translating row ID in Group")
@ -3853,7 +3914,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
if field := idx.Field(fieldName); field == nil {
return nil, ErrFieldNotFound
} else if field.keys() {
} else if field.Keys() {
other.Keys = make([]string, len(result))
for i, id := range result {
key, err := field.translateStore.TranslateID(id)
@ -4066,6 +4127,11 @@ func isString(v interface{}) bool {
return ok
}
func isCondition(v interface{}) bool {
_, ok := v.(*pql.Condition)
return ok
}
// isValidID returns whether v can be interpreted as a valid row or
// column ID. In short, is v a non-negative integer? I think the int64
// and default cases are the only ones actually used since the PQL

64
executor_test.go
View file

@ -3854,6 +3854,70 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) {
}
func TestExecutor_ForeignIndex(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
c.CreateField(t, "parent", pilosa.IndexOptions{Keys: true}, "general")
c.CreateField(t, "child", pilosa.IndexOptions{}, "parent_id",
pilosa.OptFieldTypeInt(0, math.MaxInt64),
pilosa.OptFieldForeignIndex("parent"),
)
c.CreateField(t, "child", pilosa.IndexOptions{}, "color",
pilosa.OptFieldKeys(),
)
// Populate parent data.
c.Query(t, "parent", `
Set("one", general=1)
Set("two", general=1)
Set("three", general=1)
Set("twenty-one", general=2)
Set("twenty-two", general=2)
Set("twenty-three", general=2)
Set("one", general=3)
Set("twenty-one", general=3)
`)
// Populate child data.
c.Query(t, "child", `
Set(1, parent_id="one")
Set(2, parent_id="two")
Set(3, parent_id="one")
Set(4, parent_id="twenty-one")
`)
// Populate color data.
c.Query(t, "child", `
Set(1, color="red")
Set(2, color="blue")
Set(3, color="blue")
Set(4, color="red")
`)
distinct := c.Query(t, "child", `Distinct(index="child", field="parent_id")`).Results[0].(pilosa.SignedRow)
if !reflect.DeepEqual(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) {
t.Fatalf("unexpected keys: %v", distinct.Pos.Keys)
}
eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row)
if !reflect.DeepEqual(eq.Columns(), []uint64{1, 3}) {
t.Fatalf("unexpected columns: %v", eq.Columns())
}
neq := c.Query(t, "child", `Row(parent_id!="one")`).Results[0].(*pilosa.Row)
if !reflect.DeepEqual(neq.Columns(), []uint64{2, 4}) {
t.Fatalf("unexpected columns: %v", neq.Columns())
}
join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row)
if !reflect.DeepEqual(join.Keys, []string{"one"}) {
t.Fatalf("unexpected keys: %v", join.Keys)
}
}
func TestExecutor_Execute_GroupBy(t *testing.T) {
groupByTest := func(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, 1)

153
field.go
View file

@ -84,6 +84,15 @@ type Field struct {
// Field options.
options FieldOptions
// finalOptions is used with a final call to applyOptions.
// The initial call to applyOptions is made with options
// loaded from the meta file on disk (in the case when
// a field is being re-opened). If the field creator calls
// setOptions before calling Open(), then those options
// will be held in finalOptions, and applied instead of
// those from the meta file.
finalOptions *FieldOptions
bsiGroups []*bsiGroup
// Shards with data on any node in the cluster, according to this node.
@ -94,6 +103,15 @@ type Field struct {
snapshotQueue snapshotQueue
// Instantiates new translation store on open.
OpenTranslateStore OpenTranslateStoreFunc
// Used for looking up a foreign index.
holder *Holder
// Stores whether or not the field has keys enabled.
// This is most helpful for cases where the keys are
// based on a foreign index; this prevents having to
// call holder.index.Keys() every time.
usesKeys bool
}
// FieldOption is a functional option type for pilosa.fieldOptions.
@ -108,6 +126,17 @@ func OptFieldKeys() FieldOption {
}
}
// OptFieldForeignIndex marks this field as a foreign key to another
// index. That is, the values of this field should be interpreted as
// referencing records (Pilosa columns) in another index. TODO explain
// where/how this is used by Pilosa.
func OptFieldForeignIndex(index string) FieldOption {
return func(fo *FieldOptions) error {
fo.ForeignIndex = index
return nil
}
}
// OptFieldTypeDefault is a functional option on FieldOptions
// used to set the field type and cache setting to the default values.
func OptFieldTypeDefault() FieldOption {
@ -230,6 +259,11 @@ func OptFieldTypeBool() FieldOption {
}
// NewField returns a new instance of field.
// NOTE: This function is only used in tests, which is why
// it only takes a single `FieldOption` (the assumption being
// that it's of the type `OptFieldType*`). This means
// this function couldn't be used to set, for example,
// `FieldOptions.Keys`.
func NewField(path, index, name string, opts FieldOption) (*Field, error) {
err := validateName(name)
if err != nil {
@ -260,7 +294,7 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) {
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
options: applyDefaultOptions(fo),
options: *applyDefaultOptions(&fo),
remoteAvailableShards: roaring.NewBitmap(),
@ -457,7 +491,13 @@ func (f *Field) Open() error {
return errors.Wrap(err, "loading available shards")
}
// Apply the field options loaded from meta.
// If options were provided using setOptions(), then
// use those instead of the options from the meta file.
if f.finalOptions != nil {
f.options = *f.finalOptions
}
// Apply the field options loaded from meta (or set via setOptions()).
f.logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
@ -473,9 +513,16 @@ func (f *Field) Open() error {
return errors.Wrap(err, "opening attrstore")
}
// Instantiate & open translation store.
if f.translateStore, err = f.OpenTranslateStore(filepath.Join(f.path, "keys"), f.index, f.name); err != nil {
return errors.Wrap(err, "opening translate store")
// If the field has a foreign index, and that index uses keys,
// then use that index's translateStore instead.
if f.options.ForeignIndex != "" {
if err := f.holder.checkForeignIndex(f); err != nil {
return errors.Wrap(err, "checking foreign index")
}
} else {
if err := f.applyTranslateStore(); err != nil {
return errors.Wrap(err, "applying translate store")
}
}
return nil
@ -488,6 +535,35 @@ func (f *Field) Open() error {
return nil
}
// applyTranslateStore opens the configured translate store.
func (f *Field) applyTranslateStore() error {
// Instantiate & open translation store.
var err error
f.translateStore, err = f.OpenTranslateStore(filepath.Join(f.path, "keys"), f.index, f.name)
if err != nil {
return errors.Wrap(err, "opening translate store")
}
f.usesKeys = f.options.Keys
return nil
}
// applyForeignIndex sets the field's translateStore
// to that of a foreign index in the case where the
// foreign index uses keys. If the foreign index does
// not use keys, it falls back to applying the field's
// default translate store.
func (f *Field) applyForeignIndex() error {
foreignIndex := f.holder.Index(f.options.ForeignIndex)
if foreignIndex == nil {
return errors.Wrapf(ErrForeignIndexNotFound, "%s", f.options.ForeignIndex)
} else if foreignIndex.Keys() {
f.usesKeys = true
f.translateStore = foreignIndex.translateStore
return nil
}
return f.applyTranslateStore()
}
var fieldQueue = make(chan struct{}, 16)
// openViews opens and initializes the views inside the field.
@ -594,6 +670,7 @@ func (f *Field) loadMeta() error {
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
f.options.NoStandardView = pb.NoStandardView
f.options.ForeignIndex = pb.ForeignIndex
return nil
}
@ -624,6 +701,11 @@ func (f *Field) saveMeta() error {
return nil
}
// setOptions saves options for final application during Open().
func (f *Field) setOptions(opts *FieldOptions) {
f.finalOptions = applyDefaultOptions(opts)
}
// applyOptions configures the field based on opt.
func (f *Field) applyOptions(opt FieldOptions) error {
switch opt.Type {
@ -647,6 +729,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = 0
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
f.options.ForeignIndex = ""
case FieldTypeInt, FieldTypeDecimal:
f.options.Type = opt.Type
f.options.CacheType = CacheTypeNone
@ -658,6 +741,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = opt.BitDepth
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
f.options.ForeignIndex = opt.ForeignIndex
// Create new bsiGroup.
bsig := &bsiGroup{
@ -691,6 +775,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.Close()
return errors.Wrap(err, "setting time quantum")
}
f.options.ForeignIndex = ""
case FieldTypeBool:
f.options.Type = FieldTypeBool
f.options.CacheType = CacheTypeNone
@ -701,6 +786,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = 0
f.options.TimeQuantum = ""
f.options.Keys = false
f.options.ForeignIndex = ""
default:
return errors.New("invalid field type")
}
@ -735,11 +821,11 @@ func (f *Field) Close() error {
return nil
}
// keys returns true if the field uses string keys.
func (f *Field) keys() bool {
// Keys returns true if the field uses string keys.
func (f *Field) Keys() bool {
f.mu.RLock()
defer f.mu.RUnlock()
return f.options.Keys
return f.usesKeys
}
// bsiGroup returns a bsiGroup by name.
@ -1093,6 +1179,21 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) {
return me
}
// StringValue reads an integer field value for a column, and converts
// it to a string based on a foreign index string key.
func (f *Field) StringValue(columnID uint64) (value string, exists bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return value, false, ErrBSIGroupNotFound
}
val, exists, err := f.Value(columnID)
if exists {
value, err = f.translateStore.TranslateID(uint64(val))
}
return value, exists, err
}
// FloatValue reads an integer field value for a column, and converts
// it to a float based on the configured scale.
func (f *Field) FloatValue(columnID uint64) (value float64, exists bool, err error) {
@ -1580,17 +1681,16 @@ type FieldOptions struct {
CacheType string `json:"cacheType,omitempty"`
Type string `json:"type,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
ForeignIndex string `json:"foreignIndex"`
}
// applyDefaultOptions returns a new FieldOptions object
// with default values if o does not contain a valid type.
func applyDefaultOptions(o FieldOptions) FieldOptions {
// applyDefaultOptions updates FieldOptions with the default
// values if o does not contain a valid type.
func applyDefaultOptions(o *FieldOptions) *FieldOptions {
if o.Type == "" {
return FieldOptions{
Type: DefaultFieldType,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
}
o.Type = DefaultFieldType
o.CacheType = DefaultCacheType
o.CacheSize = DefaultCacheSize
}
return o
}
@ -1616,6 +1716,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
NoStandardView: o.NoStandardView,
ForeignIndex: o.ForeignIndex,
}
}
@ -1636,7 +1737,25 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
o.CacheSize,
o.Keys,
})
case FieldTypeInt, FieldTypeDecimal:
case FieldTypeInt:
return json.Marshal(struct {
Type string `json:"type"`
Base int64 `json:"base"`
BitDepth uint `json:"bitDepth"`
Min int64 `json:"min"`
Max int64 `json:"max"`
Keys bool `json:"keys"`
ForeignIndex string `json:"foreignIndex"`
}{
o.Type,
o.Base,
o.BitDepth,
o.Min,
o.Max,
o.Keys,
o.ForeignIndex,
})
case FieldTypeDecimal:
return json.Marshal(struct {
Type string `json:"type"`
Base int64 `json:"base"`

2
field_internal_test.go
View file

@ -516,7 +516,7 @@ func TestField_ApplyOptions(t *testing.T) {
} {
fld := &Field{}
fld.options = applyDefaultOptions(FieldOptions{})
fld.options = *applyDefaultOptions(&FieldOptions{})
if err := fld.applyOptions(tt.opts); err != nil {
t.Fatal(err)

3
field_test.go
View file

@ -158,6 +158,7 @@ func TestField_NameValidation(t *testing.T) {
"under_score",
"abc123",
"trailing_",
"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
}
invalidFieldNames := []string{
"",
@ -168,7 +169,7 @@ func TestField_NameValidation(t *testing.T) {
"abc def",
"camelCase",
"UPPERCASE",
"a12345678901234567890123456789012345678901234567890123456789012345",
"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
}
path, err := ioutil.TempDir("", "pilosa-field-")

2
go.mod
View file

@ -18,7 +18,7 @@ require (
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0

2
go.sum
View file

@ -91,6 +91,8 @@ github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 h1:XOImsA5XhGklFj8Y0TxSm1qWZzEwYxom2JOXiu9GMq0=
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

36
handler.go
View file

@ -116,11 +116,12 @@ type ImportValueRequest struct {
Field string
// if Shard is MaxUint64 (an impossible shard value), this
// indicates that the column IDs may come from multiple shards.
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
FloatValues []float64
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
FloatValues []float64
StringValues []string
}
func (ivr *ImportValueRequest) Len() int { return len(ivr.ColumnIDs) }
@ -131,18 +132,31 @@ func (ivr *ImportValueRequest) Swap(i, j int) {
ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
} else if len(ivr.FloatValues) > 0 {
ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
} else if len(ivr.StringValues) > 0 {
ivr.StringValues[i], ivr.StringValues[j] = ivr.StringValues[j], ivr.StringValues[i]
}
}
func (i *ImportValueRequest) Validate() error {
if i.Index == "" || i.Field == "" {
return errors.Errorf("index and field required, but got '%s' and '%s'", i.Index, i.Field)
// Validate ensures that the payload of the request is valid.
func (ivr *ImportValueRequest) Validate() error {
if ivr.Index == "" || ivr.Field == "" {
return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
}
if len(i.ColumnIDs) != 0 && len(i.ColumnKeys) != 0 {
if len(ivr.ColumnIDs) != 0 && len(ivr.ColumnKeys) != 0 {
return errors.Errorf("must pass either column ids or keys, but not both")
}
if len(i.Values) != 0 && len(i.FloatValues) != 0 {
return errors.Errorf("must pass ints or floats but not both")
var valueSetCount int
if len(ivr.Values) != 0 {
valueSetCount++
}
if len(ivr.FloatValues) != 0 {
valueSetCount++
}
if len(ivr.StringValues) != 0 {
valueSetCount++
}
if valueSetCount > 1 {
return errors.Errorf("must pass ints, floats, or strings but not multiple")
}
return nil
}

48
holder.go
View file

@ -84,6 +84,16 @@ type Holder struct {
// Instantiates new translation stores for indexes & fields.
OpenTranslateStore OpenTranslateStoreFunc // local store
OpenTranslateReader OpenTranslateReaderFunc // replication
// Queue of fields (having a foreign index) which have
// opened before their foreign index has opened.
foreignIndexFields []*Field
// opening is set to true while Holder is opening.
// It's used to determine if foreign index application
// needs to be queued and completed after all indexes
// have opened.
opening bool
}
// lockedChan looks a little ridiculous admittedly, but exists for good reason.
@ -135,6 +145,9 @@ func NewHolder() *Holder {
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.opening = true
defer func() { h.opening = false }()
// Reset closing in case Holder is being reopened.
h.closing = make(chan struct{})
@ -196,6 +209,14 @@ func (h *Holder) Open() error {
h.indexes[index.Name()] = index
h.mu.Unlock()
}
// If any fields were opened before their foreign index
// was opened, it's safe to process those now since all index
// opens have completed by this point.
if err := h.processForeignIndexFields(); err != nil {
return errors.Wrap(err, "processing foreign index fields")
}
h.Logger.Printf("open holder: complete")
// Periodically flush cache.
@ -209,6 +230,33 @@ func (h *Holder) Open() error {
return nil
}
// checkForeignIndex is a check before applying a foreign
// index to a field; if the index is not yet available,
// (because holder is still opening and may not have opened
// the index yet), this method queues it up to be processed
// once all indexes have been opened.
func (h *Holder) checkForeignIndex(f *Field) error {
if h.opening {
if fi := h.Index(f.options.ForeignIndex); fi == nil {
h.foreignIndexFields = append(h.foreignIndexFields, f)
return nil
}
}
return f.applyForeignIndex()
}
// processForeignIndexFields applies a foreign index to any
// fields which were opened before their foreign index.
func (h *Holder) processForeignIndexFields() error {
for _, f := range h.foreignIndexFields {
if err := f.applyForeignIndex(); err != nil {
return errors.Wrap(err, "applying foreign index")
}
}
h.foreignIndexFields = h.foreignIndexFields[:0] // reset
return nil
}
// Close closes all open fragments.
func (h *Holder) Close() error {
h.Stats.Close()

59
holder_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/test"
"github.com/pkg/errors"
)
func TestHolder_Open(t *testing.T) {
@ -217,6 +218,64 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ForeignIndex", func(t *testing.T) {
t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else {
_, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("nonexistent"))
if err == nil {
t.Fatalf("expected error: %s", pilosa.ErrForeignIndexNotFound)
} else if errors.Cause(err) != pilosa.ErrForeignIndexNotFound {
t.Fatalf("expected error: %s, but got: %s", pilosa.ErrForeignIndexNotFound, err)
}
}
})
// Foreign index zzz is opened after foo/bar.
t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("zzz")); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatalf("unexpected error: %s", err)
}
})
// Foreign index aaa is opened before foo/bar.
t.Run("ForeignIndexIsOpen", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("aaa")); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatalf("unexpected error: %s", err)
}
})
})
}
func TestHolder_HasData(t *testing.T) {

14
http/handler.go
View file

@ -807,6 +807,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos = append(fos, pilosa.OptFieldKeys())
}
}
if req.Options.ForeignIndex != nil {
fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex))
}
_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
if _, ok := err.(pilosa.BadRequestError); ok {
@ -832,6 +835,7 @@ type fieldOptions struct {
TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
Keys *bool `json:"keys,omitempty"`
NoStandardView bool `json:"noStandardView,omitempty"`
ForeignIndex *string `json:"foreignIndex,omitempty"`
}
func (o *fieldOptions) validate() error {
@ -859,6 +863,8 @@ func (o *fieldOptions) validate() error {
return pilosa.NewBadRequestError(errors.New("max does not apply to field type set"))
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
} else if o.ForeignIndex != nil {
return pilosa.NewBadRequestError(errors.New("set field cannot be a foreign key"))
}
case pilosa.FieldTypeInt, pilosa.FieldTypeDecimal:
if o.CacheType != nil {
@ -867,6 +873,8 @@ func (o *fieldOptions) validate() error {
return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
}
case pilosa.FieldTypeTime:
if o.CacheType != nil {
@ -879,6 +887,8 @@ func (o *fieldOptions) validate() error {
return pilosa.NewBadRequestError(errors.New("max does not apply to field type time"))
} else if o.TimeQuantum == nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time"))
} else if o.ForeignIndex != nil {
return pilosa.NewBadRequestError(errors.New("time field cannot be a foreign key"))
}
case pilosa.FieldTypeMutex:
if o.CacheType == nil {
@ -893,6 +903,8 @@ func (o *fieldOptions) validate() error {
return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex"))
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
} else if o.ForeignIndex != nil {
return pilosa.NewBadRequestError(errors.New("mutex field cannot be a foreign key"))
}
case pilosa.FieldTypeBool:
if o.CacheType != nil {
@ -907,6 +919,8 @@ func (o *fieldOptions) validate() error {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
} else if o.Keys != nil {
return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
} else if o.ForeignIndex != nil {
return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key"))
}
default:
return errors.Errorf("invalid field type: %s", o.Type)

17
index.go
View file

@ -61,6 +61,7 @@ type Index struct {
snapshotQueue snapshotQueue
// Used for notifying holder when a field is added.
// Also passed to field for foreign-index lookup.
holder *Holder
// Instantiates new translation stores for fields.
@ -197,6 +198,10 @@ fileLoop:
return errors.Wrapf(ErrName, "'%s'", fi.Name())
}
// Pass holder through to the field for use in looking
// up a foreign index.
fld.holder = i.holder
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
@ -426,17 +431,17 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
return nil, errors.Wrap(err, "initializing")
}
// Pass holder through to the field for use in looking
// up a foreign index.
f.holder = i.holder
f.setOptions(&opt)
// Open field.
if err := f.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
}
// Apply field options.
if err := f.applyOptions(opt); err != nil {
f.Close()
return nil, errors.Wrap(err, "applying options")
}
if err := f.saveMeta(); err != nil {
f.Close()
return nil, errors.Wrap(err, "saving meta")

2898
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

1
internal/private.proto
View file

@ -19,6 +19,7 @@ message FieldOptions {
int64 Base = 13;
uint64 BitDepth = 14;
int64 Scale = 15;
string ForeignIndex = 16;
}
message ImportResponse {

2616
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

1
internal/public.proto
View file

@ -120,6 +120,7 @@ message ImportValueRequest {
repeated string ColumnKeys = 7;
repeated int64 Values = 6;
repeated double FloatValues = 8;
repeated string StringValues = 9;
}
message TranslateKeysRequest {

6
pilosa.go
View file

@ -29,6 +29,8 @@ var (
ErrIndexExists = errors.New("index already exists")
ErrIndexNotFound = errors.New("index not found")
ErrForeignIndexNotFound = errors.New("foreign index not found")
// ErrFieldRequired is returned when no field is specified.
ErrFieldRequired = errors.New("field required")
ErrFieldExists = errors.New("field already exists")
@ -48,7 +50,7 @@ var (
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters")
ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
// ErrFragmentNotFound is returned when a fragment does not exist.
@ -118,7 +120,7 @@ func newNotFoundError(err error) NotFoundError {
}
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.

4
pilosa_internal_test.go
View file

@ -21,7 +21,7 @@ import (
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e", "exists",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"longbutnottoolongaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa12345689012345689012345678901234567890",
}
for _, name := range names {
if validateName(name) != nil {
@ -33,7 +33,7 @@ func TestValidateName(t *testing.T) {
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
"long123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
}
for _, name := range names {
if validateName(name) == nil {

18
server.go
View file

@ -380,7 +380,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.holder.broadcaster = s
err = s.loadExtensions()
err = s.loadAllExtensions()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
@ -417,8 +417,18 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
func (s *Server) loadExtensions() error {
exts := ext.NewExtensions()
// loadNewExtensions loads extensions that have been
// registered since the last call to loadNewExtensions.
func (s *Server) loadNewExtensions() error { //nolint:unused
return s.loadExtensions(ext.NewExtensions())
}
// loadAllExtensions loads all extensions.
func (s *Server) loadAllExtensions() error {
return s.loadExtensions(ext.AllExtensions())
}
func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error {
var lastError error
for _, extension := range exts {
if err := s.loadExtension(extension); err != nil {
@ -436,8 +446,6 @@ func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
bitmapOps := extInfo.BitmapOps
bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
for i := range bitmapOps {
// title-case the name
bitmapOps[i].Name = strings.Title(bitmapOps[i].Name)
typ := bitmapOps[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:

88
server/grpc.go
View file

@ -19,6 +19,7 @@ import (
"crypto/tls"
"fmt"
"net"
"strings"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/logger"
@ -80,12 +81,14 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL
func fieldDataType(f *pilosa.Field) string {
switch f.Type() {
case "set", "mutex":
if f.Options().Keys {
if f.Keys() {
return "[]string"
} else {
return "[]uint64"
}
return "[]uint64"
case "int":
if f.Keys() {
return "string"
}
return "int64"
case "decimal":
return "float64"
@ -109,6 +112,10 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
var fields []*pilosa.Field
for _, field := range index.Fields() {
// exclude internal fields (starting with "_")
if strings.HasPrefix(field.Name(), "_") {
continue
}
if len(req.FilterFields) > 0 {
for _, filter := range req.FilterFields {
if filter == field.Name() {
@ -128,7 +135,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
}
offset := req.Offset
if !index.Options().Keys {
if !index.Keys() {
ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids)
if !ok {
return errors.New("invalid int columns")
@ -243,15 +250,28 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
}
case "int":
value, exists, err := field.Value(col)
if err != nil {
return errors.Wrap(err, "getting int field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
if field.Keys() {
value, exists, err := field.StringValue(col)
if err != nil {
return errors.Wrap(err, "getting string field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}})
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
}
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
value, exists, err := field.Value(col)
if err != nil {
return errors.Wrap(err, "getting int field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
}
}
case "decimal":
@ -306,10 +326,19 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
}
} else {
keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys)
if !ok {
var cols []string
switch keys := req.Columns.Type.(type) {
case *pb.IdsOrKeys_Ids:
// The default behavior (in api/client/grpc.go) is to
// send an empty set of Ids even if the index supports
// keys, so in that case we just need to ignore it.
case *pb.IdsOrKeys_Keys:
cols = keys.Keys.Vals
default:
return errToStatusError(errors.New("invalid key columns"))
}
ci := []*pb.ColumnInfo{
{Name: "_id", Datatype: "string"},
}
@ -319,7 +348,6 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
// If Columns is empty, then get the _exists list (via All()),
// from the index and loop over that instead.
cols := keys.Keys.Vals
if len(cols) > 0 {
// Apply limit/offset to the provided columns.
if int(offset) >= len(cols) {
@ -426,16 +454,30 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer
return errors.Wrap(err, "translating column key")
}
value, exists, err := field.Value(id)
if err != nil {
return errors.Wrap(err, "getting int field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
if field.Keys() {
value, exists, err := field.StringValue(id)
if err != nil {
return errors.Wrap(err, "getting string field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}})
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
}
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
value, exists, err := field.Value(id)
if err != nil {
return errors.Wrap(err, "getting int field value for column")
} else if exists {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
} else {
rowResp.Columns = append(rowResp.Columns,
&pb.ColumnResponse{ColumnVal: nil})
}
}
case "decimal":
// Translate column key.
id, err := index.TranslateStore().TranslateKey(col)