diff --git a/api.go b/api.go index 74dd32532..c8f792992 100644 --- a/api.go +++ b/api.go @@ -367,8 +367,8 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // CreateField makes the named field in the named index with the given options. -// This method currently only takes a single functional option, but that may be -// changed in the future to support multiple options. +// +// The resulting field will always have TrackExistence set. func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField") defer span.Finish() @@ -381,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str // authN/Z info requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx + // newFieldOptions is also used in the path through the index creating + // a field from an update from DAX, so it can't assume it can always + // override this. But we're the call path for creating new fields, and + // new fields should always have TrackExistence on. + opts = append(opts, OptFieldTrackExistence()) // Apply and validate functional options. fo, err := newFieldOptions(opts...) if err != nil { @@ -500,10 +505,16 @@ func importWorker(importWork chan importJob) { // incorrectly). One way to address this would be to change the logic // overall so there weren't conflicts. For now, we just // rely on the field type to inform the intended view name. - if viewName == "" { + // contrast with cleanupView, which is similar but unfortunately not quite identical + switch viewName { + case "": viewName = viewStandard - } else if j.field.Type() == FieldTypeTime { - viewName = fmt.Sprintf("%s_%s", viewStandard, viewName) + case viewStandard, viewExistence: + // do nothing, these are fine + default: // possibly a time view + if j.field.Type() == FieldTypeTime && !strings.HasPrefix(viewName, viewStandard) { + viewName = viewStandard + "_" + viewName + } } if len(viewData) == 0 { return fmt.Errorf("no data to import for view: %s", viewName) @@ -1316,7 +1327,6 @@ type ImportOptions struct { Clear bool IgnoreKeyCheck bool Presorted bool - fullySorted bool // format-aware sorting, internal use only please. suppressLog bool // test Tx atomicity if > 0 @@ -1523,7 +1533,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, return errors.Wrap(err, "validating api method") } - api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard) idx, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") @@ -1642,6 +1651,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, // across many fields in a single shard. It can both set and clear // bits and updates caches/bitDepth as appropriate, although only the // bitmap parts happen truly transactionally. +// +// This function does not attempt to do existence tracking, because +// it can't; there's no way to distinguish empty sets from not setting +// bits. As a result, users of this endpoint are responsible for +// providing corrected existence views for fields with existence +// tracking. Our batch API does that. func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error { index, err := api.Index(ctx, indexName) if err != nil { @@ -1768,12 +1783,15 @@ func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { // TODO wouldn't hurt to have consolidated logic somewhere for validating view names. switch fieldType { case FieldTypeSet, FieldTypeTime: - if viewUpdate.View == "" { - viewUpdate.View = "standard" - } - // add 'standard_' if we just have a time... this is how IDK works by default - if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { - viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View) + switch viewUpdate.View { + case "": + viewUpdate.View = viewStandard + case viewStandard, viewExistence: + // do nothing, these are fine + default: + if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { + viewUpdate.View = viewStandard + "_" + viewUpdate.View + } } case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: if viewUpdate.View == "" { @@ -2038,21 +2056,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { +func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) { ef := index.existenceField() if ef == nil { return nil } - - existenceRowIDs := make([]uint64, len(columnIDs)) - // If we don't gratuitously hand-duplicate things in field.Import, - // the fact that fragment.bulkImport rewrites its row and column - // lists can burn us if we don't make a copy before doing the - // existence field write. - columnCopy := make([]uint64, len(columnIDs)) - copy(columnCopy, columnIDs) - options := ImportOptions{} - return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options) + tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) + if err != nil { + return err + } + defer finisher(&err0) + // markExistingInView is simpler/faster than Import, but unusually, we use the + // standard view of the existence field, instead of the existence view of + // a specific field, when doing the index-wide update. + return ef.markExistingInView(tx, columnIDs, viewStandard, shard) } // ShardDistribution returns an object representing the distribution of shards diff --git a/api_directive.go b/api_directive.go index b61df8f6e..e04b4b096 100644 --- a/api_directive.go +++ b/api_directive.go @@ -964,7 +964,7 @@ func createField(idx *Index, fld *dax.Field) error { return errors.Wrapf(err, "creating field options from field: %s", fld.Name) } - if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil { + if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil { return errors.Wrapf(err, "creating field on index: %s", fld.Name) } return nil diff --git a/batch/batch.go b/batch/batch.go index e9e275774..60c8aba7f 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -573,7 +573,7 @@ func (b *Batch) Add(rec Row) error { case int64: b.values[field.Name] = append(b.values[field.Name], val) case []string: - if len(val) == 0 { + if val == nil { continue } rowIDSets, ok := b.rowIDSets[field.Name] @@ -608,7 +608,8 @@ func (b *Batch) Add(rec Row) error { } b.rowIDSets[field.Name] = append(rowIDSets, rowIDs) case []uint64: - if len(val) == 0 { + // if length is 0, that's still a valid, empty, set + if val == nil { continue } rowIDSets, ok := b.rowIDSets[field.Name] @@ -663,6 +664,9 @@ func (b *Batch) Add(rec Row) error { for i, uval := range rec.Clears { field := b.header[i] + if field.Options.Type == featurebase.FieldTypeMutex && uval != nil { + return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex") + } if _, ok := b.clearRowIDs[i]; !ok { b.clearRowIDs[i] = make(map[int]uint64) } @@ -1245,7 +1249,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error { } ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false) - b.log.Debugf("imp-roar field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) + b.log.Debugf("imp-roar field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty)) return errors.Wrapf(ferr, "importing data for %s", field) }) } @@ -1343,6 +1347,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments curShard := ^uint64(0) // impossible sentinel value for shard. var curBM *roaring.Bitmap var clearBM *roaring.Bitmap + var existCurBM *roaring.Bitmap for j := range b.ids { col := b.ids[j] row := nilSentinel @@ -1355,8 +1360,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments if col/shardWidth != curShard { curShard = col / shardWidth + // the API treats "" as standard curBM = frags.GetOrCreate(curShard, field.Name, "") clearBM = clearFrags.GetOrCreate(curShard, field.Name, "") + if opts.TrackExistence { + existCurBM = frags.GetOrCreate(curShard, field.Name, "existence") + } } if row != nilSentinel { // TODO this is super ugly, but we want to avoid setting @@ -1366,6 +1375,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments // the NoStandardView case would be great. if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) { curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + if opts.TrackExistence { + existCurBM.DirectAdd(col % shardWidth) + } } if opts.Type == featurebase.FieldTypeTime { views, err := b.times[j].views(opts.TimeQuantum) @@ -1386,6 +1398,11 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments // we want to make sure that at this point, the "set" // fragments don't contain the bit that we're clearing curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth)) + // don't set the existence bit, probably? i don't actually quite + // understand the higher level semantics here. + if opts.TrackExistence { + existCurBM.DirectRemoveN(col % shardWidth) + } } } } @@ -1404,14 +1421,22 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments opts := field.Options curShard := ^uint64(0) // impossible sentinel value for shard. var curBM *roaring.Bitmap + var existCurBM *roaring.Bitmap for j := range b.ids { col, rowIDs := b.ids[j], rowIDSets[j] - if len(rowIDs) == 0 { - continue - } if col/shardWidth != curShard { curShard = col / shardWidth curBM = frags.GetOrCreate(curShard, fname, "") + if opts.TrackExistence { + existCurBM = frags.GetOrCreate(curShard, fname, "existence") + } + } + if len(rowIDs) == 0 { + // you can validly specify an empty set, which is not the same as a null + if opts.TrackExistence && !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) && rowIDs != nil { + existCurBM.DirectAdd(col % shardWidth) + } + continue } // TODO this is super ugly, but we want to avoid setting // bits on the standard view in the specific case when @@ -1422,6 +1447,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments for _, row := range rowIDs { curBM.DirectAdd(row*shardWidth + (col % shardWidth)) } + if opts.TrackExistence { + existCurBM.DirectAdd(col % shardWidth) + } } if opts.Type == featurebase.FieldTypeTime { views, err := b.times[j].views(opts.TimeQuantum) @@ -1549,6 +1577,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, shard := ids[0] / shardWidth bitmap := frags.GetOrCreate(shard, field.Name, "standard") clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard") + var existBM, existClearBM *roaring.Bitmap + if field.Options.TrackExistence { + existBM = frags.GetOrCreate(shard, field.Name, "existence") + existClearBM = clearFrags.GetOrCreate(shard, field.Name, "existence") + } for i, id := range ids { if i+1 < len(ids) { // we only want the last value set for each id @@ -1561,6 +1594,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, shard = id / shardWidth bitmap = frags.GetOrCreate(shard, field.Name, "standard") clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard") + if field.Options.TrackExistence { + existBM = frags.GetOrCreate(shard, field.Name, "existence") + existClearBM = clearFrags.GetOrCreate(shard, field.Name, "existence") + } } fragmentColumn := id % shardWidth clearBM.Add(fragmentColumn) // Will use this to clear columns. @@ -1568,6 +1605,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, // clearSentinel is used for deletion // so this value should only be added if its not clearSentinel bitmap.Add(row*shardWidth + fragmentColumn) + if field.Options.TrackExistence { + existBM.Add(fragmentColumn) + } + } else if field.Options.TrackExistence { + existClearBM.Add(fragmentColumn) } } } @@ -1596,6 +1638,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, fragmentColumn := recID % shardWidth clearBM.Add(fragmentColumn) + if field.Options.TrackExistence { + existClearBM := clearFrags.GetOrCreate(shard, field.Name, "existence") + + existClearBM.Add(fragmentColumn) + } } } @@ -1618,6 +1665,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, fragmentColumn := recID % shardWidth clearBM.Add(fragmentColumn) + if field.Options.TrackExistence { + exist := frags.GetOrCreate(shard, field.Name, "existence") + exist.Add(fragmentColumn) + } if boolVal { bitmap.Add(trueRowOffset + fragmentColumn) diff --git a/batch/batch_test.go b/batch/batch_test.go index bcc5a65c0..cf214813d 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea Index: idx.Name, Query: "TopN(a1, n=10)", }) + if resp.Err != nil { + t.Fatalf("unexpected error from TopN query: %v", resp.Err) + } + if len(resp.Results) < 1 { + t.Fatalf("expected non-empty result set, got empty results") + } pairsField, ok := resp.Results[0].(*featurebase.PairsField) assert.True(t, ok, "wrong return type: %T", resp.Results[0]) @@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap { Name: "strslice", Options: featurebase.FieldOptions{ - Type: featurebase.FieldTypeSet, - Keys: true, - CacheType: featurebase.CacheTypeRanked, - CacheSize: 100, + Type: featurebase.FieldTypeSet, + Keys: true, + CacheType: featurebase.CacheTypeRanked, + CacheSize: 100, + TrackExistence: true, }, }, }, @@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap pql: "Row(strslice='z')", exp: []uint64{2}, }, + { + pql: "Row(strslice==null)", + exp: []uint64{1}, + }, + { + pql: "Row(strslice!=null)", + exp: []uint64{0, 2, 3, 4}, + }, } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { diff --git a/catcher.go b/catcher.go index 46c8af074..933e6ea52 100644 --- a/catcher.go +++ b/catcher.go @@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) return c.b.Remove(index, field, view, shard, a...) } +func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Removed(index, field, view, shard, a...) +} + func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { defer func() { diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 57ab9515e..0ed57a9c1 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -3302,6 +3302,9 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT return nil, errors.Wrapf(err, "orch: translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { + if ids == nil { + return []string(nil), nil + } keys := make([]string, len(ids)) for i, id := range ids { keys[i] = translations[id] @@ -3311,9 +3314,6 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT } else { datatype = "[]uint64" mapper = func(ids []uint64) (interface{}, error) { - if ids == nil { - ids = []uint64{} - } return ids, nil } } diff --git a/dax/table.go b/dax/table.go index fa37191fc..2f90cdaff 100644 --- a/dax/table.go +++ b/dax/table.go @@ -823,4 +823,5 @@ type FieldOptions struct { TimeQuantum TimeQuantum `json:"time-quantum,omitempty"` TTL time.Duration `json:"ttl,omitempty"` ForeignIndex string `json:"foreign-index,omitempty"` + TrackExistence bool `json:"track-existence"` } diff --git a/delete_test.go b/delete_test.go index 1ebb7bdc4..03047d201 100644 --- a/delete_test.go +++ b/delete_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "sort" + "strings" "testing" "time" @@ -232,8 +233,9 @@ func TestExecutor_DeleteRecords(t *testing.T) { t.Run("DeleteWithBitmapError", func(t *testing.T) { setup(t, require, c) defer tearDown(t, require, c) - _, err := c.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: `Delete(Row(setfield == 1))`}) - if err == nil || err.Error() != `executing: executeDelete: mapping on primary node: bsigroup not found` { + _, err := c.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: `Delete(Row(setfield > 1))`}) + // we don't allow `>` operators on set fields + if err == nil || !strings.Contains(err.Error(), "row call: only support") { t.Fatalf("unexpected error: %s", err) } }) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index dc8c2ad5e..b1fce6e41 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -637,6 +637,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions Keys: o.Keys, ForeignIndex: o.ForeignIndex, NoStandardView: o.NoStandardView, + TrackExistence: o.TrackExistence, } } @@ -973,6 +974,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field m.Keys = options.Keys m.ForeignIndex = options.ForeignIndex m.NoStandardView = options.NoStandardView + m.TrackExistence = options.TrackExistence } func (s Serializer) decodeDecimal(d *pb.Decimal, m *pql.Decimal) { diff --git a/executor.go b/executor.go index 6f44885d3..5b804c4c2 100644 --- a/executor.go +++ b/executor.go @@ -1525,7 +1525,22 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string if !opt.Remote { switch c.Name { case "Row": - if c.HasConditionArg() { + // We used to do this by checking for "Condition", but if we allow + // checks against null for non-BSI fields, that's not accurate. We + // can't do this check down in the per-shard stuff, because if we + // did it there, we'd be incrementing the stats once per shard. + // This is redundant with what we do there but shouldn't change + // its behavior, except for possibly the spelling of the diagnostic. + fieldName, err := c.FieldArg() + if err != nil { + return nil, err + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return nil, fmt.Errorf("row call with unknown field %s:%s", index, fieldName) + } + bsig := field.bsiGroup(fieldName) + if bsig != nil { statFn(CounterQueryRowBSITotal) } else { statFn(CounterQueryRowTotal) @@ -4624,6 +4639,27 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return ExtractedIDMatrix{}, newNotFoundError(ErrFieldNotFound, name) } + if field.options.TrackExistence { + existenceRow, err := field.Existing(tx, shard) + if err != nil { + return ExtractedIDMatrix{}, fmt.Errorf("trying to find existence bit: %w", err) + } + // filter out any rows which exist, but aren't in this data... + if existenceRow != nil { + existenceRow = existenceRow.Intersect(colsBitmap) + existing := existenceRow.Columns() + for _, col := range existing { + m[mLookup[col]].Rows[i] = []uint64{} + } + } + } else if field.options.Type == FieldTypeSet || field.options.Type == FieldTypeTime { + // time quantums and sets which don't have track-existence should treat every + // column as a non-null empty set if it has no bits, rather than as a null. + for _, col := range cols { + m[mLookup[col]].Rows[i] = []uint64{} + } + } + switch field.Type() { case FieldTypeSet, FieldTypeMutex: // Handle a set field by listing the rows and then intersecting them with the filter. @@ -4828,15 +4864,82 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return matrix, nil } +// getNullRowShard requests a row representing the elements of this shard +// which are null, that is to say, the elements that exist for the index, +// but not for this particular field. This is computed as row 0 of the +// index's existence field, minus the bsiExistsBit of the existenceView. +// We're using bsiExistsBit for consistency between BSI fields and non-BSI +// fields. In an ideal world, we'd have done this sooner and BSI fields +// would also be using a separate existenceView, perhaps? But we are not +// in that world. +func (e *executor) getNullRowShard(ctx context.Context, tx Tx, idx *Index, fld *Field, existenceView string, shard uint64) (*Row, error) { + // Make sure the index supports existence tracking. + if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", idx.name) + } + // existenceView: the view we've been asked to look in for existence data + // viewExistence: the string "existence", similar to the name "viewStandard", + // denoting the existence view we use for non-BSI fields. BSI fields always + // track existence. + if !fld.options.TrackExistence && existenceView == viewExistence { + return nil, fmt.Errorf("field does not support existence tracking: %s", fld.name) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(idx.name, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + // no existence frag -> no existence bits are set -> there are no + // records in this shard. therefore no records in this shard are + // null. more simply, we don't have to compute the things we want + // to subtract from an empty row to figure out that the result + // will stay empty. + return NewRow(), nil + } else { + var err error + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + return nil, err + } + } + + var notNull *Row + var err error + + // Retrieve notNull from fragment if it exists. + if frag := e.Holder.fragment(idx.name, fld.name, existenceView, shard); frag != nil { + if notNull, err = frag.notNull(tx); err != nil { + return nil, errors.Wrap(err, "getting fragment not null") + } + } else { + return existenceRow, nil + } + + return existenceRow.Difference(notNull), nil +} + +// getNonNullRowShard requests a row representing the existence of this shard +// which are not null. We trust the existenceView for this field to be +// correct, and not contain bits which don't exist at the index level, so we +// are not checking against the index's existence field. +func (e *executor) getNonNullRowShard(ctx context.Context, tx Tx, idx *Index, fld *Field, existenceView string, shard uint64) (*Row, error) { + // existenceView: the view we've been asked to look in for existence data + // viewExistence: the string "existence", similar to the name "viewStandard", + // denoting the existence view we use for non-BSI fields. BSI fields always + // track existence. + if !fld.options.TrackExistence && existenceView == viewExistence { + return nil, fmt.Errorf("field does not support existence tracking: %s", fld.name) + } + // Retrieve fragment. + frag := e.Holder.fragment(idx.name, fld.name, existenceView, shard) + if frag == nil { + return NewRow(), nil + } + return frag.notNull(tx) +} + func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowShard") defer span.Finish() - // Handle bsiGroup ranges differently. - if c.HasConditionArg() { - return e.executeRowBSIGroupShard(ctx, qcx, index, c, shard) - } - // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -4853,6 +4956,11 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, return nil, newNotFoundError(ErrFieldNotFound, fieldName) } + bsig := f.bsiGroup(fieldName) + if bsig != nil { + return e.executeRowBSIGroupShard(ctx, qcx, f, bsig, c, shard) + } + err = e.validateTimeCallArgs(c, index) if err != nil { return nil, err @@ -4874,15 +4982,31 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, } } - rowID, rowOK, rowErr := c.UintArg(fieldName) - if rowErr != nil { - return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) - } else if !rowOK { - return nil, fmt.Errorf("Row() must specify %v", rowLabel) + isNull, rowID, isEQ, err := c.FieldEquality(fieldName) + if err != nil { + return nil, fmt.Errorf("row call: %v", err) } // Return row if times are not set and standard view exists. timeNotSet := fromTime.IsZero() && toTime.IsZero() + if isNull { + if !timeNotSet { + return nil, errors.New("can't use a time range with a check for/against null") + } + tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) + if err != nil { + return nil, err + } + defer finisher(&err0) + if isEQ { + return e.getNullRowShard(ctx, tx, f.idx, f, viewExistence, shard) + } else { + return e.getNonNullRowShard(ctx, tx, f.idx, f, viewExistence, shard) + } + } + if !isEQ { + return nil, errors.New("only support != for null, not for other values, on set/mutex fields") + } if c.Name == "Row" && timeNotSet && !f.options.NoStandardView { frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { @@ -4941,7 +5065,7 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, } // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { +func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, fld *Field, bsig *bsiGroup, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowBSIGroupShard") defer span.Finish() @@ -4952,23 +5076,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index return nil, errors.New("Row(): too many arguments") } - // Extract conditional. - var fieldName string - var cond *pql.Condition - for k, v := range c.Args { - vv, ok := v.(*pql.Condition) - if !ok { - return nil, fmt.Errorf("Row(): %q: expected condition argument, got %v", k, v) - } - fieldName, cond = k, vv + op, value, err := c.FieldRange(fld.name) + if err != nil { + return nil, err } + viewName := viewBSIGroupPrefix + fld.name - f := e.Holder.Field(index, fieldName) - if f == nil { - return nil, newNotFoundError(ErrFieldNotFound, fieldName) - } - - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: f.idx, Shard: shard}) + tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: fld.idx, Shard: shard}) if err != nil { return nil, err } @@ -4987,50 +5101,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index // NEQ frag.RangeOp // Handle `!= null` and `== null`. - if cond.Op == pql.NEQ && cond.Value == nil { - // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) - if frag == nil { - return NewRow(), nil - } - return frag.notNull(tx) - - } else if cond.Op == pql.EQ && cond.Value == nil { - // Make sure the index supports existence tracking. - idx := e.Holder.Index(index) - if idx == nil { - return nil, newNotFoundError(ErrIndexNotFound, index) - } else if idx.existenceField() == nil { - return nil, errors.Errorf("index does not support existence tracking: %s", index) - } - - var existenceRow *Row - existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) - if existenceFrag == nil { - existenceRow = NewRow() - } else { - if existenceRow, err0 = existenceFrag.row(tx, 0); err0 != nil { - return nil, err0 - } - } - - var notNull *Row - var err error - - // Retrieve notNull from fragment if it exists. - if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { - if notNull, err = frag.notNull(tx); err != nil { - return nil, errors.Wrap(err, "getting fragment not null") - } - } else { - notNull = NewRow() - } - - return existenceRow.Difference(notNull), nil - - } else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT || - cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE { - predicates, err := getCondIntSlice(f, cond) + if op == pql.NEQ && value == nil { + return e.getNonNullRowShard(ctx, tx, fld.idx, fld, viewName, shard) + } else if op == pql.EQ && value == nil { + return e.getNullRowShard(ctx, tx, fld.idx, fld, viewName, shard) + } else if op == pql.BETWEEN || op == pql.BTWN_LT_LT || + op == pql.BTWN_LTE_LT || op == pql.BTWN_LT_LTE { + predicates, err := getCondIntSlice(fld, op, value) if err != nil { return nil, errors.Wrap(err, "getting condition value") } @@ -5044,19 +5121,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index // return f.RowBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be shard-specific. - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + frag := e.Holder.fragment(fld.idx.name, fld.name, viewName, shard) if frag == nil { return NewRow(), nil } @@ -5070,40 +5141,34 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index return frag.rangeBetween(tx, bsig.BitDepth, baseValueMin, baseValueMax) } else { - value, err := getScaledInt(f, cond.Value) + value, err := getScaledInt(fld, value) if err != nil { return nil, errors.Wrap(err, "getting scaled integer") } - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - - baseValue, outOfRange := bsig.baseValue(cond.Op, value) - if outOfRange && cond.Op != pql.NEQ { + baseValue, outOfRange := bsig.baseValue(op, value) + if outOfRange && op != pql.NEQ { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + frag := e.Holder.fragment(fld.idx.name, fld.name, viewName, shard) if frag == nil { return NewRow(), nil } // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. - if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || - (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { + if (op == pql.LT && value > bsig.Max) || (op == pql.LTE && value >= bsig.Max) || + (op == pql.GT && value < bsig.Min) || (op == pql.GTE && value <= bsig.Min) { return frag.notNull(tx) } // outOfRange for NEQ should return all not-null. - if outOfRange && cond.Op == pql.NEQ { + if outOfRange && op == pql.NEQ { return frag.notNull(tx) } - return frag.rangeOp(tx, cond.Op, bsig.BitDepth, baseValue) + return frag.rangeOp(tx, op, bsig.BitDepth, baseValue) } } @@ -7689,6 +7754,9 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index return nil, errors.Wrapf(err, "translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { + if ids == nil { + return []string(nil), nil + } keys := make([]string, len(ids)) for i, id := range ids { keys[i] = translations[id] @@ -7698,9 +7766,6 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index } else { datatype = "[]uint64" mapper = func(ids []uint64) (interface{}, error) { - if ids == nil { - ids = []uint64{} - } return ids, nil } } @@ -8687,15 +8752,15 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool return ret, false, err } -// getCondIntSlice looks at the field, the cond op type (which is +// getCondIntSlice looks at the field, the op type (which is // expected to be one of the BETWEEN ops types), and the values in the // conditional and returns a slice of int64 which is scaled for // decimal fields and has the values modulated such that the BETWEEN // op can be treated as being of the form a<=x<=b. -func getCondIntSlice(f *Field, cond *pql.Condition) ([]int64, error) { - val, ok := cond.Value.([]interface{}) +func getCondIntSlice(f *Field, op pql.Token, value interface{}) ([]int64, error) { + val, ok := value.([]interface{}) if !ok { - return nil, errors.Errorf("expected conditional to have []interface{} Value, but got %v of %[1]T", cond.Value) + return nil, errors.Errorf("expected conditional to have []interface{} Value, but got %v of %[1]T", value) } ret := make([]int64, len(val)) @@ -8714,7 +8779,7 @@ func getCondIntSlice(f *Field, cond *pql.Condition) ([]int64, error) { return ret, nil } - switch cond.Op { + switch op { case pql.BTWN_LT_LTE: // a < x <= b ret[0]++ case pql.BTWN_LTE_LT: // a <= x < b @@ -8988,7 +9053,6 @@ func DeleteRowsWithOutKeysFlow(ctx context.Context, columns *roaring.Bitmap, idx }() for _, field := range idx.Fields() { for _, view := range field.views() { - frag := view.Fragment(shard) if frag == nil { continue diff --git a/executor_test.go b/executor_test.go index 29ec09e0a..7c61f6f82 100644 --- a/executor_test.go +++ b/executor_test.go @@ -36,6 +36,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var TempDir = getTempDirString() @@ -3033,6 +3034,14 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } + if _, err := idx.CreateField("idset", "", pilosa.OptFieldTypeSet("none", 0)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("tq", "", pilosa.OptFieldTypeTime("YM", "0")); err != nil { + t.Fatal(err) + } + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: ` Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) @@ -3046,6 +3055,10 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { Set(0, other=1000) Set(0, edge=100) Set(1, edge=-100) + Set(0, idset=3) + Set(1, idset=3) + Clear(0, idset=3) + Set(50, tq=5, 2017-01-02T12:34) `}); err != nil { t.Fatal(err) } @@ -3071,6 +3084,46 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatalf("Query().Row.Columns=%#v, expected %#v", got, exp) } + // time quantum EQ null + _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq == null, from=2010-01-01T00:00)`}) + if err == nil { + t.Fatalf("expected error from invalid time quantum null query") + } + if !strings.Contains(err.Error(), "time range with a check") { + t.Fatalf("unexpected error; expecting can't use time range with a null check, got %v", err) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ + 0, + 1, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ + 50, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset == 3)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + // EQ (single = form) if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(foo = 20)`}); err != nil { t.Fatal(err) @@ -3098,6 +3151,24 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset != 3)`}); err == nil { + t.Fatal("expected error from trying != 3 on a set field") + } else if !strings.Contains(err.Error(), "only support != for null") { + t.Fatalf("expected error about only supporting != for null, got %v", err) + } }) t.Run("LT", func(t *testing.T) { @@ -4993,7 +5064,7 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: 0}, Rows: []interface{}{ - []uint64{}, + []uint64(nil), []string{ "h", "plugh", @@ -5044,11 +5115,11 @@ func TestExecutor_Execute_Extract(t *testing.T) { []uint64{ 0, }, - []string{}, + []string(nil), uint64(0), nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5058,14 +5129,14 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: 3}, Rows: []interface{}{ - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, "plugh", []uint64{ 3, }, - []string{}, + []string(nil), int64(2), pql.NewDecimal(-101, 2), time.Date(2000, time.January, 1, 0, 0, 3, 0, time.UTC), @@ -5075,12 +5146,12 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: ShardWidth}, Rows: []interface{}{ - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5093,11 +5164,11 @@ func TestExecutor_Execute_Extract(t *testing.T) { []uint64{ 4, }, - []string{}, + []string(nil), uint64(4), nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5107,10 +5178,7 @@ func TestExecutor_Execute_Extract(t *testing.T) { }, }, } - - if !reflect.DeepEqual(expect, resp.Results) { - t.Errorf("expected %v but got %v", expect, resp.Results) - } + require.Equal(t, expect, resp.Results) } func TestExecutor_Execute_Extract_Keyed(t *testing.T) { @@ -5223,7 +5291,7 @@ func TestExecutor_Execute_MaxMemory(t *testing.T) { { Column: pilosa.KeyOrID{ID: ShardWidth}, Rows: []interface{}{ - []uint64{}, + []uint64(nil), }, }, { @@ -5237,10 +5305,7 @@ func TestExecutor_Execute_MaxMemory(t *testing.T) { }, }, } - - if !reflect.DeepEqual(expect, resp.Results) { - t.Errorf("expected %v but got %v", expect, resp.Results) - } + require.Equal(t, expect, resp.Results) } func TestExecutor_Execute_Rows(t *testing.T) { diff --git a/field.go b/field.go index 67b96b40f..f8e3251e3 100644 --- a/field.go +++ b/field.go @@ -376,13 +376,27 @@ func OptFieldTypeBool() FieldOption { } } +// OptFieldTrackExistence exists mostly to allow the +// FieldFromFieldOptions/FieldOptionsFromField round-trip to work. +// If you are actually creating a field, via api.CreateField, +// it will be turned on unconditionally. You can't turn it +// off. +func OptFieldTrackExistence() FieldOption { + return func(fo *FieldOptions) error { + fo.TrackExistence = true + return nil + } +} + // newField returns a new instance of field (without name validation). -func newField(holder *Holder, path, index, name string, opts FieldOption) (*Field, error) { +func newField(holder *Holder, path, index, name string, opts ...FieldOption) (*Field, error) { // Apply functional option. fo := FieldOptions{} - err := opts(&fo) - if err != nil { - return nil, errors.Wrap(err, "applying option") + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } } f := &Field{ @@ -1253,6 +1267,15 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo } else if v { changed = v } + if f.options.TrackExistence { + view, err := f.createViewIfNotExists(viewExistence) + if err != nil { + return changed, errors.Wrap(err, "creating existence view") + } + if _, err := view.setBit(qcx, bsiExistsBit, colID); err != nil { + return changed, errors.Wrap(err, "setting existence on view") + } + } } // Exit early if no timestamp is specified. @@ -1278,6 +1301,9 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo } // ClearBit clears a bit within the field. +// +// This does not, for now, create existence bits for the field, because it +// doesn't create them for the index. func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error) { viewName := viewStandard @@ -1292,8 +1318,20 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error return false, errors.Wrap(err, "clearing on view") } else if v { changed = changed || v + if changed && f.options.TrackExistence && f.options.Type == FieldTypeMutex { + // we also want to try to clear any existence bit + existView, ok := f.viewMap[viewExistence] + if ok { + _, err := existView.clearBit(qcx, 0, colID) + if err != nil { + return false, errors.Wrap(err, "clearing existence bit") + } + } + } } - if len(f.viewMap) == 1 { // assuming no time views + // We used to check length of view map here. Now that we might + // have an existence view, that won't work. + if f.options.Type != FieldTypeTime { // assuming no time views return changed, nil } lastViewNameSize := 0 @@ -1323,30 +1361,6 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error return changed, nil } -// ClearBits clears all bits corresponding to the given record IDs in standard -// or BSI views. It does not delete bits from time quantum views. -func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error { - bsig := f.bsiGroup(f.name) - var v *view - if bsig != nil { - // looks like we're a BSI field? - v = f.view(viewBSIGroupPrefix + f.name) - } else { - v = f.view(viewStandard) - } - // it's fine if we never actually created the view, that means the - // bits are all clear! - if v == nil { - return nil - } - frag := v.Fragment(shard) - if frag == nil { - return nil - } - _, err := frag.ClearRecords(tx, recordIDs) - return err -} - func groupCompare(a, b string, offset int) (lt, eq bool) { if len(a) > offset { a = a[:offset] @@ -1369,6 +1383,12 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { i++ } } + // return the empty list if there weren't any. this could happen + // if we got called because this is a time field, but in fact + // no time views have been created. + if i == 0 { + return me[:0] + } me = me[:i] year := strings.Index(me[0].name, "_") + 4 month := year + 2 @@ -1611,6 +1631,92 @@ func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Ro return view.rangeOp(qcx, op, bsig.BitDepth, baseValue) } +// existenceViewName reports the field we should use row 0 of +// for existence data. For a BSI field (integer, decimal, +// timestamp) this is the single BSI group. For other fields, +// it's viewExistence, which is probably "existence". +func (f *Field) existenceViewName() string { + if len(f.bsiGroups) > 0 { + return f.bsiGroups[0].Name + } + return viewExistence +} + +// MarkExisting sets a range of column IDs as existing. The columnIDs +// are assumed to include the shard offset, but this will also work if +// they are shard-relative, as it's just stripping the offset. +// +// Positions aren't the same as column IDs; this function takes advantage +// of the fact that we're always doing row 0, so we don't have to think +// hard about this. It doesn't overwrite its input because the column IDs +// could be reused by other things. +// +// Note that this is subtly inefficient; if you're tracking existence for +// a field, we're computing the same column ID set to write to the index's +// existence field as we're using for the field's existence view. We don't +// have a good way to coalesce those, yet. (Also, that's not accurate in +// the ImportValue case, where we don't write to the existence view, etc.) +func (f *Field) MarkExisting(tx Tx, columnIDs []uint64, shard uint64) error { + return f.markExistingInView(tx, columnIDs, f.existenceViewName(), shard) +} + +// markExistingInView implements the internals of MarkExisting, but lets +// you use a non-standard view. It's only interesting for the existence field. +func (f *Field) markExistingInView(tx Tx, columnIDs []uint64, viewName string, shard uint64) error { + copyCols := make([]uint64, len(columnIDs)) + for i := range columnIDs { + copyCols[i] = columnIDs[i] % ShardWidth + } + eView, err := f.createViewIfNotExists(viewName) + if err != nil { + return errors.Wrapf(err, "creating view %s", viewName) + } + + eFrag, err := eView.CreateFragmentIfNotExists(shard) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + return eFrag.importPositions(tx, copyCols, nil, map[uint64]struct{}{0: {}}) +} + +// MarkNotExisting is just like MarkExisting, except it is clearing bits, +// so it doesn't have to create the view or fragment if it doesn't exist. +// Because the bits reported to us in the case we wrote this for are likely +// to be sorted by position in the fragment, not by column ID, we sort the +// list after stripping the rows from the positions. +func (f *Field) MarkNotExisting(tx Tx, columnIDs []uint64, shard uint64) error { + viewName := f.existenceViewName() + v := f.view(viewName) + if v == nil { + return nil + } + frag := v.Fragment(shard) + if frag == nil { + return nil + } + copyCols := make([]uint64, len(columnIDs)) + for i := range columnIDs { + copyCols[i] = columnIDs[i] % ShardWidth + } + sort.Slice(copyCols, func(i, j int) bool { return copyCols[i] < copyCols[j] }) + return frag.importPositions(tx, nil, copyCols, map[uint64]struct{}{0: {}}) +} + +// Existing returns the existence row for this field, which +// comes from either the BSI view or the existence view. +func (f *Field) Existing(tx Tx, shard uint64) (*Row, error) { + viewName := f.existenceViewName() + v := f.view(viewName) + if v == nil { + return nil, nil + } + frag := v.Fragment(shard) + if frag == nil { + return nil, nil + } + return frag.row(tx, bsiExistsBit) +} + // Import bulk imports data. func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) { // Determine quantum if timestamps are set. @@ -1622,6 +1728,9 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return errors.New("import clear is not supported with timestamps") } } else { + if f.options.NoStandardView { + return errors.New("can't import data with no timestamps into a field with no standard view") + } // short path: if we don't have any timestamps, we only need // to write to exactly one view, which is always viewStandard, // and *every* bit goes into that view, and we already verified that @@ -1649,7 +1758,33 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, if err != nil { return errors.Wrap(err, "creating fragment") } - + if f.options.TrackExistence { + // if we're clearing a mutex, we do something fancy. otherwise, + // if we're not clearing, we mark the existence bits. either way, + // we then fall on out to the default behavior of importing the + // bits. + switch { + case options.Clear && fieldType == FieldTypeMutex: + // special fancy case; we have to try to clear the bits first, to + // find out WHICH bits we cleared, so we can mark those bits as + // null. + var changed []uint64 + changed, err1 = frag.clearBitsReportingChanges(tx, rowIDs, columnIDs) + if err1 != nil { + return err1 + } + err1 = f.MarkNotExisting(tx, changed, shard) + return err1 + case !options.Clear: + err1 = f.MarkExisting(tx, columnIDs, shard) + if err1 != nil { + return err1 + } + default: + // nothing to do. we'll fall on out of the TrackExistence + // case and go ahead and import those bits naively + } + } err1 = frag.bulkImport(tx, rowIDs, columnIDs, options) return err1 } @@ -1737,6 +1872,16 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return err1 } } + // If we are tracking existence, and don't have NoStandardView, we + // create the existence view. + if f.options.TrackExistence && !f.options.NoStandardView { + // this dance with err1 is so the finisher gets called with + // the right error value if we hit an error + err1 = f.MarkExisting(tx, columnIDs, shard) + if err1 != nil { + return err1 + } + } return nil } @@ -1991,6 +2136,7 @@ type FieldOptions struct { Scale int64 `json:"scale,omitempty"` Keys bool `json:"keys"` NoStandardView bool `json:"noStandardView,omitempty"` + TrackExistence bool `json:"trackExistence,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` CacheType string `json:"cacheType,omitempty"` Type string `json:"type,omitempty"` diff --git a/fragment.go b/fragment.go index 5be779cb5..4a8f1964a 100644 --- a/fragment.go +++ b/fragment.go @@ -1507,32 +1507,60 @@ func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *Import return f.bulkImportStandard(tx, rowIDs, columnIDs, options) } -// rowColumnSet is a sortable set of row and column IDs which -// correspond, allowing us to ensure that we produce values in -// a predictable order -type rowColumnSet struct { - r []uint64 - c []uint64 -} - -func (r rowColumnSet) Len() int { - return len(r.r) -} - -func (r rowColumnSet) Swap(i, j int) { - r.r[i], r.r[j] = r.r[j], r.r[i] - r.c[i], r.c[j] = r.c[j], r.c[i] -} - -// Sort by row ID first, column second, to sort by fragment position -func (r rowColumnSet) Less(i, j int) bool { - if r.r[i] < r.r[j] { - return true +// clearBitsReportingChanges is a special fancy case. For existence-tracking, +// if we're clearing bits in a mutex, *successfully* cleared bits become null +// records, so we have to report, not how many records we cleared, but which +// records specifically became clear. The returned set of bits is the column +// IDs that actually got a bit cleared from them. +// +// This is basically following the logic of bulkImportStandard and +// importPositions, except that it combines them and drops some of the +// no longer needed branches. +func (f *fragment) clearBitsReportingChanges(tx Tx, rowIDs, columnIDs []uint64) ([]uint64, error) { + // Verify that there are an equal number of row ids and column ids. + if len(rowIDs) != len(columnIDs) { + return nil, fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } - if r.r[i] > r.r[j] { - return false + + // rowSet maintains the set of rowIDs present in this import. It allows the + // cache to be updated once per row, instead of once per bit. TODO: consider + // sorting by rowID/columnID first and avoiding the map allocation here. (we + // could reuse rowIDs to store the list of unique row IDs) + rowSet := make(map[uint64]struct{}) + lastRowID := uint64(1 << 63) + + // replace columnIDs with calculated positions to avoid allocation. + prevRow, prevCol := ^uint64(0), ^uint64(0) + next := 0 + for i := 0; i < len(columnIDs); i++ { + rowID, columnID := rowIDs[i], columnIDs[i] + if rowID == prevRow && columnID == prevCol { + continue + } + prevRow, prevCol = rowID, columnID + pos, err := f.pos(rowID, columnID) + if err != nil { + return nil, err + } + columnIDs[next] = pos + next++ + + // Add row to rowSet. + if rowID != lastRowID { + lastRowID = rowID + rowSet[rowID] = struct{}{} + } } - return r.c[i] < r.c[j] + clear := columnIDs[:next] + f.mu.Lock() + defer f.mu.Unlock() + CounterClearingingN.Add(float64(len(clear))) + changed, err := tx.Removed(f.index(), f.field(), f.view(), f.shard, clear...) + if err != nil { + return nil, errors.Wrap(err, "clearing positions") + } + CounterClearedN.Add(float64(len(changed))) + return changed, f.updateCaching(tx, rowSet) } // bulkImportStandard performs a bulk import on a standard fragment. May mutate @@ -1545,11 +1573,6 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options rowSet := make(map[uint64]struct{}) lastRowID := uint64(1 << 63) - // It's possible for the ingest API to have already sorted things in - // the row-first order we want for this import. - if !options.fullySorted { - sort.Sort(rowColumnSet{r: rowIDs, c: columnIDs}) - } // replace columnIDs with calculated positions to avoid allocation. prevRow, prevCol := ^uint64(0), ^uint64(0) next := 0 @@ -1757,35 +1780,6 @@ func (f *fragment) updateCaching(tx Tx, rowSet map[uint64]struct{}) error { return nil } -// sliceDifference removes everything from original that's found in remove, -// updating the slice in place, and returns the compacted slice. The input -// sets should be sorted. -func sliceDifference(original, remove []uint64) []uint64 { - if len(remove) == 0 { - return original - } - rn := 0 - rv := remove[rn] - on := 0 - ov := uint64(0) - n := 0 - - for on, ov = range original { - for rv < ov { - rn++ - if rn >= len(remove) { - return append(original[:n], original[on:]...) - } - rv = remove[rn] - } - if rv != ov { - original[n] = ov - n++ - } - } - return original[:n] -} - // bulkImportMutex performs a bulk import on a fragment while ensuring // mutex restrictions. Because the mutex requirements must be checked // against storage, this method must acquire a write lock on the fragment @@ -1794,16 +1788,10 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *I f.mu.Lock() defer f.mu.Unlock() - // if ingest promises that this is "fully sorted", then we have been - // promised that (1) there's no duplicate entries that need to be - // pruned, (2) the input is sorted by row IDs and then column IDs, - // meaning that we will generate positions in strictly sequential order. - if !options.fullySorted { - p := parallelSlices{cols: columnIDs, rows: rowIDs} - p.fullPrune() - columnIDs = p.cols - rowIDs = p.rows - } + p := parallelSlices{cols: columnIDs, rows: rowIDs} + p.fullPrune() + columnIDs = p.cols + rowIDs = p.rows // create a mask of columns we care about columns := roaring.NewSliceBitmap(columnIDs...) @@ -1822,10 +1810,6 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *I // positions are sorted by columns, but not by absolute // position. we might want them sorted, though. if pos < prev { - if options.fullySorted { - fmt.Printf("HELP! was promised fully sorted input, but previous position was %d, now generated %d\n", - prev, pos) - } unsorted = true } prev = pos diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 7d56ee11f..a8b365dc9 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -4995,69 +4995,6 @@ func TestParallelSlicesFullPrune(t *testing.T) { }) } -func compareSlices(tb testing.TB, name string, s1, s2 []uint64) { - if len(s1) != len(s2) { - tb.Fatalf("slice length mismatch %q: expected %d items %d, got %d items %d", - name, len(s1), s1, len(s2), s2) - } - for i, v := range s1 { - if s2[i] != v { - tb.Fatalf("row mismatch %q: expected item %d to be %d, got %d", - name, i, s1[i], s2[i]) - } - } -} - -type sliceDifferenceTestCase struct { - original, remove, expected []uint64 -} - -func TestSliceDifference(t *testing.T) { - testCases := map[string]sliceDifferenceTestCase{ - "noOverlap": { - original: []uint64{1, 2, 3}, - remove: []uint64{0, 5}, - expected: []uint64{1, 2, 3}, - }, - "before": { - original: []uint64{3, 5, 7}, - remove: []uint64{0, 6}, - expected: []uint64{3, 5, 7}, - }, - "after": { - original: []uint64{3, 5, 7}, - remove: []uint64{8, 10}, - expected: []uint64{3, 5, 7}, - }, - "all": { - original: []uint64{3, 5, 7}, - remove: []uint64{3, 5, 7}, - expected: []uint64{}, - }, - "first": { - original: []uint64{3, 5, 7}, - remove: []uint64{3}, - expected: []uint64{5, 7}, - }, - "last": { - original: []uint64{3, 5, 7}, - remove: []uint64{7}, - expected: []uint64{3, 5}, - }, - "middle": { - original: []uint64{3, 5, 7}, - remove: []uint64{5}, - expected: []uint64{3, 7}, - }, - } - var scratch []uint64 - for name, tc := range testCases { - scratch = append(scratch[:0], tc.original...) - result := sliceDifference(scratch, tc.remove) - compareSlices(t, name, tc.expected, result) - } -} - func TestImportRoaringSingleValued(t *testing.T) { f, _, tx := mustOpenFragment(t) defer f.Clean(t) diff --git a/http_handler_test.go b/http_handler_test.go index 35cacacd6..82f78cdc9 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -475,6 +475,7 @@ func TestGetViewAndDelete(t *testing.T) { // The above sample data should create these views: expectedViewNames := []string{ + "existence", "standard", "standard_2001", "standard_200102", diff --git a/idk/api/source_test.go b/idk/api/source_test.go index 03a32b5d5..83ac70d09 100644 --- a/idk/api/source_test.go +++ b/idk/api/source_test.go @@ -143,7 +143,7 @@ func TestIngest(t *testing.T) { "columns": [ {"column":0,"rows":[[5,7],4,["a"],"h",-13,1.01]}, {"column":1,"rows":[[6],8,["b"],null,null,1.02]}, - {"column":2,"rows":[[],null,[],null,null,null]} + {"column":2,"rows":[null,null,null,null,null,null]} ] } ] @@ -214,7 +214,7 @@ func TestIngest(t *testing.T) { "columns": [ {"column":"a","rows":[[5,7],4,["a"],"h",-13,1.01]}, {"column":"b","rows":[[6],8,["b"],null,null,1.02]}, - {"column":"c","rows":[[],null,[],null,null,null]} + {"column":"c","rows":[null,null,null,null,null,null]} ] } ] diff --git a/index.go b/index.go index ae6d18e84..5887d2a41 100644 --- a/index.go +++ b/index.go @@ -561,7 +561,47 @@ func (i *Index) recalculateCaches() { } } -// CreateField creates a field. +// createNullableField is just like CreateField, except that it allows +// the field to not have TrackExistence enabled. This should be used +// only for existing fields which were already actually created, where +// what we're really doing now is reifying them, so for instance, this +// shows up in api_directive:createField to apply directives, which +// are assumed to correctly reflect the intended behavior. +func (i *Index) createNullableField(name string, requestUserID string, opts ...FieldOption) (*Field, error) { + err := ValidateName(name) + if err != nil { + return nil, errors.Wrap(err, "validating name") + } + + // Grab lock, check for field existing, release lock. We don't want + // to stay holding the lock, but we might care about the ErrFieldExists + // part of this. + err = func() error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ensure field doesn't already exist. + if i.fields[name] != nil { + return newConflictError(ErrFieldExists) + } + return nil + }() + if err != nil { + return nil, err + } + + // Apply and validate functional options. + fo, err := newFieldOptions(opts...) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + return i.createFieldWithOptions(name, requestUserID, fo) +} + +// CreateField creates a field. This interface enforces the setting +// of the TrackExistence flag; if you don't want that, use +// createNullableField, but actually don't. That should be used only +// for applying previously-created fields. func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { @@ -590,7 +630,13 @@ func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOpti if err != nil { return nil, errors.Wrap(err, "applying option") } + fo.TrackExistence = true + return i.createFieldWithOptions(name, requestUserID, fo) +} +// createFieldWithOptions creates a field given a finalized FieldOptions +// structure, instead of functional options. +func (i *Index) createFieldWithOptions(name string, requestUserID string, fo *FieldOptions) (*Field, error) { ts := timestamp() cfm := &CreateFieldMessage{ Index: i.name, @@ -625,6 +671,8 @@ func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOpti } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +// +// Does NOT apply the "default" TrackExistence. func (i *Index) CreateFieldIfNotExists(name string, requestUserID string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { @@ -671,6 +719,8 @@ func (i *Index) CreateFieldIfNotExists(name string, requestUserID string, opts . // function options, taking a *FieldOptions struct. TODO: This should // definintely be refactored so we don't have these virtually equivalent // methods, but I'm puttin this here for now just to see if it works. +// +// Does NOT apply the "default" TrackExistence. func (i *Index) CreateFieldIfNotExistsWithOptions(name string, requestUserID string, opt *FieldOptions) (*Field, error) { err := ValidateName(name) if err != nil { diff --git a/null_test.go b/null_test.go new file mode 100644 index 000000000..8df7d56e3 --- /dev/null +++ b/null_test.go @@ -0,0 +1,620 @@ +// Copyright 2023 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa_test + +import ( + "context" + "fmt" + "strings" + "testing" + + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/batch" + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/test" + "github.com/stretchr/testify/require" +) + +// the new "handle nulls for non-BSI fields" logic has a lot of +// implications, and needs to test things across multiple ways +// of importing data, so we want to have some consistent +// behavior. +// +// We're not testing keyed indexes, because the way index keys +// work doesn't have any relation to the code that's used to +// track existence, but we do want to test both keyed and +// unkeyed fields, and we want to test direct Set/Clear +// operations, SetRow, ClearRow (maybe? for mutexes?), +// Import operations, and the batch code. Note that direct +// use of ImportRoaring, outside of the batch code, makes +// no guarantees; it's up to a user providing roaring bitmaps +// to handle existence views. +// +// This is necessary because it's simply not *possible* to +// detect the distinction between "no bits provided" and +// "an empty set" from the bits written to a view other than +// an existence view. We could, in principle, spend a lot +// of time computing a best-guess that all non-empty sets +// are non-null, but this would be less accurate and much +// more expensive than doing that work on the batch side. + +// setupNullHandlingSchema yields a cluster, and API, which point at an +// index with the given name, containing fields {mu, mk, su, sk, tu, tk} +// which are mutex/set/timequantum fields which are unkeyed/keyed respectively. +func setupNullHandlingSchema(t *testing.T, indexSuffix string) (*test.Cluster, string, *pilosa.API) { + c := test.MustRunCluster(t, 3) + api := c.GetNode(0).API + index := c.Idx(indexSuffix) + // we have six cases we care about; {mutex,set,timequantum} * {keyed, unkeyed} + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "mu", pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "mk", pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "su", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0)) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "sk", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "tu", pilosa.OptFieldTypeTime("YMD", "0")) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "tk", pilosa.OptFieldTypeTime("YMD", "0"), pilosa.OptFieldKeys()) + return c, index, api +} + +var nullHandlingFieldMasks = map[string]int{ + "mu": 1, + "mk": 2, + "su": 4, + "sk": 8, + "tu": 16, + "tk": 32, +} +var nullHandlingExpectedResults = generateNullHandlingExpectedResults() + +type nullSet map[bool][]uint64 + +func (n nullSet) null(v uint64) { + n[true] = append(n[true], v) +} + +func (n nullSet) notNull(v uint64) { + n[false] = append(n[false], v) +} + +func (n nullSet) clone() nullSet { + f := n[false] + t := n[true] + nf := make([]uint64, len(f)) + nt := make([]uint64, len(t)) + copy(nf, f) + copy(nt, t) + return nullSet{false: nf, true: nt} +} + +// trimSlice removes elements of s for which func returns +// true, returning the modified slice. it is destructive. +func trimSlice(s []uint64, fn func(uint64) bool) []uint64 { + n := 0 + for i, v := range s { + if fn(v) { + continue + } else { + s[n] = s[i] + n++ + } + } + return s[:n] +} + +// trim removes values matching f +func (ns nullSet) trim(fn func(uint64) bool) { + ns[false] = trimSlice(ns[false], fn) + ns[true] = trimSlice(ns[true], fn) +} + +// This outlines the expected results of a series of steps. +// We have six fields, keyed/unkeyed sets, mutexes, and time +// quantums. We only ever set value 0/"a" in them. +// +// First, we set those bits for the records 0..63, by treating +// each field as having a bitmask 1/2/4/8/16/32, and setting the +// bits for each field that's a 1 in the record's ID. +// +// Then, we *clear* every bit for every 5th record. I used to do +// every 3rd, but this worked out poorly, because 63%3 == 0. This +// means that all those entries should still exist, but should be +// considered nulls. +// +// Then, we delete everything that has either the 16 or the 32 +// bit (or both) set. This means that all the records which are +// *not* divisible by 5 should no longer be considered null, because +// the records don't exist. The ones which are divisible by 5 are +// still null. +// +// Then, we set a bit in "tk" for record 63. At this point, the +// others should all be null. Remember that, prior to the delete, +// they were all non-null; 63 was the record that had every bit set. +// So we're verifying that, after a delete, recreating the record +// does not show things as not-null because they had been set *prior* +// to the delete. +func generateNullHandlingExpectedResults() map[int]map[string]nullSet { + out := map[int]map[string]nullSet{} + for phase := 0; phase < 4; phase++ { + out[phase] = map[string]nullSet{} + for k := range nullHandlingFieldMasks { + out[phase][k] = nullSet{} + } + } + // We can't combine these two inner loops, because a field is + // only null for records which exist, so record 0, having no + // values set at all, isn't even a null. + phase := 0 + for i := 0; i < (1 << 6); i++ { + madeAny := false + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + out[phase][k].notNull(uint64(i)) + madeAny = true + } + } + if madeAny { + for k, v := range nullHandlingFieldMasks { + if i&v == 0 { + out[phase][k].null(uint64(i)) + } + } + } + } + // phase 1: we clear every bit in records divisible by 5. + // this should not change nullness of sets or time quantums, + // but should change their results. it should make mutexes null. + // we also clear several bits in row 1/"b" -- bits which were + // never set. this should have no impact on anything, so we don't + // reflect it here. + phase = 1 + for i := 0; i < (1 << 6); i++ { + madeAny := false + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[0] == 'm' && (i%5) == 0 { + out[phase][k].null(uint64(i)) + } else { + out[phase][k].notNull(uint64(i)) + } + // even if the only value was the mutex field, the record + // was ever created, so the record still exists even if + // every field is null. + madeAny = true + } + } + if madeAny { + for k, v := range nullHandlingFieldMasks { + if i&v == 0 { + out[phase][k].null(uint64(i)) + } + } + } + } + + // phase 2: delete the &16 and &32 rows. we expect deleted records + // to be neither null nor not null. We delete both the &16 and &32 + // rows because the delete flow is different for keys and no-keys. + phase = 2 + for k := range nullHandlingFieldMasks { + ns := out[1][k].clone() + ns.trim(func(v uint64) bool { return v >= 16 && (v%5) != 0 }) + out[phase][k] = ns + } + + // phase 3: create a record which had previously existed. + // we expect previously-existing fields to now show as null, + // unless we created them again. + phase = 3 + for k, v := range nullHandlingFieldMasks { + ns := out[2][k].clone() + if v == 32 { + ns.notNull(63) + } else { + ns.null(63) + } + out[phase][k] = ns + } + return out +} + +func nullTestQuery(t *testing.T, api *pilosa.API, req *pilosa.QueryRequest) pilosa.QueryResponse { + t.Helper() + resp, err := api.Query(context.Background(), req) + if err != nil { + t.Fatalf("running request: %v", err) + } + if resp.Err != nil { + t.Fatalf("request returned unexpected error: %v", resp.Err) + } + return resp +} + +func nullTestRows(t *testing.T, api *pilosa.API, req *pilosa.QueryRequest) [][]uint64 { + t.Helper() + resp := nullTestQuery(t, api, req) + out := make([][]uint64, 0, len(resp.Results)) + for _, result := range resp.Results { + row, ok := result.(*pilosa.Row) + if !ok { + t.Fatalf("expected row result from query, got %T", result) + } + out = append(out, row.Columns()) + } + return out +} + +func nullTestImport(t *testing.T, api *pilosa.API, req *pilosa.ImportRequest) { + t.Helper() + qcx := api.Txf().NewQcx() + defer qcx.Abort() + err := api.Import(context.Background(), qcx, req) + if err != nil { + t.Fatalf("importing: %v", err) + } + err = qcx.Finish() + if err != nil { + t.Fatalf("committing: %v", err) + } +} + +func nullTestExpectResults(t *testing.T, api *pilosa.API, index string, phase int) { + t.Helper() + for k, expected := range nullHandlingExpectedResults[phase] { + t.Run(fmt.Sprintf("%s-phase-%d", k, phase), func(t *testing.T) { + req := &pilosa.QueryRequest{ + Index: index, + Query: fmt.Sprintf(`Row(%s == null) + Row(%s != null)`, k, k), + } + rows := nullTestRows(t, api, req) + t.Run("true", func(t *testing.T) { + require.Equal(t, expected[true], rows[0]) + }) + t.Run("false", func(t *testing.T) { + require.Equal(t, expected[false], rows[1]) + }) + }) + } +} + +// TestNullHandlingSet only tests the behavior of null values +// in non-BSI fields. It tests this using set/clear. +func TestNullHandlingSet(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + + phase := 0 + var reqs []string + for i := 0; i < (1 << 6); i++ { + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Set(%d, %s="a")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Set(%d, %s=0)`, i, k)) + } + } + } + } + req := &pilosa.QueryRequest{ + Index: index, + Query: strings.Join(reqs, "\n"), + } + resp := nullTestQuery(t, api, req) + for i, v := range resp.Results { + if v != true { + t.Fatalf("result %d (request %s): %#v", i, reqs[i], v) + } + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + reqs = reqs[:0] + for i := 0; i < (1 << 6); i += 5 { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s="a")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s=0)`, i, k)) + } + } + } + // clear a lot of bits that weren't set in the first place. this should have + // no effect on null/not-null state. + for i := 16; i < 48; i++ { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s="b")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s=1)`, i, k)) + } + } + } + req = &pilosa.QueryRequest{ + Index: index, + Query: strings.Join(reqs, "\n"), + } + resp = nullTestQuery(t, api, req) + for i, v := range resp.Results { + // it's okay to get either true or false, because some of those bits + // wouldn't have existed before, so the clear would fail. that's fine. + if v != true && v != false { + t.Fatalf("result %d (request %s): %#v", i, reqs[i], v) + } + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + req = &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp = nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + + phase = 3 + req = &pilosa.QueryRequest{ + Index: index, + Query: `Set(63, tk="a")`, + } + resp = nullTestQuery(t, api, req) + if len(resp.Results) != 1 || resp.Results[0] != true { + t.Fatalf("expected single true result, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) +} + +// TestNullHandlingImport only tests the behavior of null values +// in non-BSI fields. It tests this using the old Import API to +// import values. +func TestNullHandlingImport(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + + phase := 0 + reqs := map[string]*pilosa.ImportRequest{} + for k := range nullHandlingFieldMasks { + reqs[k] = &pilosa.ImportRequest{ + Index: index, + Field: k, + Shard: ^uint64(0), + } + } + for i := 0; i < (1 << 6); i++ { + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "a") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + } + } + } + } + for _, req := range reqs { + nullTestImport(t, api, req) + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + for k := range nullHandlingFieldMasks { + reqs[k].ColumnIDs = reqs[k].ColumnIDs[:0] + if k[1] == 'k' { + reqs[k].RowKeys = reqs[k].RowKeys[:0] + // the import stashed its computed RowIDs in the req, + // remove them. + reqs[k].RowIDs = nil + } else { + reqs[k].RowIDs = reqs[k].RowIDs[:0] + } + reqs[k].Clear = true + reqs[k].Shard = ^uint64(0) + } + + for i := 0; i < (1 << 6); i += 5 { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "a") + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "b") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + } + } + } + // clear a lot of bits that never previously got set, expecting no impact. + for i := 16; i < 48; i++ { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "b") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 1) + } + } + } + for _, req := range reqs { + t.Logf("req: %#v", req) + nullTestImport(t, api, req) + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + // no Delete in Import API, we use the old API for it. + req := &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp := nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + phase = 3 + reqImport := &pilosa.ImportRequest{ + Index: index, + Field: "tk", + ColumnIDs: []uint64{63}, + RowKeys: []string{"a"}, + Shard: ^uint64(0), + } + nullTestImport(t, api, reqImport) + nullTestExpectResults(t, api, index, phase) +} + +func TestNullHandlingBatch(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + ctx := context.Background() + fapi := pilosa.NewOnPremSchema(api) + imp := pilosa.NewOnPremImporter(api) + tbl, err := fapi.TableByName(ctx, dax.TableName(index)) + if err != nil { + t.Fatalf("getting table defs: %v", err) + } + idxInfoBase := pilosa.TableToIndexInfo(tbl) + fields := make([]*pilosa.FieldInfo, 0, len(nullHandlingFieldMasks)) + for k := range nullHandlingFieldMasks { + fields = append(fields, idxInfoBase.Field(k)) + } + + phase := 0 + b, err := batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + var row batch.Row + row.Values = make([]interface{}, len(fields)) + // we don't add the all-null row for ID 0 + for i := 1; i < (1 << 6); i++ { + row.ID = uint64(i) + row.Values = row.Values[:0] + for _, fld := range fields { + if i&nullHandlingFieldMasks[fld.Name] != 0 { + if fld.Name[1] == 'k' { + row.Values = append(row.Values, "a") + } else { + row.Values = append(row.Values, uint64(0)) + } + } else { + row.Values = append(row.Values, nil) + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + // delete everything which is a multiple of 5, but + // don't try to delete 0 because this creates an all-null + // record. + for i := 5; i < (1 << 6); i += 5 { + row.ID = uint64(i) + row.Values = row.Values[:0] + row.Clears = map[int]interface{}{} + for i, fld := range fields { + row.Values = append(row.Values, nil) + if fld.Name[0] == 'm' { + // can't specify a particular bit to clear in a mutex + row.Clears[i] = nil + } else { + if fld.Name[1] == 'k' { + row.Clears[i] = "a" + } else { + row.Clears[i] = uint64(0) + } + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + // delete everything which is a multiple of 5 + for i := 16; i < 48; i++ { + row.ID = uint64(i) + row.Values = row.Values[:0] + row.Clears = map[int]interface{}{} + for i, fld := range fields { + row.Values = append(row.Values, nil) + // no way to specify "clear a specific bit", so we don't clear the mutexes. + if fld.Name[0] != 'm' { + if fld.Name[1] == 'k' { + row.Clears[i] = "b" + } else { + row.Clears[i] = uint64(1) + } + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + // no Delete in batch API, we use the old API for it. + req := &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp := nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + + phase = 3 + // truncate fields to only have the one field in it + for _, f := range fields { + if f.Name == "tk" { + fields[0] = f + fields = fields[:1] + break + } + } + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + row.ID = uint64(63) + row.Values = []interface{}{[]string{"a"}} + err = b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) +} diff --git a/pb/private.pb.go b/pb/private.pb.go index dbf939301..6ea3b50de 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -102,6 +102,7 @@ type FieldOptions struct { Max *Decimal `protobuf:"bytes,18,opt,name=Max,proto3" json:"Max,omitempty"` TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"` TTL string `protobuf:"bytes,20,opt,name=TTL,proto3" json:"TTL,omitempty"` + TrackExistence bool `protobuf:"varint,21,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -252,6 +253,13 @@ func (m *FieldOptions) GetTTL() string { return "" } +func (m *FieldOptions) GetTrackExistence() bool { + if m != nil { + return m.TrackExistence + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2951,116 +2959,117 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1742 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x23, 0x49, - 0x15, 0xa6, 0x2f, 0xf1, 0xe5, 0x38, 0xce, 0x38, 0xb5, 0x21, 0xf4, 0x64, 0x87, 0xc8, 0x53, 0xa0, - 0x19, 0x33, 0x12, 0x41, 0x64, 0x1f, 0x16, 0xb1, 0x2f, 0x3b, 0xb1, 0x33, 0x8b, 0xd9, 0x9d, 0xcb, - 0x56, 0x2e, 0x8f, 0xa0, 0x4a, 0xbb, 0x48, 0x5a, 0x69, 0x77, 0x9b, 0xee, 0x76, 0x62, 0xef, 0x03, - 0x12, 0x48, 0x08, 0x5e, 0x78, 0x47, 0x3c, 0xf0, 0x2f, 0xf8, 0x03, 0x3c, 0xf1, 0x82, 0xc4, 0x4f, - 0x40, 0xc3, 0x1f, 0x41, 0x75, 0xaa, 0xaa, 0xbb, 0xec, 0x38, 0xc9, 0x10, 0xf1, 0xd6, 0xe7, 0x3b, - 0xd5, 0xa7, 0xbe, 0x73, 0xa9, 0x53, 0xa7, 0x1b, 0xda, 0x93, 0x2c, 0xba, 0xe2, 0x85, 0xd8, 0x9b, - 0x64, 0x69, 0x91, 0x12, 0x77, 0x72, 0xb6, 0xb3, 0x3e, 0x99, 0x9e, 0xc5, 0x51, 0xa8, 0x10, 0x1a, - 0x41, 0x73, 0x98, 0x8c, 0xc4, 0xec, 0xb5, 0x28, 0x38, 0x21, 0xe0, 0x7f, 0x29, 0xe6, 0x79, 0xe0, - 0x75, 0x9d, 0x5e, 0x83, 0xe1, 0x33, 0x79, 0x06, 0x1b, 0xc7, 0x19, 0x0f, 0x2f, 0x0f, 0x67, 0x51, - 0x5e, 0x88, 0x24, 0x14, 0x81, 0x8f, 0xda, 0x25, 0x94, 0x74, 0xa1, 0x35, 0x10, 0x79, 0x98, 0x45, - 0x93, 0x22, 0x4a, 0x93, 0x60, 0xad, 0xeb, 0xf4, 0x9a, 0xcc, 0x86, 0xe8, 0xdf, 0x3d, 0x58, 0x7f, - 0x15, 0x89, 0x78, 0xf4, 0x16, 0xe5, 0x5c, 0x6e, 0x77, 0x3c, 0x9f, 0x88, 0xa0, 0x81, 0x6b, 0xf1, - 0x99, 0x3c, 0x81, 0x66, 0x9f, 0x87, 0x17, 0x02, 0x15, 0x1e, 0x2a, 0x2a, 0xa0, 0xd4, 0x1e, 0x45, - 0xdf, 0x28, 0x1e, 0x6d, 0x56, 0x01, 0x92, 0xc2, 0x71, 0x34, 0x16, 0x5f, 0x4f, 0x79, 0x52, 0x4c, - 0xc7, 0x86, 0x82, 0x05, 0x91, 0x6d, 0xa8, 0xbd, 0x8d, 0x47, 0xaf, 0xa3, 0x24, 0x68, 0x76, 0x9d, - 0x9e, 0xc7, 0xb4, 0x64, 0x70, 0x3e, 0x0b, 0xa0, 0xc2, 0xf9, 0xac, 0x0c, 0x48, 0x6b, 0x31, 0x20, - 0x6f, 0xd2, 0xa3, 0x82, 0x27, 0x23, 0x9e, 0x8d, 0x4e, 0x23, 0x71, 0x1d, 0xac, 0xab, 0x80, 0x2c, - 0xa2, 0xf2, 0xdd, 0x03, 0x9e, 0x8b, 0xa0, 0x8d, 0x16, 0xf1, 0x99, 0xec, 0x40, 0xe3, 0x20, 0x2a, - 0x06, 0x62, 0x52, 0x5c, 0x04, 0x1b, 0x5d, 0xa7, 0xe7, 0xb3, 0x52, 0x26, 0x5b, 0xb0, 0x76, 0x14, - 0xf2, 0x58, 0x04, 0x8f, 0xf0, 0x05, 0x25, 0x10, 0x0a, 0xeb, 0xaf, 0xd2, 0x4c, 0x44, 0xe7, 0x09, - 0xa6, 0x29, 0xe8, 0xa0, 0x53, 0x0b, 0x18, 0xf9, 0x2e, 0x78, 0xd2, 0xa5, 0xcd, 0xae, 0xd3, 0x6b, - 0xed, 0xb7, 0xf6, 0x26, 0x67, 0x7b, 0x03, 0x11, 0x46, 0x63, 0x1e, 0x33, 0x89, 0xa3, 0x9a, 0xcf, - 0x02, 0xb2, 0x4a, 0xcd, 0x67, 0x92, 0x93, 0x0c, 0xd1, 0x49, 0x12, 0x15, 0xc1, 0x47, 0x68, 0xbd, - 0x94, 0x49, 0x07, 0xbc, 0xe3, 0xe3, 0xaf, 0x82, 0x2d, 0x84, 0xe5, 0x23, 0xa5, 0xb0, 0x31, 0x1c, - 0x4f, 0xd2, 0xac, 0x60, 0x22, 0x9f, 0xa4, 0x49, 0x2e, 0xe4, 0x9a, 0xc3, 0x2c, 0x0b, 0x1c, 0xb5, - 0xe6, 0x30, 0xcb, 0xe8, 0x6f, 0xa0, 0x73, 0x10, 0xa7, 0xe1, 0xe5, 0x80, 0x17, 0x9c, 0x89, 0x5f, - 0x4f, 0x45, 0x5e, 0x48, 0xef, 0x94, 0x03, 0x6a, 0x9d, 0x12, 0x24, 0x8a, 0x15, 0x11, 0xb8, 0x0a, - 0x45, 0x41, 0x46, 0x0e, 0xe3, 0xaa, 0x12, 0x88, 0xcf, 0x18, 0x9d, 0x0b, 0x9e, 0x8d, 0x30, 0xeb, - 0x3e, 0x53, 0x82, 0x44, 0x71, 0x27, 0xac, 0x14, 0x9f, 0x29, 0x81, 0x0e, 0x61, 0xd3, 0xda, 0x5f, - 0xd3, 0xdc, 0x86, 0x1a, 0x4b, 0xaf, 0x87, 0x83, 0x3c, 0x70, 0xba, 0x5e, 0xcf, 0x67, 0x5a, 0xc2, - 0x92, 0x4a, 0xe3, 0xe9, 0x38, 0x91, 0x2a, 0x17, 0x55, 0x15, 0x40, 0x1f, 0xc3, 0x1a, 0xd6, 0x97, - 0xf4, 0xb2, 0x7a, 0x57, 0x3e, 0xd2, 0xdf, 0x3a, 0xd0, 0x7c, 0xcd, 0x67, 0x48, 0x24, 0x27, 0x9f, - 0x42, 0xc3, 0x64, 0x1f, 0x17, 0xb5, 0xf6, 0x3f, 0x96, 0x91, 0x2e, 0x17, 0xec, 0x19, 0xed, 0x61, - 0x52, 0x64, 0x73, 0x56, 0x2e, 0xde, 0xf9, 0x0c, 0xda, 0x0b, 0x2a, 0xb9, 0xd3, 0xa5, 0x98, 0x9b, - 0x78, 0x5e, 0x8a, 0xb9, 0xf4, 0xf2, 0x8a, 0xc7, 0x53, 0x81, 0x51, 0xf2, 0x99, 0x12, 0x7e, 0xea, - 0xfe, 0xc4, 0xa1, 0xa7, 0x40, 0xfa, 0x99, 0xe0, 0x85, 0xc0, 0x4d, 0x5e, 0x8b, 0x3c, 0xe7, 0xe7, - 0xe2, 0xbe, 0x58, 0x7b, 0x76, 0xac, 0xcb, 0xb8, 0xba, 0x56, 0x5c, 0xe9, 0x0b, 0x20, 0x03, 0x11, - 0x8b, 0x42, 0xe8, 0xde, 0x70, 0x87, 0x5d, 0x19, 0x07, 0x4d, 0xe2, 0xfe, 0xc5, 0xe4, 0x29, 0xf8, - 0xb2, 0xd3, 0xe0, 0x6e, 0xad, 0xfd, 0xb6, 0x0c, 0x51, 0xd9, 0x7e, 0x18, 0xaa, 0x30, 0x21, 0x68, - 0x6e, 0xf4, 0xb2, 0x40, 0xae, 0x1e, 0xab, 0x00, 0x69, 0xf6, 0xed, 0x75, 0x22, 0x32, 0x5d, 0x1c, - 0x4a, 0xa0, 0x7f, 0x29, 0x39, 0xa0, 0x57, 0x1f, 0x18, 0x88, 0x85, 0xa2, 0xfb, 0xbe, 0x66, 0xe6, - 0x21, 0xb3, 0x8e, 0x64, 0x66, 0x37, 0xab, 0x55, 0xe4, 0xfc, 0x0f, 0x23, 0xf7, 0x7b, 0x07, 0xc8, - 0xc9, 0x64, 0xb4, 0x4c, 0xee, 0xd5, 0x2a, 0xca, 0xc8, 0xb4, 0xb5, 0xbf, 0x2d, 0xb7, 0xbf, 0xa9, - 0x65, 0xab, 0x9c, 0x7c, 0x0e, 0x35, 0x65, 0x5d, 0x07, 0xf5, 0x51, 0x49, 0x5d, 0xc1, 0x4c, 0xab, - 0xe9, 0x67, 0xd0, 0xb2, 0x60, 0xec, 0x79, 0xaa, 0x57, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0x71, 0x5a, - 0x56, 0x5b, 0x93, 0x29, 0x81, 0x7e, 0x6e, 0x2a, 0xe2, 0xa1, 0x01, 0xa6, 0x21, 0x7c, 0xac, 0x2c, - 0xbc, 0xbc, 0xe2, 0x51, 0xcc, 0xcf, 0xe2, 0xff, 0xa9, 0x68, 0x17, 0x72, 0x15, 0x40, 0x1d, 0xdf, - 0x1d, 0x0e, 0xf4, 0xc1, 0x37, 0x22, 0x9d, 0x42, 0xd5, 0x43, 0xde, 0xf0, 0xb1, 0xd0, 0xd6, 0xf0, - 0xb9, 0x4c, 0xb1, 0x7b, 0x67, 0x8a, 0xa5, 0xff, 0x91, 0xb8, 0x96, 0xb7, 0xa0, 0x87, 0xfe, 0x4b, - 0xe1, 0xee, 0xc4, 0xd3, 0x1f, 0x42, 0xed, 0x28, 0xbc, 0x10, 0x63, 0x4e, 0xbe, 0x07, 0x75, 0x64, - 0x2e, 0x72, 0xdd, 0x06, 0x9a, 0x65, 0x8d, 0x33, 0xa3, 0x91, 0x15, 0xa1, 0xfd, 0x5b, 0x45, 0x73, - 0x61, 0x2b, 0x77, 0xb9, 0xc6, 0x9e, 0x43, 0x5d, 0xf3, 0xc5, 0x2a, 0xbb, 0x71, 0x88, 0x8c, 0x96, - 0x3c, 0x85, 0x1a, 0x7a, 0x97, 0x07, 0x7e, 0x45, 0x04, 0x11, 0xa6, 0x15, 0xf4, 0x10, 0xbc, 0x13, - 0x36, 0x94, 0x95, 0x80, 0xec, 0x0d, 0x0d, 0x2d, 0x49, 0x72, 0x3f, 0x4b, 0xf3, 0x42, 0xc7, 0x1e, - 0x9f, 0x25, 0xf6, 0x2e, 0xcd, 0xd4, 0xc1, 0x6c, 0x33, 0x7c, 0xa6, 0x7f, 0x74, 0xc0, 0x7f, 0x93, - 0x8e, 0x04, 0xd9, 0x00, 0x77, 0x38, 0xd0, 0x46, 0xdc, 0xe1, 0x80, 0x3c, 0x46, 0xfb, 0x3a, 0xde, - 0x75, 0xb9, 0xff, 0x09, 0x1b, 0x32, 0xdc, 0xf3, 0x09, 0x34, 0x87, 0xf9, 0xbb, 0x2c, 0x1a, 0xf3, - 0x6c, 0xae, 0xe7, 0x8d, 0x0a, 0xc0, 0xae, 0x54, 0xc8, 0x92, 0xf6, 0x55, 0xda, 0x51, 0x20, 0x4f, - 0xa1, 0xfe, 0x05, 0x7b, 0xd7, 0x97, 0x26, 0xd7, 0x16, 0x4d, 0x1a, 0x9c, 0x7e, 0x0e, 0x1d, 0xc9, - 0x04, 0xd7, 0x9b, 0xca, 0xda, 0x86, 0x9a, 0xc4, 0x4a, 0x66, 0x5a, 0xaa, 0x36, 0x71, 0xad, 0x4d, - 0xe8, 0x2b, 0x65, 0xe1, 0xf0, 0x4a, 0x24, 0x85, 0x55, 0x9b, 0x28, 0xa3, 0x81, 0x36, 0x53, 0x02, - 0x79, 0xa2, 0xbc, 0xd6, 0xee, 0x35, 0x24, 0x17, 0x29, 0x33, 0x44, 0xe9, 0x1c, 0xc0, 0x30, 0x99, - 0xe6, 0xe5, 0x5a, 0x67, 0xd5, 0x5a, 0x42, 0x4d, 0xf9, 0xe8, 0xee, 0x03, 0x52, 0xaf, 0x10, 0x66, - 0x0a, 0xeb, 0x07, 0x55, 0x61, 0xa9, 0x7c, 0x3e, 0x2a, 0xf3, 0xae, 0xf6, 0xa8, 0xca, 0xeb, 0x02, - 0x5a, 0x16, 0xbe, 0xb2, 0xc6, 0x9e, 0x97, 0xc5, 0xe1, 0x56, 0xc6, 0x10, 0xd1, 0xc6, 0xb4, 0xfa, - 0xee, 0x6e, 0x4c, 0x23, 0xdd, 0x52, 0xee, 0xd8, 0xa9, 0x07, 0x8f, 0x16, 0x0f, 0xbc, 0xb9, 0x65, - 0x97, 0xe1, 0x7b, 0xb6, 0xfa, 0x83, 0x03, 0xed, 0x7e, 0x3c, 0xcd, 0x0b, 0x91, 0x95, 0x31, 0x6d, - 0x6a, 0xa0, 0x4c, 0x6d, 0x05, 0xac, 0xce, 0x2e, 0xd9, 0x85, 0x35, 0x19, 0x71, 0x75, 0xb8, 0xed, - 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5b, 0x26, 0xe8, 0x29, 0x34, 0x0e, 0x8e, 0x86, 0x5f, 0x64, - 0xe9, 0x74, 0xb2, 0xd2, 0x63, 0x33, 0xd6, 0xba, 0xd6, 0x58, 0xdb, 0x51, 0x23, 0x9a, 0xf2, 0x0a, - 0xa7, 0xb2, 0x8e, 0x9a, 0xca, 0x7c, 0x8d, 0xf0, 0x19, 0x3d, 0x82, 0x4d, 0xe5, 0xae, 0xec, 0x38, - 0x0f, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xff, 0x69, 0xf4, - 0x9f, 0x2e, 0x6c, 0x32, 0x91, 0x47, 0xdf, 0x88, 0x61, 0x92, 0x17, 0xd9, 0x34, 0x34, 0x17, 0xc7, - 0xcf, 0xd3, 0x33, 0x9d, 0x0b, 0x8f, 0x29, 0xe1, 0xee, 0x53, 0x42, 0x28, 0xd4, 0xed, 0x26, 0x60, - 0x2f, 0x30, 0x0a, 0xf2, 0x02, 0xea, 0x47, 0xe9, 0x34, 0x0b, 0xcb, 0xca, 0xc7, 0xce, 0xad, 0xf6, - 0x57, 0x0a, 0x66, 0x16, 0x90, 0x2f, 0x81, 0x1c, 0x67, 0x3c, 0xc9, 0x63, 0x2e, 0x29, 0x99, 0xd7, - 0x1a, 0xd5, 0x40, 0x66, 0x69, 0x17, 0x2c, 0xac, 0x78, 0x8d, 0xec, 0xd9, 0x47, 0x38, 0xa8, 0x23, - 0xbf, 0x0d, 0xc3, 0x4f, 0x9f, 0x13, 0xfb, 0x90, 0x7f, 0xba, 0x54, 0xa1, 0x41, 0x0d, 0x5f, 0xd9, - 0xc4, 0xcb, 0xdc, 0x56, 0xb0, 0xc5, 0x75, 0xf4, 0x77, 0x0e, 0xac, 0xdb, 0x6c, 0xee, 0x69, 0x17, - 0x65, 0xfa, 0xdc, 0xfb, 0xe7, 0x3b, 0x93, 0x3e, 0x7f, 0xd5, 0x2c, 0xbd, 0x66, 0xcf, 0x7c, 0x29, - 0x7c, 0xe7, 0x96, 0xe0, 0x3c, 0x88, 0x4e, 0x17, 0x5a, 0xef, 0x78, 0x56, 0x44, 0xd2, 0x98, 0xbe, - 0xa7, 0xd7, 0x98, 0x0d, 0x51, 0x01, 0x8f, 0x6f, 0x14, 0x51, 0x3f, 0x1d, 0x4f, 0x64, 0xb5, 0x3e, - 0xa8, 0x98, 0x64, 0x9b, 0xce, 0xb2, 0x34, 0x33, 0x11, 0x40, 0x81, 0x1e, 0x40, 0xe3, 0x38, 0x9d, - 0xa4, 0x71, 0x7a, 0x3e, 0xbf, 0xa7, 0x65, 0x04, 0x50, 0x57, 0x57, 0x83, 0x6a, 0x51, 0x4d, 0x66, - 0x44, 0xfa, 0x91, 0xac, 0xf7, 0x90, 0xc7, 0xe1, 0x34, 0xe6, 0x85, 0xc0, 0x2f, 0x02, 0x04, 0xbf, - 0x4a, 0xf9, 0x48, 0x75, 0x05, 0x7d, 0xb4, 0xe8, 0x2f, 0x75, 0x01, 0x72, 0x74, 0xc7, 0xba, 0x82, - 0x5e, 0x86, 0xf6, 0xac, 0xa5, 0x24, 0xf2, 0x63, 0x68, 0x59, 0xab, 0xed, 0x01, 0xce, 0x82, 0x99, - 0xbd, 0x86, 0xfe, 0xcd, 0x59, 0x78, 0xe7, 0xc6, 0x9d, 0xab, 0xb7, 0xba, 0x52, 0x41, 0x6a, 0x30, - 0x2d, 0x49, 0xd7, 0x0f, 0x67, 0x61, 0x3c, 0xcd, 0xa5, 0x4a, 0x5f, 0xb8, 0x25, 0x20, 0x5d, 0x97, - 0x1f, 0x7d, 0xe9, 0xd4, 0x0c, 0x37, 0x46, 0x94, 0x9f, 0x87, 0x03, 0xc1, 0x47, 0x71, 0x94, 0x08, - 0xac, 0x17, 0x8f, 0x95, 0x32, 0x79, 0xa1, 0x7a, 0xac, 0x29, 0xf4, 0xad, 0x25, 0xe2, 0xa8, 0x53, - 0x9d, 0x37, 0xa7, 0x04, 0x3a, 0xcb, 0x2a, 0xba, 0x05, 0x44, 0x55, 0xc0, 0xcb, 0xb3, 0x34, 0x33, - 0xb7, 0x2d, 0xed, 0x9b, 0xe6, 0x22, 0xa3, 0x7f, 0xdf, 0x25, 0x5e, 0x45, 0xd6, 0xb5, 0x23, 0x4b, - 0x7f, 0x01, 0x1b, 0x7a, 0xb6, 0x13, 0x19, 0x16, 0xb4, 0x0c, 0x00, 0x13, 0x61, 0x2a, 0xc7, 0x44, - 0xf3, 0x1d, 0x57, 0x01, 0xd2, 0x0e, 0x0e, 0xba, 0xe6, 0x76, 0xd2, 0x12, 0xce, 0x46, 0xd1, 0x79, - 0x22, 0x46, 0x78, 0x63, 0x78, 0x4c, 0x4b, 0xf4, 0x4f, 0x2e, 0x6c, 0xa9, 0xa1, 0x33, 0x39, 0x17, - 0x79, 0x51, 0x6d, 0x83, 0x63, 0x35, 0xf6, 0xff, 0x72, 0xac, 0xc6, 0x1b, 0xe0, 0x19, 0x6c, 0xf4, - 0x63, 0xc1, 0xb3, 0x8a, 0x83, 0xda, 0x68, 0x09, 0x95, 0xe7, 0x06, 0x11, 0x7d, 0x3d, 0xab, 0x21, - 0xd4, 0x86, 0xc8, 0x01, 0x34, 0xb4, 0x6b, 0xa6, 0x21, 0x3e, 0xc3, 0x5b, 0x6a, 0x05, 0x1b, 0x33, - 0xdf, 0xe6, 0xfa, 0xab, 0xd3, 0x88, 0x3b, 0x6f, 0xa1, 0xbd, 0xa0, 0x5a, 0xf1, 0xd5, 0xd9, 0xb3, - 0xbf, 0x3a, 0x5b, 0xfb, 0xc4, 0x1a, 0x97, 0xb5, 0x75, 0xfb, 0x4b, 0xb4, 0x0f, 0xdf, 0x5e, 0x45, - 0x20, 0x27, 0x2f, 0xc0, 0x93, 0x44, 0xd5, 0x30, 0x1c, 0xdc, 0x46, 0x94, 0xc9, 0x45, 0xf4, 0xaf, - 0x8e, 0x0e, 0xaa, 0xd0, 0x7a, 0xf3, 0xf7, 0xe0, 0x13, 0xdb, 0xc8, 0xd3, 0xd2, 0xc8, 0xd2, 0xb2, - 0xbd, 0xd2, 0x51, 0xb9, 0x7a, 0xe7, 0x6b, 0x68, 0xac, 0x72, 0xcf, 0x57, 0xee, 0xfd, 0x68, 0xd1, - 0xbd, 0xc7, 0xb7, 0x31, 0xcb, 0x6d, 0x2f, 0xf7, 0x60, 0x5b, 0xdd, 0xa6, 0x03, 0x5e, 0xf0, 0x5f, - 0x65, 0x7c, 0x2c, 0xee, 0xbc, 0x52, 0x0f, 0x3a, 0xff, 0x78, 0xbf, 0xeb, 0xfc, 0xeb, 0xfd, 0xae, - 0xf3, 0xef, 0xf7, 0xbb, 0xce, 0x9f, 0xff, 0xb3, 0xfb, 0xad, 0xb3, 0x1a, 0xfe, 0x76, 0xfb, 0xe4, - 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa4, 0xc1, 0x04, 0xc8, 0x99, 0x13, 0x00, 0x00, + // 1750 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6e, 0x24, 0x49, + 0x11, 0xa6, 0x7e, 0xdc, 0x3f, 0xd1, 0x6e, 0x4f, 0x3b, 0xd7, 0x6b, 0x6a, 0xbc, 0x83, 0xd5, 0x93, + 0xa0, 0x99, 0x66, 0x24, 0x8c, 0xf0, 0x1e, 0x16, 0xb1, 0x97, 0x1d, 0x77, 0x7b, 0x96, 0x66, 0x77, + 0x7e, 0x36, 0xed, 0x99, 0x23, 0x28, 0x5d, 0x9d, 0xd8, 0xa5, 0xa9, 0xae, 0x6a, 0xaa, 0xaa, 0x3d, + 0xdd, 0x7b, 0x40, 0x02, 0x09, 0xc1, 0x85, 0x3b, 0xe2, 0xc0, 0x5b, 0xf0, 0x0e, 0x5c, 0x90, 0x78, + 0x04, 0x34, 0xdc, 0x78, 0x0a, 0x94, 0x91, 0x99, 0x55, 0xd9, 0xed, 0xb2, 0xbd, 0x58, 0x7b, 0xab, + 0xf8, 0x22, 0x2b, 0xf2, 0x8b, 0x9f, 0x8c, 0x8c, 0x2a, 0xe8, 0xce, 0xb2, 0xe8, 0x92, 0x17, 0xe2, + 0x60, 0x96, 0xa5, 0x45, 0x4a, 0xdc, 0xd9, 0xd9, 0xde, 0xe6, 0x6c, 0x7e, 0x16, 0x47, 0xa1, 0x42, + 0x68, 0x04, 0xed, 0x71, 0x32, 0x11, 0x8b, 0xe7, 0xa2, 0xe0, 0x84, 0x80, 0xff, 0x85, 0x58, 0xe6, + 0x81, 0xd7, 0x77, 0x06, 0x2d, 0x86, 0xcf, 0xe4, 0x11, 0x6c, 0x9d, 0x66, 0x3c, 0x7c, 0x7b, 0xbc, + 0x88, 0xf2, 0x42, 0x24, 0xa1, 0x08, 0x7c, 0xd4, 0xae, 0xa1, 0xa4, 0x0f, 0x9d, 0x91, 0xc8, 0xc3, + 0x2c, 0x9a, 0x15, 0x51, 0x9a, 0x04, 0x1b, 0x7d, 0x67, 0xd0, 0x66, 0x36, 0x44, 0xff, 0xeb, 0xc1, + 0xe6, 0xb3, 0x48, 0xc4, 0x93, 0x97, 0x28, 0xe7, 0x72, 0xbb, 0xd3, 0xe5, 0x4c, 0x04, 0x2d, 0x5c, + 0x8b, 0xcf, 0xe4, 0x01, 0xb4, 0x87, 0x3c, 0xbc, 0x10, 0xa8, 0xf0, 0x50, 0x51, 0x01, 0xa5, 0xf6, + 0x24, 0xfa, 0x5a, 0xf1, 0xe8, 0xb2, 0x0a, 0x90, 0x14, 0x4e, 0xa3, 0xa9, 0xf8, 0x6a, 0xce, 0x93, + 0x62, 0x3e, 0x35, 0x14, 0x2c, 0x88, 0xec, 0x42, 0xe3, 0x65, 0x3c, 0x79, 0x1e, 0x25, 0x41, 0xbb, + 0xef, 0x0c, 0x3c, 0xa6, 0x25, 0x83, 0xf3, 0x45, 0x00, 0x15, 0xce, 0x17, 0x65, 0x40, 0x3a, 0xab, + 0x01, 0x79, 0x91, 0x9e, 0x14, 0x3c, 0x99, 0xf0, 0x6c, 0xf2, 0x26, 0x12, 0xef, 0x82, 0x4d, 0x15, + 0x90, 0x55, 0x54, 0xbe, 0x7b, 0xc4, 0x73, 0x11, 0x74, 0xd1, 0x22, 0x3e, 0x93, 0x3d, 0x68, 0x1d, + 0x45, 0xc5, 0x48, 0xcc, 0x8a, 0x8b, 0x60, 0xab, 0xef, 0x0c, 0x7c, 0x56, 0xca, 0x64, 0x07, 0x36, + 0x4e, 0x42, 0x1e, 0x8b, 0xe0, 0x1e, 0xbe, 0xa0, 0x04, 0x42, 0x61, 0xf3, 0x59, 0x9a, 0x89, 0xe8, + 0x3c, 0xc1, 0x34, 0x05, 0x3d, 0x74, 0x6a, 0x05, 0x23, 0xdf, 0x03, 0x4f, 0xba, 0xb4, 0xdd, 0x77, + 0x06, 0x9d, 0xc3, 0xce, 0xc1, 0xec, 0xec, 0x60, 0x24, 0xc2, 0x68, 0xca, 0x63, 0x26, 0x71, 0x54, + 0xf3, 0x45, 0x40, 0xea, 0xd4, 0x7c, 0x21, 0x39, 0xc9, 0x10, 0xbd, 0x4e, 0xa2, 0x22, 0xf8, 0x00, + 0xad, 0x97, 0x32, 0xe9, 0x81, 0x77, 0x7a, 0xfa, 0x65, 0xb0, 0x83, 0xb0, 0x7c, 0xac, 0x29, 0x87, + 0x0f, 0xeb, 0xca, 0x81, 0x52, 0xd8, 0x1a, 0x4f, 0x67, 0x69, 0x56, 0x30, 0x91, 0xcf, 0xd2, 0x24, + 0x17, 0xd2, 0xd6, 0x71, 0x96, 0x05, 0x8e, 0xb2, 0x75, 0x9c, 0x65, 0xf4, 0xb7, 0xd0, 0x3b, 0x8a, + 0xd3, 0xf0, 0xed, 0x88, 0x17, 0x9c, 0x89, 0xdf, 0xcc, 0x45, 0x5e, 0xc8, 0x28, 0x28, 0x47, 0xd5, + 0x3a, 0x25, 0x48, 0x14, 0x2b, 0x27, 0x70, 0x15, 0x8a, 0x82, 0x8c, 0x30, 0xc6, 0x5f, 0x25, 0x1a, + 0x9f, 0x31, 0x8a, 0x17, 0x3c, 0x9b, 0x60, 0x75, 0xf8, 0x4c, 0x09, 0x12, 0xc5, 0x9d, 0xb0, 0xa2, + 0x7c, 0xa6, 0x04, 0x3a, 0x86, 0x6d, 0x6b, 0x7f, 0x4d, 0x73, 0x17, 0x1a, 0x2c, 0x7d, 0x37, 0x1e, + 0xe5, 0x81, 0xd3, 0xf7, 0x06, 0x3e, 0xd3, 0x12, 0x96, 0x5e, 0x1a, 0xcf, 0xa7, 0x89, 0x54, 0xb9, + 0xa8, 0xaa, 0x00, 0x7a, 0x1f, 0x36, 0xb0, 0x0e, 0xa5, 0x97, 0xd5, 0xbb, 0xf2, 0x91, 0xfe, 0xce, + 0x81, 0xf6, 0x73, 0xbe, 0x40, 0x22, 0x39, 0xf9, 0x04, 0x5a, 0xa6, 0x4a, 0x70, 0x51, 0xe7, 0xf0, + 0x23, 0x99, 0x91, 0x72, 0xc1, 0x81, 0xd1, 0x1e, 0x27, 0x45, 0xb6, 0x64, 0xe5, 0xe2, 0xbd, 0x4f, + 0xa1, 0xbb, 0xa2, 0x92, 0x3b, 0xbd, 0x15, 0x4b, 0x13, 0xcf, 0xb7, 0x62, 0x29, 0xbd, 0xbc, 0xe4, + 0xf1, 0x5c, 0x60, 0x94, 0x7c, 0xa6, 0x84, 0x9f, 0xb9, 0x3f, 0x75, 0xe8, 0x1b, 0x20, 0xc3, 0x4c, + 0xf0, 0x42, 0xe0, 0x26, 0xcf, 0x45, 0x9e, 0xf3, 0x73, 0x71, 0x5b, 0xac, 0x3d, 0x3b, 0xd6, 0x65, + 0x5c, 0x5d, 0x2b, 0xae, 0xf4, 0x09, 0x90, 0x91, 0x88, 0x45, 0x21, 0x74, 0x0f, 0xb9, 0xc1, 0xae, + 0x8c, 0x83, 0x26, 0x71, 0xfb, 0x62, 0xf2, 0x10, 0x7c, 0xd9, 0x91, 0x70, 0xb7, 0xce, 0x61, 0x57, + 0x86, 0xa8, 0x6c, 0x53, 0x0c, 0x55, 0x98, 0x10, 0x34, 0x37, 0x79, 0x5a, 0x20, 0x57, 0x8f, 0x55, + 0x80, 0x34, 0xfb, 0xf2, 0x5d, 0x22, 0x32, 0x5d, 0x1c, 0x4a, 0xa0, 0x7f, 0x2d, 0x39, 0xa0, 0x57, + 0xdf, 0x30, 0x10, 0x2b, 0x45, 0xf7, 0x03, 0xcd, 0xcc, 0x43, 0x66, 0x3d, 0xc9, 0xcc, 0x6e, 0x6a, + 0x75, 0xe4, 0xfc, 0x6f, 0x46, 0xee, 0x0f, 0x0e, 0x90, 0xd7, 0xb3, 0xc9, 0x3a, 0xb9, 0x67, 0x75, + 0x94, 0x91, 0x69, 0xe7, 0x70, 0x57, 0x6e, 0x7f, 0x55, 0xcb, 0xea, 0x9c, 0x7c, 0x0c, 0x0d, 0x65, + 0x5d, 0x07, 0xf5, 0x5e, 0x49, 0x5d, 0xc1, 0x4c, 0xab, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0x8d, + 0xaa, 0xa7, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0xf1, 0xa6, 0xac, 0xb6, 0x36, 0x53, 0x02, 0xfd, 0xcc, + 0x54, 0xc4, 0x5d, 0x03, 0x4c, 0x43, 0xf8, 0x48, 0x59, 0x78, 0x7a, 0xc9, 0xa3, 0x98, 0x9f, 0xc5, + 0xff, 0x57, 0xd1, 0xae, 0xe4, 0x2a, 0x80, 0x26, 0xbe, 0x3b, 0x1e, 0xe9, 0x83, 0x6f, 0x44, 0x3a, + 0x87, 0xaa, 0x87, 0xbc, 0xe0, 0x53, 0xa1, 0xad, 0xe1, 0x73, 0x99, 0x62, 0xf7, 0xc6, 0x14, 0x4b, + 0xff, 0x23, 0xf1, 0x4e, 0xde, 0x96, 0x1e, 0xfa, 0x2f, 0x85, 0x9b, 0x13, 0x4f, 0x7f, 0x04, 0x8d, + 0x93, 0xf0, 0x42, 0x4c, 0x39, 0xf9, 0x3e, 0x34, 0x91, 0xb9, 0xc8, 0x75, 0x1b, 0x68, 0x97, 0x35, + 0xce, 0x8c, 0x46, 0x56, 0x84, 0xf6, 0xaf, 0x8e, 0xe6, 0xca, 0x56, 0xee, 0x7a, 0x8d, 0x3d, 0x86, + 0xa6, 0xe6, 0x8b, 0x55, 0x76, 0xe5, 0x10, 0x19, 0x2d, 0x79, 0x08, 0x0d, 0xf4, 0x2e, 0x0f, 0xfc, + 0x8a, 0x08, 0x22, 0x4c, 0x2b, 0xe8, 0x31, 0x78, 0xaf, 0xd9, 0x58, 0x56, 0x02, 0xb2, 0x37, 0x34, + 0xb4, 0x24, 0xc9, 0xfd, 0x3c, 0xcd, 0x0b, 0x1d, 0x7b, 0x7c, 0x96, 0xd8, 0xab, 0x34, 0x53, 0x07, + 0xb3, 0xcb, 0xf0, 0x99, 0xfe, 0xc9, 0x01, 0xff, 0x45, 0x3a, 0x11, 0x64, 0x0b, 0xdc, 0xf1, 0x48, + 0x1b, 0x71, 0xc7, 0x23, 0x72, 0x1f, 0xed, 0xeb, 0x78, 0x37, 0xe5, 0xfe, 0xaf, 0xd9, 0x98, 0xe1, + 0x9e, 0x0f, 0xa0, 0x3d, 0xce, 0x5f, 0x65, 0xd1, 0x94, 0x67, 0x4b, 0x3d, 0x97, 0x54, 0x00, 0x76, + 0xa5, 0x42, 0x96, 0xb4, 0xaf, 0xd2, 0x8e, 0x02, 0x79, 0x08, 0xcd, 0xcf, 0xd9, 0xab, 0xa1, 0x34, + 0xb9, 0xb1, 0x6a, 0xd2, 0xe0, 0xf4, 0x33, 0xe8, 0x49, 0x26, 0xb8, 0xde, 0x54, 0xd6, 0x2e, 0x34, + 0x24, 0x56, 0x32, 0xd3, 0x52, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x9f, 0x29, 0x0b, 0xc7, 0x97, 0x22, + 0x29, 0xac, 0xda, 0x44, 0x19, 0x0d, 0x74, 0x99, 0x12, 0xc8, 0x03, 0xe5, 0xb5, 0x76, 0xaf, 0x25, + 0xb9, 0x48, 0x99, 0x21, 0x4a, 0x97, 0x00, 0x86, 0xc9, 0x3c, 0x2f, 0xd7, 0x3a, 0x75, 0x6b, 0x09, + 0x35, 0xe5, 0xa3, 0xbb, 0x0f, 0x48, 0xbd, 0x42, 0x98, 0x29, 0xac, 0x1f, 0x56, 0x85, 0xa5, 0xf2, + 0x79, 0xaf, 0xcc, 0xbb, 0xda, 0xa3, 0x2a, 0xaf, 0x0b, 0xe8, 0x58, 0x78, 0x6d, 0x8d, 0x3d, 0x2e, + 0x8b, 0xc3, 0xad, 0x8c, 0x21, 0xa2, 0x8d, 0x69, 0xf5, 0xcd, 0xdd, 0x98, 0x46, 0xba, 0xa5, 0xdc, + 0xb0, 0xd3, 0x00, 0xee, 0xad, 0x1e, 0x78, 0x73, 0xcb, 0xae, 0xc3, 0xb7, 0x6c, 0xf5, 0x47, 0x07, + 0xba, 0xc3, 0x78, 0x9e, 0x17, 0x22, 0x2b, 0x63, 0xda, 0xd6, 0x40, 0x99, 0xda, 0x0a, 0xa8, 0xcf, + 0x2e, 0xd9, 0x87, 0x0d, 0x19, 0x71, 0x75, 0xb8, 0xed, 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5d, + 0x26, 0xe8, 0x1b, 0x68, 0x1d, 0x9d, 0x8c, 0x3f, 0xcf, 0xd2, 0xf9, 0xac, 0xd6, 0x63, 0x33, 0xfe, + 0xba, 0xd6, 0xf8, 0xdb, 0x53, 0xa3, 0x9c, 0xf2, 0x0a, 0xa7, 0xb7, 0x9e, 0x9a, 0xde, 0x7c, 0x8d, + 0xf0, 0x05, 0x3d, 0x81, 0x6d, 0xe5, 0xae, 0xec, 0x38, 0x77, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, + 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xdb, 0x34, 0xfa, 0x4f, 0x17, 0xb6, 0x99, 0xc8, 0xa3, 0xaf, + 0xc5, 0x38, 0xc9, 0x8b, 0x6c, 0x1e, 0x9a, 0x8b, 0xe3, 0x17, 0xe9, 0x99, 0xce, 0x85, 0xc7, 0x94, + 0x70, 0xf3, 0x29, 0x21, 0x14, 0x9a, 0x76, 0x13, 0xb0, 0x17, 0x18, 0x05, 0x79, 0x02, 0xcd, 0x93, + 0x74, 0x9e, 0x85, 0x65, 0xe5, 0x63, 0xe7, 0x56, 0xfb, 0x2b, 0x05, 0x33, 0x0b, 0xc8, 0x17, 0x40, + 0x4e, 0x33, 0x9e, 0xe4, 0x31, 0x97, 0x94, 0xcc, 0x6b, 0xad, 0x6a, 0x20, 0xb3, 0xb4, 0x2b, 0x16, + 0x6a, 0x5e, 0x23, 0x07, 0xf6, 0x11, 0x0e, 0x9a, 0xc8, 0x6f, 0xcb, 0xf0, 0xd3, 0xe7, 0xc4, 0x3e, + 0xe4, 0x9f, 0xac, 0x55, 0x68, 0xd0, 0xc0, 0x57, 0xb6, 0xf1, 0x32, 0xb7, 0x15, 0x6c, 0x75, 0x1d, + 0xfd, 0xbd, 0x03, 0x9b, 0x36, 0x9b, 0x5b, 0xda, 0x45, 0x99, 0x3e, 0xf7, 0xf6, 0xf9, 0xce, 0xa4, + 0xcf, 0xaf, 0x9b, 0xa5, 0x37, 0xec, 0x99, 0x2f, 0x85, 0xef, 0x5e, 0x13, 0x9c, 0x3b, 0xd1, 0xe9, + 0x43, 0xe7, 0x15, 0xcf, 0x8a, 0x48, 0x1a, 0xd3, 0xf7, 0xf4, 0x06, 0xb3, 0x21, 0x2a, 0xe0, 0xfe, + 0x95, 0x22, 0x1a, 0xa6, 0xd3, 0x99, 0xac, 0xd6, 0x3b, 0x15, 0x93, 0x6c, 0xd3, 0x59, 0x96, 0x66, + 0x26, 0x02, 0x28, 0xd0, 0x23, 0x68, 0x9d, 0xa6, 0xb3, 0x34, 0x4e, 0xcf, 0x97, 0xb7, 0xb4, 0x8c, + 0x00, 0x9a, 0xea, 0x6a, 0x50, 0x2d, 0xaa, 0xcd, 0x8c, 0x48, 0x3f, 0x90, 0xf5, 0x1e, 0xf2, 0x38, + 0x9c, 0xc7, 0xbc, 0x10, 0xf8, 0x45, 0x80, 0xe0, 0x97, 0x29, 0x9f, 0xa8, 0xae, 0xa0, 0x8f, 0x16, + 0xfd, 0x95, 0x2e, 0x40, 0x8e, 0xee, 0x58, 0x57, 0xd0, 0xd3, 0xd0, 0x9e, 0xb5, 0x94, 0x44, 0x7e, + 0x02, 0x1d, 0x6b, 0xb5, 0x3d, 0xc0, 0x59, 0x30, 0xb3, 0xd7, 0xd0, 0xbf, 0x3b, 0x2b, 0xef, 0x5c, + 0xb9, 0x73, 0xf5, 0x56, 0x97, 0x2a, 0x48, 0x2d, 0xa6, 0x25, 0xe9, 0xfa, 0xf1, 0x22, 0x8c, 0xe7, + 0xb9, 0x54, 0xe9, 0x0b, 0xb7, 0x04, 0xa4, 0xeb, 0xf2, 0xe3, 0x30, 0x9d, 0x9b, 0xe1, 0xc6, 0x88, + 0xf2, 0x33, 0x72, 0x24, 0xf8, 0x24, 0x8e, 0x12, 0x81, 0xf5, 0xe2, 0xb1, 0x52, 0x26, 0x4f, 0x54, + 0x8f, 0x35, 0x85, 0xbe, 0xb3, 0x46, 0x1c, 0x75, 0xaa, 0xf3, 0xe6, 0x94, 0x40, 0x6f, 0x5d, 0x45, + 0x77, 0x80, 0xa8, 0x0a, 0x78, 0x7a, 0x96, 0x66, 0xe6, 0xb6, 0xa5, 0x43, 0xd3, 0x5c, 0x64, 0xf4, + 0x6f, 0xbb, 0xc4, 0xab, 0xc8, 0xba, 0x76, 0x64, 0xe9, 0x2f, 0x61, 0x4b, 0xcf, 0x76, 0x22, 0xc3, + 0x82, 0x96, 0x01, 0x60, 0x22, 0x4c, 0xe5, 0x98, 0x68, 0xbe, 0xe3, 0x2a, 0x40, 0xda, 0xc1, 0x41, + 0xd7, 0xdc, 0x4e, 0x5a, 0xc2, 0xd9, 0x28, 0x3a, 0x4f, 0xc4, 0x04, 0x6f, 0x0c, 0x8f, 0x69, 0x89, + 0xfe, 0xd9, 0x85, 0x1d, 0x35, 0x74, 0x26, 0xe7, 0x22, 0x2f, 0xaa, 0x6d, 0x70, 0xac, 0xc6, 0xfe, + 0x5f, 0x8e, 0xd5, 0x78, 0x03, 0x3c, 0x82, 0xad, 0x61, 0x2c, 0x78, 0x56, 0x71, 0x50, 0x1b, 0xad, + 0xa1, 0xf2, 0xdc, 0x20, 0xa2, 0xaf, 0x67, 0x35, 0x84, 0xda, 0x10, 0x39, 0x82, 0x96, 0x76, 0xcd, + 0x34, 0xc4, 0x47, 0x78, 0x4b, 0xd5, 0xb0, 0x31, 0xf3, 0x6d, 0xae, 0xbf, 0x3a, 0x8d, 0xb8, 0xf7, + 0x12, 0xba, 0x2b, 0xaa, 0x9a, 0xaf, 0xce, 0x81, 0xfd, 0xd5, 0xd9, 0x39, 0x24, 0xd6, 0xb8, 0xac, + 0xad, 0xdb, 0x5f, 0xa2, 0x43, 0xf8, 0xb0, 0x8e, 0x40, 0x4e, 0x9e, 0x80, 0x27, 0x89, 0xaa, 0x61, + 0x38, 0xb8, 0x8e, 0x28, 0x93, 0x8b, 0xe8, 0xdf, 0x1c, 0x1d, 0x54, 0xa1, 0xf5, 0xe6, 0xef, 0xc1, + 0xc7, 0xb6, 0x91, 0x87, 0xa5, 0x91, 0xb5, 0x65, 0x07, 0xa5, 0xa3, 0x72, 0xf5, 0xde, 0x57, 0xd0, + 0xaa, 0x73, 0xcf, 0x57, 0xee, 0xfd, 0x78, 0xd5, 0xbd, 0xfb, 0xd7, 0x31, 0xcb, 0x6d, 0x2f, 0x0f, + 0x60, 0x57, 0xdd, 0xa6, 0x23, 0x5e, 0xf0, 0x5f, 0x67, 0x7c, 0x2a, 0x6e, 0xbc, 0x52, 0x8f, 0x7a, + 0xff, 0x78, 0xbf, 0xef, 0xfc, 0xeb, 0xfd, 0xbe, 0xf3, 0xef, 0xf7, 0xfb, 0xce, 0x5f, 0xfe, 0xb3, + 0xff, 0x9d, 0xb3, 0x06, 0xfe, 0x9e, 0xfb, 0xf8, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x0d, + 0x5d, 0xdb, 0xc1, 0x13, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3141,6 +3150,18 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.TrackExistence { + i-- + if m.TrackExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa8 + } if len(m.TTL) > 0 { i -= len(m.TTL) copy(dAtA[i:], m.TTL) @@ -5636,6 +5657,9 @@ func (m *FieldOptions) Size() (n int) { if l > 0 { n += 2 + l + sovPrivate(uint64(l)) } + if m.TrackExistence { + n += 3 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7255,6 +7279,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } m.TTL = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 21: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TrackExistence", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.TrackExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/pb/private.proto b/pb/private.proto index a82b676ef..6822a872d 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -27,6 +27,7 @@ message FieldOptions { Decimal Max = 18; string TimeUnit = 19; string TTL = 20; + bool TrackExistence = 21; } message ImportResponse { diff --git a/pql/ast.go b/pql/ast.go index e72de72d2..362d63f67 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -1009,6 +1009,67 @@ func (c *Call) HasConditionArg() bool { return false } +// FieldEquality returns an equality test suitable for a non-BSI field. +// Given a field name, it returns an equality test from the corresponding +// argument. An equality test indicates whether the test is actually +// against null, or if not what specific uint64 value it's against, and +// whether it's an equal or non-equal test. This exists mostly to be able +// to extract `== null` and `!= null` tests consistently even for non-BSI +// fields. +func (c *Call) FieldEquality(k string) (isNull bool, value uint64, equal bool, err error) { + v := c.Args[k] + equal = true + if cond, ok := v.(*Condition); ok { + // only exactly EQ and NEQ are equality tests. + if cond.Op == EQ || cond.Op == NEQ { + equal = cond.Op == EQ + if cond.Value == nil { + return true, 0, equal, nil + } + // pick either cond.Value, or cond.Value[0] if it's + // a slice of interface{}, to be our new "v" and then + // fall through to handling we would have done for + // a non-condition. + if values, ok := cond.Value.([]interface{}); ok { + if len(values) != 1 { + return false, 0, false, fmt.Errorf("expected exactly one value for EQ/NEQ, got %d", len(values)) + } + v = values[0] + } else { + v = cond.Value + } + } else { + return false, 0, false, fmt.Errorf("only support == or != conditions, got %s", cond.Op) + } + // + } + if u, ok := v.(uint64); ok { + return false, u, equal, nil + } + if i, ok := v.(int64); ok { + return false, uint64(i), equal, nil + } + return false, 0, false, fmt.Errorf("expected integer or nil value, got %T", v) +} + +// FieldRange yields the range test corresponding to the given key, +// which means either it's a Condition, or it's just a raw equality +// to a value, which we treat as {EQ, []any{value}}. This is suitable +// for use with BSI fields, which generally get their operations as +// Conditions, and simplifies the caller side of this. +func (c *Call) FieldRange(k string) (op Token, value interface{}, err error) { + v := c.Args[k] + if cond, ok := v.(*Condition); ok { + return cond.Op, cond.Value, nil + } + // this shows up if someone wrote `foo=3`, which was + // how we spelled it for non-BSI fields. we want to handle that + // gracefully. If it had been a condition, we'd have written + // it as {Op: EQ, Value: []interface{}{v}}, so we return what + // the above would have produced in that case. Sneaky, huh. + return EQ, []interface{}{v}, nil +} + // TranslateInfo returns the relevant translation fields. func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) { switch c.Name { diff --git a/rbf.go b/rbf.go index 78081f1bc..15ecca41b 100644 --- a/rbf.go +++ b/rbf.go @@ -218,6 +218,96 @@ func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (c return tx.addOrRemove(index, field, view, shard, true, a...) } +// Removed clears the specified bits and tells you which ones it actually removed. +func (tx *RBFTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + if len(a) == 0 { + return a, nil + } + name := rbfName(index, field, view, shard) + // this special case can/should possibly go away, except that it + // turns out to be by far the most common case, and we need to know + // there's at least two items to simplify the check-sorted thing. + if len(a) == 1 { + hi, lo := highbits(a[0]), lowbits(a[0]) + rc, err := tx.tx.Container(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to retrieve container") + } + if rc.N() == 0 { + return a[:0], nil + } + rc1, chng := rc.Remove(lo) + if !chng { + return a[:0], nil + } + if rc1.N() == 0 { + err = tx.tx.RemoveContainer(name, hi) + } else { + err = tx.tx.PutContainer(name, hi, rc1) + } + if err != nil { + return a[:0], err + } + return a[:1], nil + } + + changeCount := 0 + changed = a + + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + + for i, v := range a { + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if rc == nil || rc.N() == 0 { + err = tx.tx.RemoveContainer(name, lastHi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to remove container") + } + } else { + err = tx.tx.PutContainer(name, lastHi, rc) + if err != nil { + return a[:0], errors.Wrap(err, "failed to put container") + } + } + } + // get the next container + rc, err = tx.tx.Container(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to retrieve container") + } + } // else same container, keep adding bits to rct. + chng := false + rc, chng = rc.Remove(lo) + if chng { + changed[changeCount] = v + changeCount++ + } + lastHi = hi + } + // write the last updates. + + if rc == nil || rc.N() == 0 { + err = tx.tx.RemoveContainer(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to remove container") + } + } else { + err = tx.tx.PutContainer(name, hi, rc) + if err != nil { + return a[:0], errors.Wrap(err, "failed to put container") + } + } + return changed[:changeCount], nil +} + // sortedParanoia is a flag to enable a check for unsorted inputs to addOrRemove, // which is expensive in practice and only really useful occasionally. const sortedParanoia = false diff --git a/schema.go b/schema.go index 3dacfa24d..a8944dd62 100644 --- a/schema.go +++ b/schema.go @@ -264,6 +264,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { TimeQuantum: timeQuantum, TTL: fo.TTL, ForeignIndex: foreignIndex, + TrackExistence: fo.TrackExistence, }, } } @@ -367,6 +368,7 @@ func FieldToFieldInfo(fld *dax.Field) *FieldInfo { TimeQuantum: TimeQuantum(fld.Options.TimeQuantum), TTL: fld.Options.TTL, ForeignIndex: fld.Options.ForeignIndex, + TrackExistence: fld.Options.TrackExistence, }, Views: nil, // TODO(tlt): do we need views populated? } @@ -410,6 +412,13 @@ func fieldToFieldType(f *dax.Field) string { } } +// FieldFromFieldOptions creates a dax.Field given a set of existing +// field options. It should possibly be unconditionally setting +// TrackExistence, because it's called in two places in SQL3 both +// of which are creating new tables, but for now I'm trying to keep +// its behavior transparent, and handle the enabling of TrackExistence +// in the code that knows it is creating a field, thus, in sql's +// create/alter table, or in api.CreateField. func FieldFromFieldOptions(fname dax.FieldName, opts ...FieldOption) (*dax.Field, error) { fo, err := newFieldOptions(opts...) if err != nil { @@ -485,6 +494,9 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) { default: return nil, errors.Errorf("unsupport field type: %s", fld.Type) } + if fld.Options.TrackExistence { + opts = append(opts, OptFieldTrackExistence()) + } return opts, nil } diff --git a/server.go b/server.go index b04feea7f..18591dd55 100644 --- a/server.go +++ b/server.go @@ -954,20 +954,37 @@ func (s *Server) ViewsRemoval(ctx context.Context) { } } } - if field.Options().NoStandardView && field.view(viewStandard) != nil { - // delete view "standard" if NoStandardView is true and view "standard" exists - for _, shard := range field.AvailableShards(true).Slice() { - err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) - if err != nil { - s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) + if field.Options().NoStandardView { + if field.view(viewStandard) != nil { + // delete view "standard" if NoStandardView is true and view "standard" exists + for _, shard := range field.AvailableShards(true).Slice() { + err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) + if err != nil { + s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) + } } - } - err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) - if err != nil { - s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) + if err != nil { + s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) + } + s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name) + } + if field.view(viewExistence) != nil { + // delete view "existence" if NoStandardView is true and view "existence" exists + for _, shard := range field.AvailableShards(true).Slice() { + err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewExistence, shard, nil) + if err != nil { + s.logger.Errorf("delete view %s from shard %d: %s", viewExistence, shard, err) + } + } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewExistence) + if err != nil { + s.logger.Errorf("view: %s, delete view: %s", viewExistence, err) + } + s.logger.Infof("view %s deleted - index: %s, field: %s ", viewExistence, index.name, field.name) } - s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name) } } } diff --git a/server/grpc_test.go b/server/grpc_test.go index bc6e2a0b0..876570870 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -566,7 +566,7 @@ func TestQuerySQL(t *testing.T) { {[]columnResponse{uint64(5), int64(16), []string{"blue"}, int64(60), int64(-2), "2014-01-02T12:32:00Z"}}, {[]columnResponse{uint64(6), int64(34), []string{"blue"}, int64(70), int64(100), "2010-05-02T12:32:00Z"}}, {[]columnResponse{uint64(7), int64(27), []string{"blue"}, int64(80), int64(0), "2016-08-02T12:32:00Z"}}, - {[]columnResponse{uint64(8), int64(16), []string{}, int64(90), int64(-13), "2020-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(8), int64(16), []string(nil), int64(90), int64(-13), "2020-01-02T12:32:00Z"}}, {[]columnResponse{uint64(9), int64(16), []string{"red"}, int64(100), int64(80), "2000-03-02T12:32:00Z"}}, {[]columnResponse{uint64(10), int64(31), []string{"red"}, int64(110), int64(-2), "2018-01-02T12:32:00Z"}}, }, diff --git a/server/server_test.go b/server/server_test.go index 88e60d803..e46547b6e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -694,7 +694,7 @@ func TestMain_ImportTimestamp(t *testing.T) { } exp := []string{ - "standard", "standard_2018", "standard_201801", "standard_20180101", + "existence", "standard", "standard_2018", "standard_201801", "standard_20180101", "standard_2019", "standard_201912", "standard_20191231", } got := []string{} diff --git a/server_test.go b/server_test.go index 4ea5f0286..7bd8ea57f 100644 --- a/server_test.go +++ b/server_test.go @@ -42,7 +42,7 @@ func TestViewsRemovalTTL(t *testing.T) { { name: "date_old", date: "2001-02-03T04:05", - expViews: []string{"standard"}, + expViews: []string{"existence", "standard"}, /* date_old (2001-02-03T04:05), this will create these views: - standard - standard_2001 @@ -56,6 +56,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_now", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -73,6 +74,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_yesterday", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateYesterday.Year()), "standard_" + fmt.Sprintf("%d%02d", dateYesterday.Year(), dateYesterday.Month()), @@ -107,6 +109,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_last_of_month", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour(), dateLastDayOfMonth.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateLastDayOfMonth.Year()), "standard_" + fmt.Sprintf("%d%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month()), @@ -124,6 +127,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_first_hour_day", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0, 0), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -141,6 +145,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_last_hour_day", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23, 59), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -228,7 +233,7 @@ func TestViewsRemovalStandard(t *testing.T) { name: "t2_keep_standard", date: "2001-02-03T04:05", noStandardView: "false", - expViews: []string{"standard"}, + expViews: []string{"existence", "standard"}, /* date 2001-02-03T04:05, this will create these views: - standard - standard_2001 diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index f80500534..c6f2022b7 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -541,12 +541,19 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex pqlOp = pql.NEQ } switch typ := expr.lhs.Type().(type) { - case *parser.DataTypeID: + case *parser.DataTypeID, *parser.DataTypeString, *parser.DataTypeIDSet, *parser.DataTypeStringSet: if strings.EqualFold(lhs.columnName, string(dax.PrimaryKeyFieldName)) { return nil, sql3.NewErrInvalidColumnInFilterExpression(0, 0, string(dax.PrimaryKeyFieldName), "is/is not null") } - return nil, sql3.NewErrInvalidTypeInFilterExpression(0, 0, typ.TypeDescription(), "is/is not null") - + return &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + lhs.columnName: &pql.Condition{ + Op: pqlOp, + Value: nil, + }, + }, + }, nil case *parser.DataTypeInt, *parser.DataTypeDecimal, *parser.DataTypeTimestamp: return &pql.Call{ Name: "Row", diff --git a/sql3/planner/opaltertable.go b/sql3/planner/opaltertable.go index 3b3b362f4..6da8db0e5 100644 --- a/sql3/planner/opaltertable.go +++ b/sql3/planner/opaltertable.go @@ -94,6 +94,8 @@ func (i *alterTableRowIter) Next(ctx context.Context) (types.Row, error) { fos := i.columnDef.fos fld, err := pilosa.FieldFromFieldOptions(fname, fos...) + // all newly created fields unconditionally have TrackExistence turned on. + fld.Options.TrackExistence = true if err != nil { return nil, err } diff --git a/sql3/planner/opcreatetable.go b/sql3/planner/opcreatetable.go index fe9c877f3..25de51bbf 100644 --- a/sql3/planner/opcreatetable.go +++ b/sql3/planner/opcreatetable.go @@ -112,6 +112,8 @@ func (i *createTableRowIter) Next(ctx context.Context) (types.Row, error) { for _, f := range i.columns { fld, err := pilosa.FieldFromFieldOptions(dax.FieldName(f.name), f.fos...) + // We unconditionally turn on TrackExistence for all newly-created fields. + fld.Options.TrackExistence = true if err != nil { return nil, errors.Wrapf(err, "creating field from field options: %s", f.name) } diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index b1f0cbeff..e6267e12d 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -265,24 +265,22 @@ func (i *distinctScanRowIter) Next(ctx context.Context) (types.Row, error) { row[0] = pql.NewDecimal(val, t.Scale) case *parser.DataTypeIDSet: - //empty sets are null val, ok := result.([]uint64) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result) } - if len(val) == 0 { + if val == nil { row[0] = nil } else { row[0] = val } case *parser.DataTypeStringSet: - //empty sets are null val, ok := result.([]string) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result) } - if len(val) == 0 { + if val == nil { row[0] = nil } else { row[0] = val diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 6c876cea3..47154a65e 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -314,24 +314,22 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { } else { switch mappedColumn.dataType.(type) { case *parser.DataTypeIDSet: - //empty sets are null val, ok := result.Rows[mappedSrcColIdx].([]uint64) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result.Rows[mappedSrcColIdx]) } - if len(val) == 0 { + if val == nil { row[mappedColIdx] = nil } else { row[mappedColIdx] = val } case *parser.DataTypeStringSet: - //empty sets are null val, ok := result.Rows[mappedSrcColIdx].([]string) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result.Rows[mappedSrcColIdx]) } - if len(val) == 0 { + if val == nil { row[mappedColIdx] = nil } else { row[mappedColIdx] = val diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index ac007da79..5d35d2c5c 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -452,10 +452,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "int", constraints: "min 100 max 10000", expOptions: pilosa.FieldOptions{ - Type: "int", - Base: 100, - Min: pql.NewDecimal(100, 0), - Max: pql.NewDecimal(10000, 0), + Type: "int", + Base: 100, + Min: pql.NewDecimal(100, 0), + Max: pql.NewDecimal(10000, 0), + TrackExistence: true, }, }, { @@ -463,7 +464,8 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "bool", constraints: "", expOptions: pilosa.FieldOptions{ - Type: "bool", + Type: "bool", + TrackExistence: true, }, }, //test creates timestamp column without the epoch and expects the "Base" to be defaulted 0, which is the unix epoch @@ -472,11 +474,12 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "timestamp", constraints: "timeunit 'ms'", expOptions: pilosa.FieldOptions{ - Base: 0, - Type: "timestamp", - TimeUnit: "ms", - Min: pql.NewDecimal(-62135596799000, 0), - Max: pql.NewDecimal(253402300799000, 0), + Base: 0, + Type: "timestamp", + TimeUnit: "ms", + Min: pql.NewDecimal(-62135596799000, 0), + Max: pql.NewDecimal(253402300799000, 0), + TrackExistence: true, }, }, { @@ -484,10 +487,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "decimal(2)", constraints: "", expOptions: pilosa.FieldOptions{ - Type: "decimal", - Scale: 2, - Min: pql.NewDecimal(-9223372036854775808, 2), - Max: pql.NewDecimal(9223372036854775807, 2), + Type: "decimal", + Scale: 2, + Min: pql.NewDecimal(-9223372036854775808, 2), + Max: pql.NewDecimal(9223372036854775807, 2), + TrackExistence: true, }, }, { @@ -495,10 +499,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "string", constraints: "cachetype ranked size 1000", expOptions: pilosa.FieldOptions{ - Type: "mutex", - Keys: true, - CacheType: "ranked", - CacheSize: 1000, + Type: "mutex", + Keys: true, + CacheType: "ranked", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -506,10 +511,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "stringset", constraints: "cachetype lru size 1000", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: true, - CacheType: "lru", - CacheSize: 1000, + Type: "set", + Keys: true, + CacheType: "lru", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -517,12 +523,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "stringsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ - Type: "time", - Keys: true, - CacheType: "", - CacheSize: 0, - TimeQuantum: "YMD", - TTL: time.Duration(24 * time.Hour), + Type: "time", + Keys: true, + CacheType: "", + CacheSize: 0, + TimeQuantum: "YMD", + TTL: time.Duration(24 * time.Hour), + TrackExistence: true, }, }, { @@ -530,10 +537,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "id", constraints: "cachetype ranked size 1000", expOptions: pilosa.FieldOptions{ - Type: "mutex", - Keys: false, - CacheType: "ranked", - CacheSize: 1000, + Type: "mutex", + Keys: false, + CacheType: "ranked", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -541,10 +549,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idset", constraints: "cachetype lru", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: false, - CacheType: "lru", - CacheSize: pilosa.DefaultCacheSize, + Type: "set", + Keys: false, + CacheType: "lru", + CacheSize: pilosa.DefaultCacheSize, + TrackExistence: true, }, }, { @@ -552,10 +561,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idset", constraints: "cachetype lru size 1000", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: false, - CacheType: "lru", - CacheSize: 1000, + Type: "set", + Keys: false, + CacheType: "lru", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -563,12 +573,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ - Type: "time", - Keys: false, - CacheType: "", - CacheSize: 0, - TimeQuantum: "YMD", - TTL: time.Duration(24 * time.Hour), + Type: "time", + Keys: false, + CacheType: "", + CacheSize: 0, + TimeQuantum: "YMD", + TTL: time.Duration(24 * time.Hour), + TrackExistence: true, }, }, } diff --git a/sql3/test/defs/defs_set_functions.go b/sql3/test/defs/defs_set_functions.go index 893f9bfbf..c3acc23c1 100644 --- a/sql3/test/defs/defs_set_functions.go +++ b/sql3/test/defs/defs_set_functions.go @@ -97,8 +97,8 @@ var setFunctionTests = TableTest{ ), srcRows( srcRow(int64(1), int64(10), int64(100), []string{"POST"}, []int64{101}), - srcRow(int64(2), int64(20), int64(200), []string{"GET"}, nil), - srcRow(int64(3), int64(30), int64(300), []string{"GET", "POST"}, nil), + srcRow(int64(2), int64(20), int64(200), []string{"GET"}, []int64(nil)), + srcRow(int64(3), int64(30), int64(300), []string{"GET", "POST"}, []int64(nil)), ), ), SQLTests: []SQLTest{ diff --git a/sql3/test/defs/types.go b/sql3/test/defs/types.go index 9a1a0486c..bbbf493b0 100644 --- a/sql3/test/defs/types.go +++ b/sql3/test/defs/types.go @@ -226,14 +226,18 @@ func (sr sourceRows) insertTuples(t *testing.T) string { case float64: sb.WriteString(fmt.Sprintf("%.2f", v)) case []int64: - strs := make([]string, len(v)) - for i := range v { - strs[i] = fmt.Sprintf("%d", v[i]) + if v == nil { + sb.WriteString("NULL") + } else { + strs := make([]string, len(v)) + for i := range v { + strs[i] = fmt.Sprintf("%d", v[i]) + } + sb.WriteString("[" + strings.Join(strs, ",") + "]") } - sb.WriteString("[" + strings.Join(strs, ",") + "]") case []string: - if len(v) == 0 { - sb.WriteString("[]") + if v == nil { + sb.WriteString("NULL") } else { sb.WriteString("['" + strings.Join(v, "','") + "']") } diff --git a/stattx.go b/stattx.go index ec40537c0..45ff5b50b 100644 --- a/stattx.go +++ b/stattx.go @@ -149,6 +149,7 @@ const ( kRemoveContainer kAdd kRemove + kRemoved kContains kContainerIterator kCount @@ -182,6 +183,8 @@ func (k kall) String() string { return "kAdd" case kRemove: return "kRemove" + case kRemoved: + return "kRemoved" case kContains: return "kContains" case kContainerIterator: @@ -358,6 +361,23 @@ func (c *statTx) Remove(index, field, view string, shard uint64, a ...uint64) (c return c.b.Remove(index, field, view, shard, a...) } +func (c *statTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + me := kRemoved + + t0 := time.Now() + defer func() { + c.stats.add(me, time.Since(t0)) + }() + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Removed(index, field, view, shard, a...) +} + func (c *statTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { me := kContains diff --git a/tx.go b/tx.go index 3a4079f5b..f43967277 100644 --- a/tx.go +++ b/tx.go @@ -101,6 +101,10 @@ type Tx interface { // Remove removes the 'a' values from the Bitmap for the fragment. Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) + // Removed removes values, returning the set of values it removed. + // It may overwrite the slice passed to it. + Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) + // Contains tests if the uint64 v is stored in the fragment's Bitmap. Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) diff --git a/view.go b/view.go index d73caf530..cc1e18c1c 100644 --- a/view.go +++ b/view.go @@ -24,9 +24,12 @@ import ( // View layout modes. const ( + // standard view holds regular set/mutex data viewStandard = "standard" - + // bsig_X view holds BSI data for X viewBSIGroupPrefix = "bsig_" + // existence view holds existence bits for a specific field + viewExistence = "existence" ) // view represents a container for field data.