From 54dbeec1af9d82d050e4f73abdf022418019757c Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 6 Mar 2023 16:47:36 -0600 Subject: [PATCH] support null/non-null tests for non-BSI fields There's a lot going on here. First, we were treating "the test is a Condition" as implying BSI, which it doesn't anymore. Second, the behavior of conditions was weird and BSI-specific. Third, we had to propagate these changes and features throughout a bunch of code, including both the core featurebase code and the DAX replacements/copies of it, plus the SQL3 layer. We refactor this so that tests for equality and inequality work for non-BSI fields, so now if you accidentally use `==` in a Row call on a non-BSI field, it still works; that's not specific to BSI fields anymore. We add a TrackExistence flag to fields, and propagate it through things like our protobuf code, etcetera, so that we can successfully create fields. Newly-created fields get this by default, because we add it unconditionally to them, but the paths that are being called with existing fields don't add it. So, when we "create" (really, just load the definition of) a field from something stored in the schema, we don't add TrackExistence to it, but any path to creating a new field should. A time quantum field with NoStandardView will *effectively* lack TrackExistence. For sets, mutexes, and time quantums with a standard view, anything that sets bits will also set a corresponding bit for the record in a new "existence" view. This allows us to distinguish between an empty set and a null, and also allows null checks to be constant-time. When clearing bits, we don't clear existence bits EXCEPT that if you clear a bit in a mutex, *and the bit actually existed*, we clear the existence bit. For sets and time quantums, clearing bits never clears the existence bit. Deleting records clears the existence bit. We also add code to the `batch` subpackage to generate suitable existence field bitmaps and import them. This logic correctly handles empty sets and nils. The `batch` package does not allow specification of anything equivalent to clearing a single bit from an existing record, so we don't have to deal with the mutex complexity in that case, which is good because it would be impossible. This requires a number of other subtle changes, such as allowing new fields to have more than one FieldOption specified for them. We also drop the handful of implementation bits relating to the "fullySorted" internal-use-only import flag, which existed only to support the JSON ingest API, which we've removed. The most dangerous part of this is that the mutex semantics are impossible to implement on top of our existing API, because they require us to know, not how *many* bits we cleared, but which *specific* bits we cleared. I've implemented this as a new Tx method, which is almost certainly going to be tech debt one day; if we some day drop the Import API, we should remove that. The testing for this is only currently covering the Set/Clear behavior of PQL, and the Import API. The batch tests haven't been written yet. Fields that don't have existence tracking enabled refuse to perform null/not-null tests. They should also report themselves as having no null values -- if a record exists, sets in it are considered empty rather than null. The SQL3 support requires a number of subtle modifications to both featurebase and some addon tooling. The essential thing is dropping the unconditional translation of nil slices to non-nil empty slices in translateResult, both in the executor and the orchestrator. We also modify the logic that handles generating results from Extract calls, to ensure that non-null sets get an empty slice created for them even if they never have any values assigned. The expected results for some tests are different now; we expect to get nil slices, rather than 0-length non-nil slices, for fields which were never written for a given record. Most tests were not changed. (In every case, if a test was failing, I actually checked the logic before changing expected results. This required a lot of tracking down of edge cases.) The batch package now rejects as an error attempts to clear single bits from mutex fields, because so far as I can tell it's simply impossible to have a roaring import that specifies the correct semantics there; you can't tell whether to clear an existence bit without access to the currently-set bits, which the batch API doesn't have. We already supported the special case of specifying a clear value of nil for clearing a mutex field; now that is the only allowed value for a mutex field to have in row.Clears. We change the logic for fixing up incoming view names (in two places) to stop assuming that any view in a time field other than "" that does not have viewStandard as a prefix is a partial time quantum name that should have "standard_" prepended to it. This allows us to submit bitmaps for "existence" to time quantum fields and not have them silently transformed into "standard_existence" because that's what we'd do with "202203". We drop the field ClearBits method, which was totally unused. We drop the sliceDifference function, which was used in a previous mutex implementation and hasn't been used in ages, and the test case for it, and the helper function used only by that test case. --- api.go | 65 +-- api_directive.go | 2 +- batch/batch.go | 63 ++- batch/batch_test.go | 23 +- catcher.go | 11 + dax/queryer/orchestrator.go | 6 +- dax/table.go | 1 + delete_test.go | 6 +- encoding/proto/proto.go | 2 + executor.go | 266 +++++++----- executor_test.go | 111 ++++- field.go | 206 +++++++-- fragment.go | 128 +++--- fragment_internal_test.go | 63 --- http_handler_test.go | 1 + idk/api/source_test.go | 4 +- index.go | 52 ++- null_test.go | 620 +++++++++++++++++++++++++++ pb/private.pb.go | 264 +++++++----- pb/private.proto | 1 + pql/ast.go | 61 +++ rbf.go | 90 ++++ schema.go | 12 + server.go | 39 +- server/grpc_test.go | 2 +- server/server_test.go | 2 +- server_test.go | 9 +- sql3/planner/expressionpql.go | 13 +- sql3/planner/opaltertable.go | 2 + sql3/planner/opcreatetable.go | 2 + sql3/planner/oppqldistinctscan.go | 6 +- sql3/planner/oppqltablescan.go | 6 +- sql3/sql_complex_test.go | 103 +++-- sql3/test/defs/defs_set_functions.go | 4 +- sql3/test/defs/types.go | 16 +- stattx.go | 20 + tx.go | 4 + view.go | 5 +- 38 files changed, 1768 insertions(+), 523 deletions(-) create mode 100644 null_test.go diff --git a/api.go b/api.go index 74dd32532..c8f792992 100644 --- a/api.go +++ b/api.go @@ -367,8 +367,8 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // CreateField makes the named field in the named index with the given options. -// This method currently only takes a single functional option, but that may be -// changed in the future to support multiple options. +// +// The resulting field will always have TrackExistence set. func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField") defer span.Finish() @@ -381,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str // authN/Z info requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx + // newFieldOptions is also used in the path through the index creating + // a field from an update from DAX, so it can't assume it can always + // override this. But we're the call path for creating new fields, and + // new fields should always have TrackExistence on. + opts = append(opts, OptFieldTrackExistence()) // Apply and validate functional options. fo, err := newFieldOptions(opts...) if err != nil { @@ -500,10 +505,16 @@ func importWorker(importWork chan importJob) { // incorrectly). One way to address this would be to change the logic // overall so there weren't conflicts. For now, we just // rely on the field type to inform the intended view name. - if viewName == "" { + // contrast with cleanupView, which is similar but unfortunately not quite identical + switch viewName { + case "": viewName = viewStandard - } else if j.field.Type() == FieldTypeTime { - viewName = fmt.Sprintf("%s_%s", viewStandard, viewName) + case viewStandard, viewExistence: + // do nothing, these are fine + default: // possibly a time view + if j.field.Type() == FieldTypeTime && !strings.HasPrefix(viewName, viewStandard) { + viewName = viewStandard + "_" + viewName + } } if len(viewData) == 0 { return fmt.Errorf("no data to import for view: %s", viewName) @@ -1316,7 +1327,6 @@ type ImportOptions struct { Clear bool IgnoreKeyCheck bool Presorted bool - fullySorted bool // format-aware sorting, internal use only please. suppressLog bool // test Tx atomicity if > 0 @@ -1523,7 +1533,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, return errors.Wrap(err, "validating api method") } - api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard) idx, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") @@ -1642,6 +1651,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, // across many fields in a single shard. It can both set and clear // bits and updates caches/bitDepth as appropriate, although only the // bitmap parts happen truly transactionally. +// +// This function does not attempt to do existence tracking, because +// it can't; there's no way to distinguish empty sets from not setting +// bits. As a result, users of this endpoint are responsible for +// providing corrected existence views for fields with existence +// tracking. Our batch API does that. func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error { index, err := api.Index(ctx, indexName) if err != nil { @@ -1768,12 +1783,15 @@ func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { // TODO wouldn't hurt to have consolidated logic somewhere for validating view names. switch fieldType { case FieldTypeSet, FieldTypeTime: - if viewUpdate.View == "" { - viewUpdate.View = "standard" - } - // add 'standard_' if we just have a time... this is how IDK works by default - if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { - viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View) + switch viewUpdate.View { + case "": + viewUpdate.View = viewStandard + case viewStandard, viewExistence: + // do nothing, these are fine + default: + if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { + viewUpdate.View = viewStandard + "_" + viewUpdate.View + } } case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: if viewUpdate.View == "" { @@ -2038,21 +2056,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { +func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) { ef := index.existenceField() if ef == nil { return nil } - - existenceRowIDs := make([]uint64, len(columnIDs)) - // If we don't gratuitously hand-duplicate things in field.Import, - // the fact that fragment.bulkImport rewrites its row and column - // lists can burn us if we don't make a copy before doing the - // existence field write. - columnCopy := make([]uint64, len(columnIDs)) - copy(columnCopy, columnIDs) - options := ImportOptions{} - return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options) + tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) + if err != nil { + return err + } + defer finisher(&err0) + // markExistingInView is simpler/faster than Import, but unusually, we use the + // standard view of the existence field, instead of the existence view of + // a specific field, when doing the index-wide update. + return ef.markExistingInView(tx, columnIDs, viewStandard, shard) } // ShardDistribution returns an object representing the distribution of shards diff --git a/api_directive.go b/api_directive.go index b61df8f6e..e04b4b096 100644 --- a/api_directive.go +++ b/api_directive.go @@ -964,7 +964,7 @@ func createField(idx *Index, fld *dax.Field) error { return errors.Wrapf(err, "creating field options from field: %s", fld.Name) } - if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil { + if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil { return errors.Wrapf(err, "creating field on index: %s", fld.Name) } return nil diff --git a/batch/batch.go b/batch/batch.go index e9e275774..60c8aba7f 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -573,7 +573,7 @@ func (b *Batch) Add(rec Row) error { case int64: b.values[field.Name] = append(b.values[field.Name], val) case []string: - if len(val) == 0 { + if val == nil { continue } rowIDSets, ok := b.rowIDSets[field.Name] @@ -608,7 +608,8 @@ func (b *Batch) Add(rec Row) error { } b.rowIDSets[field.Name] = append(rowIDSets, rowIDs) case []uint64: - if len(val) == 0 { + // if length is 0, that's still a valid, empty, set + if val == nil { continue } rowIDSets, ok := b.rowIDSets[field.Name] @@ -663,6 +664,9 @@ func (b *Batch) Add(rec Row) error { for i, uval := range rec.Clears { field := b.header[i] + if field.Options.Type == featurebase.FieldTypeMutex && uval != nil { + return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex") + } if _, ok := b.clearRowIDs[i]; !ok { b.clearRowIDs[i] = make(map[int]uint64) } @@ -1245,7 +1249,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error { } ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false) - b.log.Debugf("imp-roar field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) + b.log.Debugf("imp-roar field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty)) return errors.Wrapf(ferr, "importing data for %s", field) }) } @@ -1343,6 +1347,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments curShard := ^uint64(0) // impossible sentinel value for shard. var curBM *roaring.Bitmap var clearBM *roaring.Bitmap + var existCurBM *roaring.Bitmap for j := range b.ids { col := b.ids[j] row := nilSentinel @@ -1355,8 +1360,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments if col/shardWidth != curShard { curShard = col / shardWidth + // the API treats "" as standard curBM = frags.GetOrCreate(curShard, field.Name, "") clearBM = clearFrags.GetOrCreate(curShard, field.Name, "") + if opts.TrackExistence { + existCurBM = frags.GetOrCreate(curShard, field.Name, "existence") + } } if row != nilSentinel { // TODO this is super ugly, but we want to avoid setting @@ -1366,6 +1375,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments // the NoStandardView case would be great. if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) { curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + if opts.TrackExistence { + existCurBM.DirectAdd(col % shardWidth) + } } if opts.Type == featurebase.FieldTypeTime { views, err := b.times[j].views(opts.TimeQuantum) @@ -1386,6 +1398,11 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments // we want to make sure that at this point, the "set" // fragments don't contain the bit that we're clearing curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth)) + // don't set the existence bit, probably? i don't actually quite + // understand the higher level semantics here. + if opts.TrackExistence { + existCurBM.DirectRemoveN(col % shardWidth) + } } } } @@ -1404,14 +1421,22 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments opts := field.Options curShard := ^uint64(0) // impossible sentinel value for shard. var curBM *roaring.Bitmap + var existCurBM *roaring.Bitmap for j := range b.ids { col, rowIDs := b.ids[j], rowIDSets[j] - if len(rowIDs) == 0 { - continue - } if col/shardWidth != curShard { curShard = col / shardWidth curBM = frags.GetOrCreate(curShard, fname, "") + if opts.TrackExistence { + existCurBM = frags.GetOrCreate(curShard, fname, "existence") + } + } + if len(rowIDs) == 0 { + // you can validly specify an empty set, which is not the same as a null + if opts.TrackExistence && !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) && rowIDs != nil { + existCurBM.DirectAdd(col % shardWidth) + } + continue } // TODO this is super ugly, but we want to avoid setting // bits on the standard view in the specific case when @@ -1422,6 +1447,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments for _, row := range rowIDs { curBM.DirectAdd(row*shardWidth + (col % shardWidth)) } + if opts.TrackExistence { + existCurBM.DirectAdd(col % shardWidth) + } } if opts.Type == featurebase.FieldTypeTime { views, err := b.times[j].views(opts.TimeQuantum) @@ -1549,6 +1577,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, shard := ids[0] / shardWidth bitmap := frags.GetOrCreate(shard, field.Name, "standard") clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard") + var existBM, existClearBM *roaring.Bitmap + if field.Options.TrackExistence { + existBM = frags.GetOrCreate(shard, field.Name, "existence") + existClearBM = clearFrags.GetOrCreate(shard, field.Name, "existence") + } for i, id := range ids { if i+1 < len(ids) { // we only want the last value set for each id @@ -1561,6 +1594,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, shard = id / shardWidth bitmap = frags.GetOrCreate(shard, field.Name, "standard") clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard") + if field.Options.TrackExistence { + existBM = frags.GetOrCreate(shard, field.Name, "existence") + existClearBM = clearFrags.GetOrCreate(shard, field.Name, "existence") + } } fragmentColumn := id % shardWidth clearBM.Add(fragmentColumn) // Will use this to clear columns. @@ -1568,6 +1605,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, // clearSentinel is used for deletion // so this value should only be added if its not clearSentinel bitmap.Add(row*shardWidth + fragmentColumn) + if field.Options.TrackExistence { + existBM.Add(fragmentColumn) + } + } else if field.Options.TrackExistence { + existClearBM.Add(fragmentColumn) } } } @@ -1596,6 +1638,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, fragmentColumn := recID % shardWidth clearBM.Add(fragmentColumn) + if field.Options.TrackExistence { + existClearBM := clearFrags.GetOrCreate(shard, field.Name, "existence") + + existClearBM.Add(fragmentColumn) + } } } @@ -1618,6 +1665,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, fragmentColumn := recID % shardWidth clearBM.Add(fragmentColumn) + if field.Options.TrackExistence { + exist := frags.GetOrCreate(shard, field.Name, "existence") + exist.Add(fragmentColumn) + } if boolVal { bitmap.Add(trueRowOffset + fragmentColumn) diff --git a/batch/batch_test.go b/batch/batch_test.go index bcc5a65c0..cf214813d 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea Index: idx.Name, Query: "TopN(a1, n=10)", }) + if resp.Err != nil { + t.Fatalf("unexpected error from TopN query: %v", resp.Err) + } + if len(resp.Results) < 1 { + t.Fatalf("expected non-empty result set, got empty results") + } pairsField, ok := resp.Results[0].(*featurebase.PairsField) assert.True(t, ok, "wrong return type: %T", resp.Results[0]) @@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap { Name: "strslice", Options: featurebase.FieldOptions{ - Type: featurebase.FieldTypeSet, - Keys: true, - CacheType: featurebase.CacheTypeRanked, - CacheSize: 100, + Type: featurebase.FieldTypeSet, + Keys: true, + CacheType: featurebase.CacheTypeRanked, + CacheSize: 100, + TrackExistence: true, }, }, }, @@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap pql: "Row(strslice='z')", exp: []uint64{2}, }, + { + pql: "Row(strslice==null)", + exp: []uint64{1}, + }, + { + pql: "Row(strslice!=null)", + exp: []uint64{0, 2, 3, 4}, + }, } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { diff --git a/catcher.go b/catcher.go index 46c8af074..933e6ea52 100644 --- a/catcher.go +++ b/catcher.go @@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) return c.b.Remove(index, field, view, shard, a...) } +func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Removed(index, field, view, shard, a...) +} + func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { defer func() { diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 57ab9515e..0ed57a9c1 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -3302,6 +3302,9 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT return nil, errors.Wrapf(err, "orch: translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { + if ids == nil { + return []string(nil), nil + } keys := make([]string, len(ids)) for i, id := range ids { keys[i] = translations[id] @@ -3311,9 +3314,6 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT } else { datatype = "[]uint64" mapper = func(ids []uint64) (interface{}, error) { - if ids == nil { - ids = []uint64{} - } return ids, nil } } diff --git a/dax/table.go b/dax/table.go index fa37191fc..2f90cdaff 100644 --- a/dax/table.go +++ b/dax/table.go @@ -823,4 +823,5 @@ type FieldOptions struct { TimeQuantum TimeQuantum `json:"time-quantum,omitempty"` TTL time.Duration `json:"ttl,omitempty"` ForeignIndex string `json:"foreign-index,omitempty"` + TrackExistence bool `json:"track-existence"` } diff --git a/delete_test.go b/delete_test.go index 1ebb7bdc4..03047d201 100644 --- a/delete_test.go +++ b/delete_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "sort" + "strings" "testing" "time" @@ -232,8 +233,9 @@ func TestExecutor_DeleteRecords(t *testing.T) { t.Run("DeleteWithBitmapError", func(t *testing.T) { setup(t, require, c) defer tearDown(t, require, c) - _, err := c.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: `Delete(Row(setfield == 1))`}) - if err == nil || err.Error() != `executing: executeDelete: mapping on primary node: bsigroup not found` { + _, err := c.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: `Delete(Row(setfield > 1))`}) + // we don't allow `>` operators on set fields + if err == nil || !strings.Contains(err.Error(), "row call: only support") { t.Fatalf("unexpected error: %s", err) } }) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index dc8c2ad5e..b1fce6e41 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -637,6 +637,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions Keys: o.Keys, ForeignIndex: o.ForeignIndex, NoStandardView: o.NoStandardView, + TrackExistence: o.TrackExistence, } } @@ -973,6 +974,7 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field m.Keys = options.Keys m.ForeignIndex = options.ForeignIndex m.NoStandardView = options.NoStandardView + m.TrackExistence = options.TrackExistence } func (s Serializer) decodeDecimal(d *pb.Decimal, m *pql.Decimal) { diff --git a/executor.go b/executor.go index 6f44885d3..5b804c4c2 100644 --- a/executor.go +++ b/executor.go @@ -1525,7 +1525,22 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string if !opt.Remote { switch c.Name { case "Row": - if c.HasConditionArg() { + // We used to do this by checking for "Condition", but if we allow + // checks against null for non-BSI fields, that's not accurate. We + // can't do this check down in the per-shard stuff, because if we + // did it there, we'd be incrementing the stats once per shard. + // This is redundant with what we do there but shouldn't change + // its behavior, except for possibly the spelling of the diagnostic. + fieldName, err := c.FieldArg() + if err != nil { + return nil, err + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return nil, fmt.Errorf("row call with unknown field %s:%s", index, fieldName) + } + bsig := field.bsiGroup(fieldName) + if bsig != nil { statFn(CounterQueryRowBSITotal) } else { statFn(CounterQueryRowTotal) @@ -4624,6 +4639,27 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return ExtractedIDMatrix{}, newNotFoundError(ErrFieldNotFound, name) } + if field.options.TrackExistence { + existenceRow, err := field.Existing(tx, shard) + if err != nil { + return ExtractedIDMatrix{}, fmt.Errorf("trying to find existence bit: %w", err) + } + // filter out any rows which exist, but aren't in this data... + if existenceRow != nil { + existenceRow = existenceRow.Intersect(colsBitmap) + existing := existenceRow.Columns() + for _, col := range existing { + m[mLookup[col]].Rows[i] = []uint64{} + } + } + } else if field.options.Type == FieldTypeSet || field.options.Type == FieldTypeTime { + // time quantums and sets which don't have track-existence should treat every + // column as a non-null empty set if it has no bits, rather than as a null. + for _, col := range cols { + m[mLookup[col]].Rows[i] = []uint64{} + } + } + switch field.Type() { case FieldTypeSet, FieldTypeMutex: // Handle a set field by listing the rows and then intersecting them with the filter. @@ -4828,15 +4864,82 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return matrix, nil } +// getNullRowShard requests a row representing the elements of this shard +// which are null, that is to say, the elements that exist for the index, +// but not for this particular field. This is computed as row 0 of the +// index's existence field, minus the bsiExistsBit of the existenceView. +// We're using bsiExistsBit for consistency between BSI fields and non-BSI +// fields. In an ideal world, we'd have done this sooner and BSI fields +// would also be using a separate existenceView, perhaps? But we are not +// in that world. +func (e *executor) getNullRowShard(ctx context.Context, tx Tx, idx *Index, fld *Field, existenceView string, shard uint64) (*Row, error) { + // Make sure the index supports existence tracking. + if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", idx.name) + } + // existenceView: the view we've been asked to look in for existence data + // viewExistence: the string "existence", similar to the name "viewStandard", + // denoting the existence view we use for non-BSI fields. BSI fields always + // track existence. + if !fld.options.TrackExistence && existenceView == viewExistence { + return nil, fmt.Errorf("field does not support existence tracking: %s", fld.name) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(idx.name, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + // no existence frag -> no existence bits are set -> there are no + // records in this shard. therefore no records in this shard are + // null. more simply, we don't have to compute the things we want + // to subtract from an empty row to figure out that the result + // will stay empty. + return NewRow(), nil + } else { + var err error + if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + return nil, err + } + } + + var notNull *Row + var err error + + // Retrieve notNull from fragment if it exists. + if frag := e.Holder.fragment(idx.name, fld.name, existenceView, shard); frag != nil { + if notNull, err = frag.notNull(tx); err != nil { + return nil, errors.Wrap(err, "getting fragment not null") + } + } else { + return existenceRow, nil + } + + return existenceRow.Difference(notNull), nil +} + +// getNonNullRowShard requests a row representing the existence of this shard +// which are not null. We trust the existenceView for this field to be +// correct, and not contain bits which don't exist at the index level, so we +// are not checking against the index's existence field. +func (e *executor) getNonNullRowShard(ctx context.Context, tx Tx, idx *Index, fld *Field, existenceView string, shard uint64) (*Row, error) { + // existenceView: the view we've been asked to look in for existence data + // viewExistence: the string "existence", similar to the name "viewStandard", + // denoting the existence view we use for non-BSI fields. BSI fields always + // track existence. + if !fld.options.TrackExistence && existenceView == viewExistence { + return nil, fmt.Errorf("field does not support existence tracking: %s", fld.name) + } + // Retrieve fragment. + frag := e.Holder.fragment(idx.name, fld.name, existenceView, shard) + if frag == nil { + return NewRow(), nil + } + return frag.notNull(tx) +} + func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowShard") defer span.Finish() - // Handle bsiGroup ranges differently. - if c.HasConditionArg() { - return e.executeRowBSIGroupShard(ctx, qcx, index, c, shard) - } - // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -4853,6 +4956,11 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, return nil, newNotFoundError(ErrFieldNotFound, fieldName) } + bsig := f.bsiGroup(fieldName) + if bsig != nil { + return e.executeRowBSIGroupShard(ctx, qcx, f, bsig, c, shard) + } + err = e.validateTimeCallArgs(c, index) if err != nil { return nil, err @@ -4874,15 +4982,31 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, } } - rowID, rowOK, rowErr := c.UintArg(fieldName) - if rowErr != nil { - return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) - } else if !rowOK { - return nil, fmt.Errorf("Row() must specify %v", rowLabel) + isNull, rowID, isEQ, err := c.FieldEquality(fieldName) + if err != nil { + return nil, fmt.Errorf("row call: %v", err) } // Return row if times are not set and standard view exists. timeNotSet := fromTime.IsZero() && toTime.IsZero() + if isNull { + if !timeNotSet { + return nil, errors.New("can't use a time range with a check for/against null") + } + tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) + if err != nil { + return nil, err + } + defer finisher(&err0) + if isEQ { + return e.getNullRowShard(ctx, tx, f.idx, f, viewExistence, shard) + } else { + return e.getNonNullRowShard(ctx, tx, f.idx, f, viewExistence, shard) + } + } + if !isEQ { + return nil, errors.New("only support != for null, not for other values, on set/mutex fields") + } if c.Name == "Row" && timeNotSet && !f.options.NoStandardView { frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { @@ -4941,7 +5065,7 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, } // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { +func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, fld *Field, bsig *bsiGroup, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowBSIGroupShard") defer span.Finish() @@ -4952,23 +5076,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index return nil, errors.New("Row(): too many arguments") } - // Extract conditional. - var fieldName string - var cond *pql.Condition - for k, v := range c.Args { - vv, ok := v.(*pql.Condition) - if !ok { - return nil, fmt.Errorf("Row(): %q: expected condition argument, got %v", k, v) - } - fieldName, cond = k, vv + op, value, err := c.FieldRange(fld.name) + if err != nil { + return nil, err } + viewName := viewBSIGroupPrefix + fld.name - f := e.Holder.Field(index, fieldName) - if f == nil { - return nil, newNotFoundError(ErrFieldNotFound, fieldName) - } - - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: f.idx, Shard: shard}) + tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: fld.idx, Shard: shard}) if err != nil { return nil, err } @@ -4987,50 +5101,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index // NEQ frag.RangeOp // Handle `!= null` and `== null`. - if cond.Op == pql.NEQ && cond.Value == nil { - // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) - if frag == nil { - return NewRow(), nil - } - return frag.notNull(tx) - - } else if cond.Op == pql.EQ && cond.Value == nil { - // Make sure the index supports existence tracking. - idx := e.Holder.Index(index) - if idx == nil { - return nil, newNotFoundError(ErrIndexNotFound, index) - } else if idx.existenceField() == nil { - return nil, errors.Errorf("index does not support existence tracking: %s", index) - } - - var existenceRow *Row - existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) - if existenceFrag == nil { - existenceRow = NewRow() - } else { - if existenceRow, err0 = existenceFrag.row(tx, 0); err0 != nil { - return nil, err0 - } - } - - var notNull *Row - var err error - - // Retrieve notNull from fragment if it exists. - if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { - if notNull, err = frag.notNull(tx); err != nil { - return nil, errors.Wrap(err, "getting fragment not null") - } - } else { - notNull = NewRow() - } - - return existenceRow.Difference(notNull), nil - - } else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT || - cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE { - predicates, err := getCondIntSlice(f, cond) + if op == pql.NEQ && value == nil { + return e.getNonNullRowShard(ctx, tx, fld.idx, fld, viewName, shard) + } else if op == pql.EQ && value == nil { + return e.getNullRowShard(ctx, tx, fld.idx, fld, viewName, shard) + } else if op == pql.BETWEEN || op == pql.BTWN_LT_LT || + op == pql.BTWN_LTE_LT || op == pql.BTWN_LT_LTE { + predicates, err := getCondIntSlice(fld, op, value) if err != nil { return nil, errors.Wrap(err, "getting condition value") } @@ -5044,19 +5121,13 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index // return f.RowBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be shard-specific. - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + frag := e.Holder.fragment(fld.idx.name, fld.name, viewName, shard) if frag == nil { return NewRow(), nil } @@ -5070,40 +5141,34 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index return frag.rangeBetween(tx, bsig.BitDepth, baseValueMin, baseValueMax) } else { - value, err := getScaledInt(f, cond.Value) + value, err := getScaledInt(fld, value) if err != nil { return nil, errors.Wrap(err, "getting scaled integer") } - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - - baseValue, outOfRange := bsig.baseValue(cond.Op, value) - if outOfRange && cond.Op != pql.NEQ { + baseValue, outOfRange := bsig.baseValue(op, value) + if outOfRange && op != pql.NEQ { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + frag := e.Holder.fragment(fld.idx.name, fld.name, viewName, shard) if frag == nil { return NewRow(), nil } // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. - if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || - (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { + if (op == pql.LT && value > bsig.Max) || (op == pql.LTE && value >= bsig.Max) || + (op == pql.GT && value < bsig.Min) || (op == pql.GTE && value <= bsig.Min) { return frag.notNull(tx) } // outOfRange for NEQ should return all not-null. - if outOfRange && cond.Op == pql.NEQ { + if outOfRange && op == pql.NEQ { return frag.notNull(tx) } - return frag.rangeOp(tx, cond.Op, bsig.BitDepth, baseValue) + return frag.rangeOp(tx, op, bsig.BitDepth, baseValue) } } @@ -7689,6 +7754,9 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index return nil, errors.Wrapf(err, "translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { + if ids == nil { + return []string(nil), nil + } keys := make([]string, len(ids)) for i, id := range ids { keys[i] = translations[id] @@ -7698,9 +7766,6 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index } else { datatype = "[]uint64" mapper = func(ids []uint64) (interface{}, error) { - if ids == nil { - ids = []uint64{} - } return ids, nil } } @@ -8687,15 +8752,15 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool return ret, false, err } -// getCondIntSlice looks at the field, the cond op type (which is +// getCondIntSlice looks at the field, the op type (which is // expected to be one of the BETWEEN ops types), and the values in the // conditional and returns a slice of int64 which is scaled for // decimal fields and has the values modulated such that the BETWEEN // op can be treated as being of the form a<=x<=b. -func getCondIntSlice(f *Field, cond *pql.Condition) ([]int64, error) { - val, ok := cond.Value.([]interface{}) +func getCondIntSlice(f *Field, op pql.Token, value interface{}) ([]int64, error) { + val, ok := value.([]interface{}) if !ok { - return nil, errors.Errorf("expected conditional to have []interface{} Value, but got %v of %[1]T", cond.Value) + return nil, errors.Errorf("expected conditional to have []interface{} Value, but got %v of %[1]T", value) } ret := make([]int64, len(val)) @@ -8714,7 +8779,7 @@ func getCondIntSlice(f *Field, cond *pql.Condition) ([]int64, error) { return ret, nil } - switch cond.Op { + switch op { case pql.BTWN_LT_LTE: // a < x <= b ret[0]++ case pql.BTWN_LTE_LT: // a <= x < b @@ -8988,7 +9053,6 @@ func DeleteRowsWithOutKeysFlow(ctx context.Context, columns *roaring.Bitmap, idx }() for _, field := range idx.Fields() { for _, view := range field.views() { - frag := view.Fragment(shard) if frag == nil { continue diff --git a/executor_test.go b/executor_test.go index 29ec09e0a..7c61f6f82 100644 --- a/executor_test.go +++ b/executor_test.go @@ -36,6 +36,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var TempDir = getTempDirString() @@ -3033,6 +3034,14 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } + if _, err := idx.CreateField("idset", "", pilosa.OptFieldTypeSet("none", 0)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("tq", "", pilosa.OptFieldTypeTime("YM", "0")); err != nil { + t.Fatal(err) + } + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: ` Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) @@ -3046,6 +3055,10 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { Set(0, other=1000) Set(0, edge=100) Set(1, edge=-100) + Set(0, idset=3) + Set(1, idset=3) + Clear(0, idset=3) + Set(50, tq=5, 2017-01-02T12:34) `}); err != nil { t.Fatal(err) } @@ -3071,6 +3084,46 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatalf("Query().Row.Columns=%#v, expected %#v", got, exp) } + // time quantum EQ null + _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq == null, from=2010-01-01T00:00)`}) + if err == nil { + t.Fatalf("expected error from invalid time quantum null query") + } + if !strings.Contains(err.Error(), "time range with a check") { + t.Fatalf("unexpected error; expecting can't use time range with a null check, got %v", err) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ + 0, + 1, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ + 50, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset == 3)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + // EQ (single = form) if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(foo = 20)`}); err != nil { t.Fatal(err) @@ -3098,6 +3151,24 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(tq != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: c.Idx(), Query: `Row(idset != 3)`}); err == nil { + t.Fatal("expected error from trying != 3 on a set field") + } else if !strings.Contains(err.Error(), "only support != for null") { + t.Fatalf("expected error about only supporting != for null, got %v", err) + } }) t.Run("LT", func(t *testing.T) { @@ -4993,7 +5064,7 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: 0}, Rows: []interface{}{ - []uint64{}, + []uint64(nil), []string{ "h", "plugh", @@ -5044,11 +5115,11 @@ func TestExecutor_Execute_Extract(t *testing.T) { []uint64{ 0, }, - []string{}, + []string(nil), uint64(0), nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5058,14 +5129,14 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: 3}, Rows: []interface{}{ - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, "plugh", []uint64{ 3, }, - []string{}, + []string(nil), int64(2), pql.NewDecimal(-101, 2), time.Date(2000, time.January, 1, 0, 0, 3, 0, time.UTC), @@ -5075,12 +5146,12 @@ func TestExecutor_Execute_Extract(t *testing.T) { { Column: pilosa.KeyOrID{ID: ShardWidth}, Rows: []interface{}{ - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5093,11 +5164,11 @@ func TestExecutor_Execute_Extract(t *testing.T) { []uint64{ 4, }, - []string{}, + []string(nil), uint64(4), nil, - []uint64{}, - []string{}, + []uint64(nil), + []string(nil), nil, nil, nil, @@ -5107,10 +5178,7 @@ func TestExecutor_Execute_Extract(t *testing.T) { }, }, } - - if !reflect.DeepEqual(expect, resp.Results) { - t.Errorf("expected %v but got %v", expect, resp.Results) - } + require.Equal(t, expect, resp.Results) } func TestExecutor_Execute_Extract_Keyed(t *testing.T) { @@ -5223,7 +5291,7 @@ func TestExecutor_Execute_MaxMemory(t *testing.T) { { Column: pilosa.KeyOrID{ID: ShardWidth}, Rows: []interface{}{ - []uint64{}, + []uint64(nil), }, }, { @@ -5237,10 +5305,7 @@ func TestExecutor_Execute_MaxMemory(t *testing.T) { }, }, } - - if !reflect.DeepEqual(expect, resp.Results) { - t.Errorf("expected %v but got %v", expect, resp.Results) - } + require.Equal(t, expect, resp.Results) } func TestExecutor_Execute_Rows(t *testing.T) { diff --git a/field.go b/field.go index 67b96b40f..f8e3251e3 100644 --- a/field.go +++ b/field.go @@ -376,13 +376,27 @@ func OptFieldTypeBool() FieldOption { } } +// OptFieldTrackExistence exists mostly to allow the +// FieldFromFieldOptions/FieldOptionsFromField round-trip to work. +// If you are actually creating a field, via api.CreateField, +// it will be turned on unconditionally. You can't turn it +// off. +func OptFieldTrackExistence() FieldOption { + return func(fo *FieldOptions) error { + fo.TrackExistence = true + return nil + } +} + // newField returns a new instance of field (without name validation). -func newField(holder *Holder, path, index, name string, opts FieldOption) (*Field, error) { +func newField(holder *Holder, path, index, name string, opts ...FieldOption) (*Field, error) { // Apply functional option. fo := FieldOptions{} - err := opts(&fo) - if err != nil { - return nil, errors.Wrap(err, "applying option") + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } } f := &Field{ @@ -1253,6 +1267,15 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo } else if v { changed = v } + if f.options.TrackExistence { + view, err := f.createViewIfNotExists(viewExistence) + if err != nil { + return changed, errors.Wrap(err, "creating existence view") + } + if _, err := view.setBit(qcx, bsiExistsBit, colID); err != nil { + return changed, errors.Wrap(err, "setting existence on view") + } + } } // Exit early if no timestamp is specified. @@ -1278,6 +1301,9 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo } // ClearBit clears a bit within the field. +// +// This does not, for now, create existence bits for the field, because it +// doesn't create them for the index. func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error) { viewName := viewStandard @@ -1292,8 +1318,20 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error return false, errors.Wrap(err, "clearing on view") } else if v { changed = changed || v + if changed && f.options.TrackExistence && f.options.Type == FieldTypeMutex { + // we also want to try to clear any existence bit + existView, ok := f.viewMap[viewExistence] + if ok { + _, err := existView.clearBit(qcx, 0, colID) + if err != nil { + return false, errors.Wrap(err, "clearing existence bit") + } + } + } } - if len(f.viewMap) == 1 { // assuming no time views + // We used to check length of view map here. Now that we might + // have an existence view, that won't work. + if f.options.Type != FieldTypeTime { // assuming no time views return changed, nil } lastViewNameSize := 0 @@ -1323,30 +1361,6 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error return changed, nil } -// ClearBits clears all bits corresponding to the given record IDs in standard -// or BSI views. It does not delete bits from time quantum views. -func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error { - bsig := f.bsiGroup(f.name) - var v *view - if bsig != nil { - // looks like we're a BSI field? - v = f.view(viewBSIGroupPrefix + f.name) - } else { - v = f.view(viewStandard) - } - // it's fine if we never actually created the view, that means the - // bits are all clear! - if v == nil { - return nil - } - frag := v.Fragment(shard) - if frag == nil { - return nil - } - _, err := frag.ClearRecords(tx, recordIDs) - return err -} - func groupCompare(a, b string, offset int) (lt, eq bool) { if len(a) > offset { a = a[:offset] @@ -1369,6 +1383,12 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { i++ } } + // return the empty list if there weren't any. this could happen + // if we got called because this is a time field, but in fact + // no time views have been created. + if i == 0 { + return me[:0] + } me = me[:i] year := strings.Index(me[0].name, "_") + 4 month := year + 2 @@ -1611,6 +1631,92 @@ func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Ro return view.rangeOp(qcx, op, bsig.BitDepth, baseValue) } +// existenceViewName reports the field we should use row 0 of +// for existence data. For a BSI field (integer, decimal, +// timestamp) this is the single BSI group. For other fields, +// it's viewExistence, which is probably "existence". +func (f *Field) existenceViewName() string { + if len(f.bsiGroups) > 0 { + return f.bsiGroups[0].Name + } + return viewExistence +} + +// MarkExisting sets a range of column IDs as existing. The columnIDs +// are assumed to include the shard offset, but this will also work if +// they are shard-relative, as it's just stripping the offset. +// +// Positions aren't the same as column IDs; this function takes advantage +// of the fact that we're always doing row 0, so we don't have to think +// hard about this. It doesn't overwrite its input because the column IDs +// could be reused by other things. +// +// Note that this is subtly inefficient; if you're tracking existence for +// a field, we're computing the same column ID set to write to the index's +// existence field as we're using for the field's existence view. We don't +// have a good way to coalesce those, yet. (Also, that's not accurate in +// the ImportValue case, where we don't write to the existence view, etc.) +func (f *Field) MarkExisting(tx Tx, columnIDs []uint64, shard uint64) error { + return f.markExistingInView(tx, columnIDs, f.existenceViewName(), shard) +} + +// markExistingInView implements the internals of MarkExisting, but lets +// you use a non-standard view. It's only interesting for the existence field. +func (f *Field) markExistingInView(tx Tx, columnIDs []uint64, viewName string, shard uint64) error { + copyCols := make([]uint64, len(columnIDs)) + for i := range columnIDs { + copyCols[i] = columnIDs[i] % ShardWidth + } + eView, err := f.createViewIfNotExists(viewName) + if err != nil { + return errors.Wrapf(err, "creating view %s", viewName) + } + + eFrag, err := eView.CreateFragmentIfNotExists(shard) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + return eFrag.importPositions(tx, copyCols, nil, map[uint64]struct{}{0: {}}) +} + +// MarkNotExisting is just like MarkExisting, except it is clearing bits, +// so it doesn't have to create the view or fragment if it doesn't exist. +// Because the bits reported to us in the case we wrote this for are likely +// to be sorted by position in the fragment, not by column ID, we sort the +// list after stripping the rows from the positions. +func (f *Field) MarkNotExisting(tx Tx, columnIDs []uint64, shard uint64) error { + viewName := f.existenceViewName() + v := f.view(viewName) + if v == nil { + return nil + } + frag := v.Fragment(shard) + if frag == nil { + return nil + } + copyCols := make([]uint64, len(columnIDs)) + for i := range columnIDs { + copyCols[i] = columnIDs[i] % ShardWidth + } + sort.Slice(copyCols, func(i, j int) bool { return copyCols[i] < copyCols[j] }) + return frag.importPositions(tx, nil, copyCols, map[uint64]struct{}{0: {}}) +} + +// Existing returns the existence row for this field, which +// comes from either the BSI view or the existence view. +func (f *Field) Existing(tx Tx, shard uint64) (*Row, error) { + viewName := f.existenceViewName() + v := f.view(viewName) + if v == nil { + return nil, nil + } + frag := v.Fragment(shard) + if frag == nil { + return nil, nil + } + return frag.row(tx, bsiExistsBit) +} + // Import bulk imports data. func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) { // Determine quantum if timestamps are set. @@ -1622,6 +1728,9 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return errors.New("import clear is not supported with timestamps") } } else { + if f.options.NoStandardView { + return errors.New("can't import data with no timestamps into a field with no standard view") + } // short path: if we don't have any timestamps, we only need // to write to exactly one view, which is always viewStandard, // and *every* bit goes into that view, and we already verified that @@ -1649,7 +1758,33 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, if err != nil { return errors.Wrap(err, "creating fragment") } - + if f.options.TrackExistence { + // if we're clearing a mutex, we do something fancy. otherwise, + // if we're not clearing, we mark the existence bits. either way, + // we then fall on out to the default behavior of importing the + // bits. + switch { + case options.Clear && fieldType == FieldTypeMutex: + // special fancy case; we have to try to clear the bits first, to + // find out WHICH bits we cleared, so we can mark those bits as + // null. + var changed []uint64 + changed, err1 = frag.clearBitsReportingChanges(tx, rowIDs, columnIDs) + if err1 != nil { + return err1 + } + err1 = f.MarkNotExisting(tx, changed, shard) + return err1 + case !options.Clear: + err1 = f.MarkExisting(tx, columnIDs, shard) + if err1 != nil { + return err1 + } + default: + // nothing to do. we'll fall on out of the TrackExistence + // case and go ahead and import those bits naively + } + } err1 = frag.bulkImport(tx, rowIDs, columnIDs, options) return err1 } @@ -1737,6 +1872,16 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return err1 } } + // If we are tracking existence, and don't have NoStandardView, we + // create the existence view. + if f.options.TrackExistence && !f.options.NoStandardView { + // this dance with err1 is so the finisher gets called with + // the right error value if we hit an error + err1 = f.MarkExisting(tx, columnIDs, shard) + if err1 != nil { + return err1 + } + } return nil } @@ -1991,6 +2136,7 @@ type FieldOptions struct { Scale int64 `json:"scale,omitempty"` Keys bool `json:"keys"` NoStandardView bool `json:"noStandardView,omitempty"` + TrackExistence bool `json:"trackExistence,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` CacheType string `json:"cacheType,omitempty"` Type string `json:"type,omitempty"` diff --git a/fragment.go b/fragment.go index 5be779cb5..4a8f1964a 100644 --- a/fragment.go +++ b/fragment.go @@ -1507,32 +1507,60 @@ func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *Import return f.bulkImportStandard(tx, rowIDs, columnIDs, options) } -// rowColumnSet is a sortable set of row and column IDs which -// correspond, allowing us to ensure that we produce values in -// a predictable order -type rowColumnSet struct { - r []uint64 - c []uint64 -} - -func (r rowColumnSet) Len() int { - return len(r.r) -} - -func (r rowColumnSet) Swap(i, j int) { - r.r[i], r.r[j] = r.r[j], r.r[i] - r.c[i], r.c[j] = r.c[j], r.c[i] -} - -// Sort by row ID first, column second, to sort by fragment position -func (r rowColumnSet) Less(i, j int) bool { - if r.r[i] < r.r[j] { - return true +// clearBitsReportingChanges is a special fancy case. For existence-tracking, +// if we're clearing bits in a mutex, *successfully* cleared bits become null +// records, so we have to report, not how many records we cleared, but which +// records specifically became clear. The returned set of bits is the column +// IDs that actually got a bit cleared from them. +// +// This is basically following the logic of bulkImportStandard and +// importPositions, except that it combines them and drops some of the +// no longer needed branches. +func (f *fragment) clearBitsReportingChanges(tx Tx, rowIDs, columnIDs []uint64) ([]uint64, error) { + // Verify that there are an equal number of row ids and column ids. + if len(rowIDs) != len(columnIDs) { + return nil, fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } - if r.r[i] > r.r[j] { - return false + + // rowSet maintains the set of rowIDs present in this import. It allows the + // cache to be updated once per row, instead of once per bit. TODO: consider + // sorting by rowID/columnID first and avoiding the map allocation here. (we + // could reuse rowIDs to store the list of unique row IDs) + rowSet := make(map[uint64]struct{}) + lastRowID := uint64(1 << 63) + + // replace columnIDs with calculated positions to avoid allocation. + prevRow, prevCol := ^uint64(0), ^uint64(0) + next := 0 + for i := 0; i < len(columnIDs); i++ { + rowID, columnID := rowIDs[i], columnIDs[i] + if rowID == prevRow && columnID == prevCol { + continue + } + prevRow, prevCol = rowID, columnID + pos, err := f.pos(rowID, columnID) + if err != nil { + return nil, err + } + columnIDs[next] = pos + next++ + + // Add row to rowSet. + if rowID != lastRowID { + lastRowID = rowID + rowSet[rowID] = struct{}{} + } } - return r.c[i] < r.c[j] + clear := columnIDs[:next] + f.mu.Lock() + defer f.mu.Unlock() + CounterClearingingN.Add(float64(len(clear))) + changed, err := tx.Removed(f.index(), f.field(), f.view(), f.shard, clear...) + if err != nil { + return nil, errors.Wrap(err, "clearing positions") + } + CounterClearedN.Add(float64(len(changed))) + return changed, f.updateCaching(tx, rowSet) } // bulkImportStandard performs a bulk import on a standard fragment. May mutate @@ -1545,11 +1573,6 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options rowSet := make(map[uint64]struct{}) lastRowID := uint64(1 << 63) - // It's possible for the ingest API to have already sorted things in - // the row-first order we want for this import. - if !options.fullySorted { - sort.Sort(rowColumnSet{r: rowIDs, c: columnIDs}) - } // replace columnIDs with calculated positions to avoid allocation. prevRow, prevCol := ^uint64(0), ^uint64(0) next := 0 @@ -1757,35 +1780,6 @@ func (f *fragment) updateCaching(tx Tx, rowSet map[uint64]struct{}) error { return nil } -// sliceDifference removes everything from original that's found in remove, -// updating the slice in place, and returns the compacted slice. The input -// sets should be sorted. -func sliceDifference(original, remove []uint64) []uint64 { - if len(remove) == 0 { - return original - } - rn := 0 - rv := remove[rn] - on := 0 - ov := uint64(0) - n := 0 - - for on, ov = range original { - for rv < ov { - rn++ - if rn >= len(remove) { - return append(original[:n], original[on:]...) - } - rv = remove[rn] - } - if rv != ov { - original[n] = ov - n++ - } - } - return original[:n] -} - // bulkImportMutex performs a bulk import on a fragment while ensuring // mutex restrictions. Because the mutex requirements must be checked // against storage, this method must acquire a write lock on the fragment @@ -1794,16 +1788,10 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *I f.mu.Lock() defer f.mu.Unlock() - // if ingest promises that this is "fully sorted", then we have been - // promised that (1) there's no duplicate entries that need to be - // pruned, (2) the input is sorted by row IDs and then column IDs, - // meaning that we will generate positions in strictly sequential order. - if !options.fullySorted { - p := parallelSlices{cols: columnIDs, rows: rowIDs} - p.fullPrune() - columnIDs = p.cols - rowIDs = p.rows - } + p := parallelSlices{cols: columnIDs, rows: rowIDs} + p.fullPrune() + columnIDs = p.cols + rowIDs = p.rows // create a mask of columns we care about columns := roaring.NewSliceBitmap(columnIDs...) @@ -1822,10 +1810,6 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *I // positions are sorted by columns, but not by absolute // position. we might want them sorted, though. if pos < prev { - if options.fullySorted { - fmt.Printf("HELP! was promised fully sorted input, but previous position was %d, now generated %d\n", - prev, pos) - } unsorted = true } prev = pos diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 7d56ee11f..a8b365dc9 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -4995,69 +4995,6 @@ func TestParallelSlicesFullPrune(t *testing.T) { }) } -func compareSlices(tb testing.TB, name string, s1, s2 []uint64) { - if len(s1) != len(s2) { - tb.Fatalf("slice length mismatch %q: expected %d items %d, got %d items %d", - name, len(s1), s1, len(s2), s2) - } - for i, v := range s1 { - if s2[i] != v { - tb.Fatalf("row mismatch %q: expected item %d to be %d, got %d", - name, i, s1[i], s2[i]) - } - } -} - -type sliceDifferenceTestCase struct { - original, remove, expected []uint64 -} - -func TestSliceDifference(t *testing.T) { - testCases := map[string]sliceDifferenceTestCase{ - "noOverlap": { - original: []uint64{1, 2, 3}, - remove: []uint64{0, 5}, - expected: []uint64{1, 2, 3}, - }, - "before": { - original: []uint64{3, 5, 7}, - remove: []uint64{0, 6}, - expected: []uint64{3, 5, 7}, - }, - "after": { - original: []uint64{3, 5, 7}, - remove: []uint64{8, 10}, - expected: []uint64{3, 5, 7}, - }, - "all": { - original: []uint64{3, 5, 7}, - remove: []uint64{3, 5, 7}, - expected: []uint64{}, - }, - "first": { - original: []uint64{3, 5, 7}, - remove: []uint64{3}, - expected: []uint64{5, 7}, - }, - "last": { - original: []uint64{3, 5, 7}, - remove: []uint64{7}, - expected: []uint64{3, 5}, - }, - "middle": { - original: []uint64{3, 5, 7}, - remove: []uint64{5}, - expected: []uint64{3, 7}, - }, - } - var scratch []uint64 - for name, tc := range testCases { - scratch = append(scratch[:0], tc.original...) - result := sliceDifference(scratch, tc.remove) - compareSlices(t, name, tc.expected, result) - } -} - func TestImportRoaringSingleValued(t *testing.T) { f, _, tx := mustOpenFragment(t) defer f.Clean(t) diff --git a/http_handler_test.go b/http_handler_test.go index 35cacacd6..82f78cdc9 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -475,6 +475,7 @@ func TestGetViewAndDelete(t *testing.T) { // The above sample data should create these views: expectedViewNames := []string{ + "existence", "standard", "standard_2001", "standard_200102", diff --git a/idk/api/source_test.go b/idk/api/source_test.go index 03a32b5d5..83ac70d09 100644 --- a/idk/api/source_test.go +++ b/idk/api/source_test.go @@ -143,7 +143,7 @@ func TestIngest(t *testing.T) { "columns": [ {"column":0,"rows":[[5,7],4,["a"],"h",-13,1.01]}, {"column":1,"rows":[[6],8,["b"],null,null,1.02]}, - {"column":2,"rows":[[],null,[],null,null,null]} + {"column":2,"rows":[null,null,null,null,null,null]} ] } ] @@ -214,7 +214,7 @@ func TestIngest(t *testing.T) { "columns": [ {"column":"a","rows":[[5,7],4,["a"],"h",-13,1.01]}, {"column":"b","rows":[[6],8,["b"],null,null,1.02]}, - {"column":"c","rows":[[],null,[],null,null,null]} + {"column":"c","rows":[null,null,null,null,null,null]} ] } ] diff --git a/index.go b/index.go index ae6d18e84..5887d2a41 100644 --- a/index.go +++ b/index.go @@ -561,7 +561,47 @@ func (i *Index) recalculateCaches() { } } -// CreateField creates a field. +// createNullableField is just like CreateField, except that it allows +// the field to not have TrackExistence enabled. This should be used +// only for existing fields which were already actually created, where +// what we're really doing now is reifying them, so for instance, this +// shows up in api_directive:createField to apply directives, which +// are assumed to correctly reflect the intended behavior. +func (i *Index) createNullableField(name string, requestUserID string, opts ...FieldOption) (*Field, error) { + err := ValidateName(name) + if err != nil { + return nil, errors.Wrap(err, "validating name") + } + + // Grab lock, check for field existing, release lock. We don't want + // to stay holding the lock, but we might care about the ErrFieldExists + // part of this. + err = func() error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ensure field doesn't already exist. + if i.fields[name] != nil { + return newConflictError(ErrFieldExists) + } + return nil + }() + if err != nil { + return nil, err + } + + // Apply and validate functional options. + fo, err := newFieldOptions(opts...) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + return i.createFieldWithOptions(name, requestUserID, fo) +} + +// CreateField creates a field. This interface enforces the setting +// of the TrackExistence flag; if you don't want that, use +// createNullableField, but actually don't. That should be used only +// for applying previously-created fields. func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { @@ -590,7 +630,13 @@ func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOpti if err != nil { return nil, errors.Wrap(err, "applying option") } + fo.TrackExistence = true + return i.createFieldWithOptions(name, requestUserID, fo) +} +// createFieldWithOptions creates a field given a finalized FieldOptions +// structure, instead of functional options. +func (i *Index) createFieldWithOptions(name string, requestUserID string, fo *FieldOptions) (*Field, error) { ts := timestamp() cfm := &CreateFieldMessage{ Index: i.name, @@ -625,6 +671,8 @@ func (i *Index) CreateField(name string, requestUserID string, opts ...FieldOpti } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +// +// Does NOT apply the "default" TrackExistence. func (i *Index) CreateFieldIfNotExists(name string, requestUserID string, opts ...FieldOption) (*Field, error) { err := ValidateName(name) if err != nil { @@ -671,6 +719,8 @@ func (i *Index) CreateFieldIfNotExists(name string, requestUserID string, opts . // function options, taking a *FieldOptions struct. TODO: This should // definintely be refactored so we don't have these virtually equivalent // methods, but I'm puttin this here for now just to see if it works. +// +// Does NOT apply the "default" TrackExistence. func (i *Index) CreateFieldIfNotExistsWithOptions(name string, requestUserID string, opt *FieldOptions) (*Field, error) { err := ValidateName(name) if err != nil { diff --git a/null_test.go b/null_test.go new file mode 100644 index 000000000..8df7d56e3 --- /dev/null +++ b/null_test.go @@ -0,0 +1,620 @@ +// Copyright 2023 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa_test + +import ( + "context" + "fmt" + "strings" + "testing" + + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/batch" + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/test" + "github.com/stretchr/testify/require" +) + +// the new "handle nulls for non-BSI fields" logic has a lot of +// implications, and needs to test things across multiple ways +// of importing data, so we want to have some consistent +// behavior. +// +// We're not testing keyed indexes, because the way index keys +// work doesn't have any relation to the code that's used to +// track existence, but we do want to test both keyed and +// unkeyed fields, and we want to test direct Set/Clear +// operations, SetRow, ClearRow (maybe? for mutexes?), +// Import operations, and the batch code. Note that direct +// use of ImportRoaring, outside of the batch code, makes +// no guarantees; it's up to a user providing roaring bitmaps +// to handle existence views. +// +// This is necessary because it's simply not *possible* to +// detect the distinction between "no bits provided" and +// "an empty set" from the bits written to a view other than +// an existence view. We could, in principle, spend a lot +// of time computing a best-guess that all non-empty sets +// are non-null, but this would be less accurate and much +// more expensive than doing that work on the batch side. + +// setupNullHandlingSchema yields a cluster, and API, which point at an +// index with the given name, containing fields {mu, mk, su, sk, tu, tk} +// which are mutex/set/timequantum fields which are unkeyed/keyed respectively. +func setupNullHandlingSchema(t *testing.T, indexSuffix string) (*test.Cluster, string, *pilosa.API) { + c := test.MustRunCluster(t, 3) + api := c.GetNode(0).API + index := c.Idx(indexSuffix) + // we have six cases we care about; {mutex,set,timequantum} * {keyed, unkeyed} + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "mu", pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "mk", pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "su", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0)) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "sk", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "tu", pilosa.OptFieldTypeTime("YMD", "0")) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "tk", pilosa.OptFieldTypeTime("YMD", "0"), pilosa.OptFieldKeys()) + return c, index, api +} + +var nullHandlingFieldMasks = map[string]int{ + "mu": 1, + "mk": 2, + "su": 4, + "sk": 8, + "tu": 16, + "tk": 32, +} +var nullHandlingExpectedResults = generateNullHandlingExpectedResults() + +type nullSet map[bool][]uint64 + +func (n nullSet) null(v uint64) { + n[true] = append(n[true], v) +} + +func (n nullSet) notNull(v uint64) { + n[false] = append(n[false], v) +} + +func (n nullSet) clone() nullSet { + f := n[false] + t := n[true] + nf := make([]uint64, len(f)) + nt := make([]uint64, len(t)) + copy(nf, f) + copy(nt, t) + return nullSet{false: nf, true: nt} +} + +// trimSlice removes elements of s for which func returns +// true, returning the modified slice. it is destructive. +func trimSlice(s []uint64, fn func(uint64) bool) []uint64 { + n := 0 + for i, v := range s { + if fn(v) { + continue + } else { + s[n] = s[i] + n++ + } + } + return s[:n] +} + +// trim removes values matching f +func (ns nullSet) trim(fn func(uint64) bool) { + ns[false] = trimSlice(ns[false], fn) + ns[true] = trimSlice(ns[true], fn) +} + +// This outlines the expected results of a series of steps. +// We have six fields, keyed/unkeyed sets, mutexes, and time +// quantums. We only ever set value 0/"a" in them. +// +// First, we set those bits for the records 0..63, by treating +// each field as having a bitmask 1/2/4/8/16/32, and setting the +// bits for each field that's a 1 in the record's ID. +// +// Then, we *clear* every bit for every 5th record. I used to do +// every 3rd, but this worked out poorly, because 63%3 == 0. This +// means that all those entries should still exist, but should be +// considered nulls. +// +// Then, we delete everything that has either the 16 or the 32 +// bit (or both) set. This means that all the records which are +// *not* divisible by 5 should no longer be considered null, because +// the records don't exist. The ones which are divisible by 5 are +// still null. +// +// Then, we set a bit in "tk" for record 63. At this point, the +// others should all be null. Remember that, prior to the delete, +// they were all non-null; 63 was the record that had every bit set. +// So we're verifying that, after a delete, recreating the record +// does not show things as not-null because they had been set *prior* +// to the delete. +func generateNullHandlingExpectedResults() map[int]map[string]nullSet { + out := map[int]map[string]nullSet{} + for phase := 0; phase < 4; phase++ { + out[phase] = map[string]nullSet{} + for k := range nullHandlingFieldMasks { + out[phase][k] = nullSet{} + } + } + // We can't combine these two inner loops, because a field is + // only null for records which exist, so record 0, having no + // values set at all, isn't even a null. + phase := 0 + for i := 0; i < (1 << 6); i++ { + madeAny := false + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + out[phase][k].notNull(uint64(i)) + madeAny = true + } + } + if madeAny { + for k, v := range nullHandlingFieldMasks { + if i&v == 0 { + out[phase][k].null(uint64(i)) + } + } + } + } + // phase 1: we clear every bit in records divisible by 5. + // this should not change nullness of sets or time quantums, + // but should change their results. it should make mutexes null. + // we also clear several bits in row 1/"b" -- bits which were + // never set. this should have no impact on anything, so we don't + // reflect it here. + phase = 1 + for i := 0; i < (1 << 6); i++ { + madeAny := false + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[0] == 'm' && (i%5) == 0 { + out[phase][k].null(uint64(i)) + } else { + out[phase][k].notNull(uint64(i)) + } + // even if the only value was the mutex field, the record + // was ever created, so the record still exists even if + // every field is null. + madeAny = true + } + } + if madeAny { + for k, v := range nullHandlingFieldMasks { + if i&v == 0 { + out[phase][k].null(uint64(i)) + } + } + } + } + + // phase 2: delete the &16 and &32 rows. we expect deleted records + // to be neither null nor not null. We delete both the &16 and &32 + // rows because the delete flow is different for keys and no-keys. + phase = 2 + for k := range nullHandlingFieldMasks { + ns := out[1][k].clone() + ns.trim(func(v uint64) bool { return v >= 16 && (v%5) != 0 }) + out[phase][k] = ns + } + + // phase 3: create a record which had previously existed. + // we expect previously-existing fields to now show as null, + // unless we created them again. + phase = 3 + for k, v := range nullHandlingFieldMasks { + ns := out[2][k].clone() + if v == 32 { + ns.notNull(63) + } else { + ns.null(63) + } + out[phase][k] = ns + } + return out +} + +func nullTestQuery(t *testing.T, api *pilosa.API, req *pilosa.QueryRequest) pilosa.QueryResponse { + t.Helper() + resp, err := api.Query(context.Background(), req) + if err != nil { + t.Fatalf("running request: %v", err) + } + if resp.Err != nil { + t.Fatalf("request returned unexpected error: %v", resp.Err) + } + return resp +} + +func nullTestRows(t *testing.T, api *pilosa.API, req *pilosa.QueryRequest) [][]uint64 { + t.Helper() + resp := nullTestQuery(t, api, req) + out := make([][]uint64, 0, len(resp.Results)) + for _, result := range resp.Results { + row, ok := result.(*pilosa.Row) + if !ok { + t.Fatalf("expected row result from query, got %T", result) + } + out = append(out, row.Columns()) + } + return out +} + +func nullTestImport(t *testing.T, api *pilosa.API, req *pilosa.ImportRequest) { + t.Helper() + qcx := api.Txf().NewQcx() + defer qcx.Abort() + err := api.Import(context.Background(), qcx, req) + if err != nil { + t.Fatalf("importing: %v", err) + } + err = qcx.Finish() + if err != nil { + t.Fatalf("committing: %v", err) + } +} + +func nullTestExpectResults(t *testing.T, api *pilosa.API, index string, phase int) { + t.Helper() + for k, expected := range nullHandlingExpectedResults[phase] { + t.Run(fmt.Sprintf("%s-phase-%d", k, phase), func(t *testing.T) { + req := &pilosa.QueryRequest{ + Index: index, + Query: fmt.Sprintf(`Row(%s == null) + Row(%s != null)`, k, k), + } + rows := nullTestRows(t, api, req) + t.Run("true", func(t *testing.T) { + require.Equal(t, expected[true], rows[0]) + }) + t.Run("false", func(t *testing.T) { + require.Equal(t, expected[false], rows[1]) + }) + }) + } +} + +// TestNullHandlingSet only tests the behavior of null values +// in non-BSI fields. It tests this using set/clear. +func TestNullHandlingSet(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + + phase := 0 + var reqs []string + for i := 0; i < (1 << 6); i++ { + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Set(%d, %s="a")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Set(%d, %s=0)`, i, k)) + } + } + } + } + req := &pilosa.QueryRequest{ + Index: index, + Query: strings.Join(reqs, "\n"), + } + resp := nullTestQuery(t, api, req) + for i, v := range resp.Results { + if v != true { + t.Fatalf("result %d (request %s): %#v", i, reqs[i], v) + } + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + reqs = reqs[:0] + for i := 0; i < (1 << 6); i += 5 { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s="a")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s=0)`, i, k)) + } + } + } + // clear a lot of bits that weren't set in the first place. this should have + // no effect on null/not-null state. + for i := 16; i < 48; i++ { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s="b")`, i, k)) + } else { + reqs = append(reqs, fmt.Sprintf(`Clear(%d, %s=1)`, i, k)) + } + } + } + req = &pilosa.QueryRequest{ + Index: index, + Query: strings.Join(reqs, "\n"), + } + resp = nullTestQuery(t, api, req) + for i, v := range resp.Results { + // it's okay to get either true or false, because some of those bits + // wouldn't have existed before, so the clear would fail. that's fine. + if v != true && v != false { + t.Fatalf("result %d (request %s): %#v", i, reqs[i], v) + } + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + req = &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp = nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + + phase = 3 + req = &pilosa.QueryRequest{ + Index: index, + Query: `Set(63, tk="a")`, + } + resp = nullTestQuery(t, api, req) + if len(resp.Results) != 1 || resp.Results[0] != true { + t.Fatalf("expected single true result, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) +} + +// TestNullHandlingImport only tests the behavior of null values +// in non-BSI fields. It tests this using the old Import API to +// import values. +func TestNullHandlingImport(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + + phase := 0 + reqs := map[string]*pilosa.ImportRequest{} + for k := range nullHandlingFieldMasks { + reqs[k] = &pilosa.ImportRequest{ + Index: index, + Field: k, + Shard: ^uint64(0), + } + } + for i := 0; i < (1 << 6); i++ { + for k, v := range nullHandlingFieldMasks { + if i&v != 0 { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "a") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + } + } + } + } + for _, req := range reqs { + nullTestImport(t, api, req) + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + for k := range nullHandlingFieldMasks { + reqs[k].ColumnIDs = reqs[k].ColumnIDs[:0] + if k[1] == 'k' { + reqs[k].RowKeys = reqs[k].RowKeys[:0] + // the import stashed its computed RowIDs in the req, + // remove them. + reqs[k].RowIDs = nil + } else { + reqs[k].RowIDs = reqs[k].RowIDs[:0] + } + reqs[k].Clear = true + reqs[k].Shard = ^uint64(0) + } + + for i := 0; i < (1 << 6); i += 5 { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "a") + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "b") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 0) + } + } + } + // clear a lot of bits that never previously got set, expecting no impact. + for i := 16; i < 48; i++ { + for k := range nullHandlingFieldMasks { + if k[1] == 'k' { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowKeys = append(reqs[k].RowKeys, "b") + } else { + reqs[k].ColumnIDs = append(reqs[k].ColumnIDs, uint64(i)) + reqs[k].RowIDs = append(reqs[k].RowIDs, 1) + } + } + } + for _, req := range reqs { + t.Logf("req: %#v", req) + nullTestImport(t, api, req) + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + // no Delete in Import API, we use the old API for it. + req := &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp := nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + phase = 3 + reqImport := &pilosa.ImportRequest{ + Index: index, + Field: "tk", + ColumnIDs: []uint64{63}, + RowKeys: []string{"a"}, + Shard: ^uint64(0), + } + nullTestImport(t, api, reqImport) + nullTestExpectResults(t, api, index, phase) +} + +func TestNullHandlingBatch(t *testing.T) { + c, index, api := setupNullHandlingSchema(t, "i") + defer c.Close() + ctx := context.Background() + fapi := pilosa.NewOnPremSchema(api) + imp := pilosa.NewOnPremImporter(api) + tbl, err := fapi.TableByName(ctx, dax.TableName(index)) + if err != nil { + t.Fatalf("getting table defs: %v", err) + } + idxInfoBase := pilosa.TableToIndexInfo(tbl) + fields := make([]*pilosa.FieldInfo, 0, len(nullHandlingFieldMasks)) + for k := range nullHandlingFieldMasks { + fields = append(fields, idxInfoBase.Field(k)) + } + + phase := 0 + b, err := batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + var row batch.Row + row.Values = make([]interface{}, len(fields)) + // we don't add the all-null row for ID 0 + for i := 1; i < (1 << 6); i++ { + row.ID = uint64(i) + row.Values = row.Values[:0] + for _, fld := range fields { + if i&nullHandlingFieldMasks[fld.Name] != 0 { + if fld.Name[1] == 'k' { + row.Values = append(row.Values, "a") + } else { + row.Values = append(row.Values, uint64(0)) + } + } else { + row.Values = append(row.Values, nil) + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) + + phase = 1 + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + // delete everything which is a multiple of 5, but + // don't try to delete 0 because this creates an all-null + // record. + for i := 5; i < (1 << 6); i += 5 { + row.ID = uint64(i) + row.Values = row.Values[:0] + row.Clears = map[int]interface{}{} + for i, fld := range fields { + row.Values = append(row.Values, nil) + if fld.Name[0] == 'm' { + // can't specify a particular bit to clear in a mutex + row.Clears[i] = nil + } else { + if fld.Name[1] == 'k' { + row.Clears[i] = "a" + } else { + row.Clears[i] = uint64(0) + } + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + // delete everything which is a multiple of 5 + for i := 16; i < 48; i++ { + row.ID = uint64(i) + row.Values = row.Values[:0] + row.Clears = map[int]interface{}{} + for i, fld := range fields { + row.Values = append(row.Values, nil) + // no way to specify "clear a specific bit", so we don't clear the mutexes. + if fld.Name[0] != 'm' { + if fld.Name[1] == 'k' { + row.Clears[i] = "b" + } else { + row.Clears[i] = uint64(1) + } + } + } + err := b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) + + phase = 2 + // no Delete in batch API, we use the old API for it. + req := &pilosa.QueryRequest{ + Index: index, + Query: `Delete(Row(tk="a")) Delete(Row(tu=0))`, + } + resp := nullTestQuery(t, api, req) + if len(resp.Results) != 2 || resp.Results[0] != true || resp.Results[1] != true { + t.Fatalf("expected two trues, got %#v", resp.Results) + } + nullTestExpectResults(t, api, index, phase) + + phase = 3 + // truncate fields to only have the one field in it + for _, f := range fields { + if f.Name == "tk" { + fields[0] = f + fields = fields[:1] + break + } + } + b, err = batch.NewBatch(imp, 10000, tbl, fields, batch.OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + row.ID = uint64(63) + row.Values = []interface{}{[]string{"a"}} + err = b.Add(row) + if err != nil { + t.Fatalf("adding row: %v", err) + } + if err := b.Import(); err != nil { + t.Fatalf("importing batch: %v", err) + } + nullTestExpectResults(t, api, index, phase) +} diff --git a/pb/private.pb.go b/pb/private.pb.go index dbf939301..6ea3b50de 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -102,6 +102,7 @@ type FieldOptions struct { Max *Decimal `protobuf:"bytes,18,opt,name=Max,proto3" json:"Max,omitempty"` TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"` TTL string `protobuf:"bytes,20,opt,name=TTL,proto3" json:"TTL,omitempty"` + TrackExistence bool `protobuf:"varint,21,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -252,6 +253,13 @@ func (m *FieldOptions) GetTTL() string { return "" } +func (m *FieldOptions) GetTrackExistence() bool { + if m != nil { + return m.TrackExistence + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2951,116 +2959,117 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1742 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x23, 0x49, - 0x15, 0xa6, 0x2f, 0xf1, 0xe5, 0x38, 0xce, 0x38, 0xb5, 0x21, 0xf4, 0x64, 0x87, 0xc8, 0x53, 0xa0, - 0x19, 0x33, 0x12, 0x41, 0x64, 0x1f, 0x16, 0xb1, 0x2f, 0x3b, 0xb1, 0x33, 0x8b, 0xd9, 0x9d, 0xcb, - 0x56, 0x2e, 0x8f, 0xa0, 0x4a, 0xbb, 0x48, 0x5a, 0x69, 0x77, 0x9b, 0xee, 0x76, 0x62, 0xef, 0x03, - 0x12, 0x48, 0x08, 0x5e, 0x78, 0x47, 0x3c, 0xf0, 0x2f, 0xf8, 0x03, 0x3c, 0xf1, 0x82, 0xc4, 0x4f, - 0x40, 0xc3, 0x1f, 0x41, 0x75, 0xaa, 0xaa, 0xbb, 0xec, 0x38, 0xc9, 0x10, 0xf1, 0xd6, 0xe7, 0x3b, - 0xd5, 0xa7, 0xbe, 0x73, 0xa9, 0x53, 0xa7, 0x1b, 0xda, 0x93, 0x2c, 0xba, 0xe2, 0x85, 0xd8, 0x9b, - 0x64, 0x69, 0x91, 0x12, 0x77, 0x72, 0xb6, 0xb3, 0x3e, 0x99, 0x9e, 0xc5, 0x51, 0xa8, 0x10, 0x1a, - 0x41, 0x73, 0x98, 0x8c, 0xc4, 0xec, 0xb5, 0x28, 0x38, 0x21, 0xe0, 0x7f, 0x29, 0xe6, 0x79, 0xe0, - 0x75, 0x9d, 0x5e, 0x83, 0xe1, 0x33, 0x79, 0x06, 0x1b, 0xc7, 0x19, 0x0f, 0x2f, 0x0f, 0x67, 0x51, - 0x5e, 0x88, 0x24, 0x14, 0x81, 0x8f, 0xda, 0x25, 0x94, 0x74, 0xa1, 0x35, 0x10, 0x79, 0x98, 0x45, - 0x93, 0x22, 0x4a, 0x93, 0x60, 0xad, 0xeb, 0xf4, 0x9a, 0xcc, 0x86, 0xe8, 0xdf, 0x3d, 0x58, 0x7f, - 0x15, 0x89, 0x78, 0xf4, 0x16, 0xe5, 0x5c, 0x6e, 0x77, 0x3c, 0x9f, 0x88, 0xa0, 0x81, 0x6b, 0xf1, - 0x99, 0x3c, 0x81, 0x66, 0x9f, 0x87, 0x17, 0x02, 0x15, 0x1e, 0x2a, 0x2a, 0xa0, 0xd4, 0x1e, 0x45, - 0xdf, 0x28, 0x1e, 0x6d, 0x56, 0x01, 0x92, 0xc2, 0x71, 0x34, 0x16, 0x5f, 0x4f, 0x79, 0x52, 0x4c, - 0xc7, 0x86, 0x82, 0x05, 0x91, 0x6d, 0xa8, 0xbd, 0x8d, 0x47, 0xaf, 0xa3, 0x24, 0x68, 0x76, 0x9d, - 0x9e, 0xc7, 0xb4, 0x64, 0x70, 0x3e, 0x0b, 0xa0, 0xc2, 0xf9, 0xac, 0x0c, 0x48, 0x6b, 0x31, 0x20, - 0x6f, 0xd2, 0xa3, 0x82, 0x27, 0x23, 0x9e, 0x8d, 0x4e, 0x23, 0x71, 0x1d, 0xac, 0xab, 0x80, 0x2c, - 0xa2, 0xf2, 0xdd, 0x03, 0x9e, 0x8b, 0xa0, 0x8d, 0x16, 0xf1, 0x99, 0xec, 0x40, 0xe3, 0x20, 0x2a, - 0x06, 0x62, 0x52, 0x5c, 0x04, 0x1b, 0x5d, 0xa7, 0xe7, 0xb3, 0x52, 0x26, 0x5b, 0xb0, 0x76, 0x14, - 0xf2, 0x58, 0x04, 0x8f, 0xf0, 0x05, 0x25, 0x10, 0x0a, 0xeb, 0xaf, 0xd2, 0x4c, 0x44, 0xe7, 0x09, - 0xa6, 0x29, 0xe8, 0xa0, 0x53, 0x0b, 0x18, 0xf9, 0x2e, 0x78, 0xd2, 0xa5, 0xcd, 0xae, 0xd3, 0x6b, - 0xed, 0xb7, 0xf6, 0x26, 0x67, 0x7b, 0x03, 0x11, 0x46, 0x63, 0x1e, 0x33, 0x89, 0xa3, 0x9a, 0xcf, - 0x02, 0xb2, 0x4a, 0xcd, 0x67, 0x92, 0x93, 0x0c, 0xd1, 0x49, 0x12, 0x15, 0xc1, 0x47, 0x68, 0xbd, - 0x94, 0x49, 0x07, 0xbc, 0xe3, 0xe3, 0xaf, 0x82, 0x2d, 0x84, 0xe5, 0x23, 0xa5, 0xb0, 0x31, 0x1c, - 0x4f, 0xd2, 0xac, 0x60, 0x22, 0x9f, 0xa4, 0x49, 0x2e, 0xe4, 0x9a, 0xc3, 0x2c, 0x0b, 0x1c, 0xb5, - 0xe6, 0x30, 0xcb, 0xe8, 0x6f, 0xa0, 0x73, 0x10, 0xa7, 0xe1, 0xe5, 0x80, 0x17, 0x9c, 0x89, 0x5f, - 0x4f, 0x45, 0x5e, 0x48, 0xef, 0x94, 0x03, 0x6a, 0x9d, 0x12, 0x24, 0x8a, 0x15, 0x11, 0xb8, 0x0a, - 0x45, 0x41, 0x46, 0x0e, 0xe3, 0xaa, 0x12, 0x88, 0xcf, 0x18, 0x9d, 0x0b, 0x9e, 0x8d, 0x30, 0xeb, - 0x3e, 0x53, 0x82, 0x44, 0x71, 0x27, 0xac, 0x14, 0x9f, 0x29, 0x81, 0x0e, 0x61, 0xd3, 0xda, 0x5f, - 0xd3, 0xdc, 0x86, 0x1a, 0x4b, 0xaf, 0x87, 0x83, 0x3c, 0x70, 0xba, 0x5e, 0xcf, 0x67, 0x5a, 0xc2, - 0x92, 0x4a, 0xe3, 0xe9, 0x38, 0x91, 0x2a, 0x17, 0x55, 0x15, 0x40, 0x1f, 0xc3, 0x1a, 0xd6, 0x97, - 0xf4, 0xb2, 0x7a, 0x57, 0x3e, 0xd2, 0xdf, 0x3a, 0xd0, 0x7c, 0xcd, 0x67, 0x48, 0x24, 0x27, 0x9f, - 0x42, 0xc3, 0x64, 0x1f, 0x17, 0xb5, 0xf6, 0x3f, 0x96, 0x91, 0x2e, 0x17, 0xec, 0x19, 0xed, 0x61, - 0x52, 0x64, 0x73, 0x56, 0x2e, 0xde, 0xf9, 0x0c, 0xda, 0x0b, 0x2a, 0xb9, 0xd3, 0xa5, 0x98, 0x9b, - 0x78, 0x5e, 0x8a, 0xb9, 0xf4, 0xf2, 0x8a, 0xc7, 0x53, 0x81, 0x51, 0xf2, 0x99, 0x12, 0x7e, 0xea, - 0xfe, 0xc4, 0xa1, 0xa7, 0x40, 0xfa, 0x99, 0xe0, 0x85, 0xc0, 0x4d, 0x5e, 0x8b, 0x3c, 0xe7, 0xe7, - 0xe2, 0xbe, 0x58, 0x7b, 0x76, 0xac, 0xcb, 0xb8, 0xba, 0x56, 0x5c, 0xe9, 0x0b, 0x20, 0x03, 0x11, - 0x8b, 0x42, 0xe8, 0xde, 0x70, 0x87, 0x5d, 0x19, 0x07, 0x4d, 0xe2, 0xfe, 0xc5, 0xe4, 0x29, 0xf8, - 0xb2, 0xd3, 0xe0, 0x6e, 0xad, 0xfd, 0xb6, 0x0c, 0x51, 0xd9, 0x7e, 0x18, 0xaa, 0x30, 0x21, 0x68, - 0x6e, 0xf4, 0xb2, 0x40, 0xae, 0x1e, 0xab, 0x00, 0x69, 0xf6, 0xed, 0x75, 0x22, 0x32, 0x5d, 0x1c, - 0x4a, 0xa0, 0x7f, 0x29, 0x39, 0xa0, 0x57, 0x1f, 0x18, 0x88, 0x85, 0xa2, 0xfb, 0xbe, 0x66, 0xe6, - 0x21, 0xb3, 0x8e, 0x64, 0x66, 0x37, 0xab, 0x55, 0xe4, 0xfc, 0x0f, 0x23, 0xf7, 0x7b, 0x07, 0xc8, - 0xc9, 0x64, 0xb4, 0x4c, 0xee, 0xd5, 0x2a, 0xca, 0xc8, 0xb4, 0xb5, 0xbf, 0x2d, 0xb7, 0xbf, 0xa9, - 0x65, 0xab, 0x9c, 0x7c, 0x0e, 0x35, 0x65, 0x5d, 0x07, 0xf5, 0x51, 0x49, 0x5d, 0xc1, 0x4c, 0xab, - 0xe9, 0x67, 0xd0, 0xb2, 0x60, 0xec, 0x79, 0xaa, 0x57, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0x71, 0x5a, - 0x56, 0x5b, 0x93, 0x29, 0x81, 0x7e, 0x6e, 0x2a, 0xe2, 0xa1, 0x01, 0xa6, 0x21, 0x7c, 0xac, 0x2c, - 0xbc, 0xbc, 0xe2, 0x51, 0xcc, 0xcf, 0xe2, 0xff, 0xa9, 0x68, 0x17, 0x72, 0x15, 0x40, 0x1d, 0xdf, - 0x1d, 0x0e, 0xf4, 0xc1, 0x37, 0x22, 0x9d, 0x42, 0xd5, 0x43, 0xde, 0xf0, 0xb1, 0xd0, 0xd6, 0xf0, - 0xb9, 0x4c, 0xb1, 0x7b, 0x67, 0x8a, 0xa5, 0xff, 0x91, 0xb8, 0x96, 0xb7, 0xa0, 0x87, 0xfe, 0x4b, - 0xe1, 0xee, 0xc4, 0xd3, 0x1f, 0x42, 0xed, 0x28, 0xbc, 0x10, 0x63, 0x4e, 0xbe, 0x07, 0x75, 0x64, - 0x2e, 0x72, 0xdd, 0x06, 0x9a, 0x65, 0x8d, 0x33, 0xa3, 0x91, 0x15, 0xa1, 0xfd, 0x5b, 0x45, 0x73, - 0x61, 0x2b, 0x77, 0xb9, 0xc6, 0x9e, 0x43, 0x5d, 0xf3, 0xc5, 0x2a, 0xbb, 0x71, 0x88, 0x8c, 0x96, - 0x3c, 0x85, 0x1a, 0x7a, 0x97, 0x07, 0x7e, 0x45, 0x04, 0x11, 0xa6, 0x15, 0xf4, 0x10, 0xbc, 0x13, - 0x36, 0x94, 0x95, 0x80, 0xec, 0x0d, 0x0d, 0x2d, 0x49, 0x72, 0x3f, 0x4b, 0xf3, 0x42, 0xc7, 0x1e, - 0x9f, 0x25, 0xf6, 0x2e, 0xcd, 0xd4, 0xc1, 0x6c, 0x33, 0x7c, 0xa6, 0x7f, 0x74, 0xc0, 0x7f, 0x93, - 0x8e, 0x04, 0xd9, 0x00, 0x77, 0x38, 0xd0, 0x46, 0xdc, 0xe1, 0x80, 0x3c, 0x46, 0xfb, 0x3a, 0xde, - 0x75, 0xb9, 0xff, 0x09, 0x1b, 0x32, 0xdc, 0xf3, 0x09, 0x34, 0x87, 0xf9, 0xbb, 0x2c, 0x1a, 0xf3, - 0x6c, 0xae, 0xe7, 0x8d, 0x0a, 0xc0, 0xae, 0x54, 0xc8, 0x92, 0xf6, 0x55, 0xda, 0x51, 0x20, 0x4f, - 0xa1, 0xfe, 0x05, 0x7b, 0xd7, 0x97, 0x26, 0xd7, 0x16, 0x4d, 0x1a, 0x9c, 0x7e, 0x0e, 0x1d, 0xc9, - 0x04, 0xd7, 0x9b, 0xca, 0xda, 0x86, 0x9a, 0xc4, 0x4a, 0x66, 0x5a, 0xaa, 0x36, 0x71, 0xad, 0x4d, - 0xe8, 0x2b, 0x65, 0xe1, 0xf0, 0x4a, 0x24, 0x85, 0x55, 0x9b, 0x28, 0xa3, 0x81, 0x36, 0x53, 0x02, - 0x79, 0xa2, 0xbc, 0xd6, 0xee, 0x35, 0x24, 0x17, 0x29, 0x33, 0x44, 0xe9, 0x1c, 0xc0, 0x30, 0x99, - 0xe6, 0xe5, 0x5a, 0x67, 0xd5, 0x5a, 0x42, 0x4d, 0xf9, 0xe8, 0xee, 0x03, 0x52, 0xaf, 0x10, 0x66, - 0x0a, 0xeb, 0x07, 0x55, 0x61, 0xa9, 0x7c, 0x3e, 0x2a, 0xf3, 0xae, 0xf6, 0xa8, 0xca, 0xeb, 0x02, - 0x5a, 0x16, 0xbe, 0xb2, 0xc6, 0x9e, 0x97, 0xc5, 0xe1, 0x56, 0xc6, 0x10, 0xd1, 0xc6, 0xb4, 0xfa, - 0xee, 0x6e, 0x4c, 0x23, 0xdd, 0x52, 0xee, 0xd8, 0xa9, 0x07, 0x8f, 0x16, 0x0f, 0xbc, 0xb9, 0x65, - 0x97, 0xe1, 0x7b, 0xb6, 0xfa, 0x83, 0x03, 0xed, 0x7e, 0x3c, 0xcd, 0x0b, 0x91, 0x95, 0x31, 0x6d, - 0x6a, 0xa0, 0x4c, 0x6d, 0x05, 0xac, 0xce, 0x2e, 0xd9, 0x85, 0x35, 0x19, 0x71, 0x75, 0xb8, 0xed, - 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5b, 0x26, 0xe8, 0x29, 0x34, 0x0e, 0x8e, 0x86, 0x5f, 0x64, - 0xe9, 0x74, 0xb2, 0xd2, 0x63, 0x33, 0xd6, 0xba, 0xd6, 0x58, 0xdb, 0x51, 0x23, 0x9a, 0xf2, 0x0a, - 0xa7, 0xb2, 0x8e, 0x9a, 0xca, 0x7c, 0x8d, 0xf0, 0x19, 0x3d, 0x82, 0x4d, 0xe5, 0xae, 0xec, 0x38, - 0x0f, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xff, 0x69, 0xf4, - 0x9f, 0x2e, 0x6c, 0x32, 0x91, 0x47, 0xdf, 0x88, 0x61, 0x92, 0x17, 0xd9, 0x34, 0x34, 0x17, 0xc7, - 0xcf, 0xd3, 0x33, 0x9d, 0x0b, 0x8f, 0x29, 0xe1, 0xee, 0x53, 0x42, 0x28, 0xd4, 0xed, 0x26, 0x60, - 0x2f, 0x30, 0x0a, 0xf2, 0x02, 0xea, 0x47, 0xe9, 0x34, 0x0b, 0xcb, 0xca, 0xc7, 0xce, 0xad, 0xf6, - 0x57, 0x0a, 0x66, 0x16, 0x90, 0x2f, 0x81, 0x1c, 0x67, 0x3c, 0xc9, 0x63, 0x2e, 0x29, 0x99, 0xd7, - 0x1a, 0xd5, 0x40, 0x66, 0x69, 0x17, 0x2c, 0xac, 0x78, 0x8d, 0xec, 0xd9, 0x47, 0x38, 0xa8, 0x23, - 0xbf, 0x0d, 0xc3, 0x4f, 0x9f, 0x13, 0xfb, 0x90, 0x7f, 0xba, 0x54, 0xa1, 0x41, 0x0d, 0x5f, 0xd9, - 0xc4, 0xcb, 0xdc, 0x56, 0xb0, 0xc5, 0x75, 0xf4, 0x77, 0x0e, 0xac, 0xdb, 0x6c, 0xee, 0x69, 0x17, - 0x65, 0xfa, 0xdc, 0xfb, 0xe7, 0x3b, 0x93, 0x3e, 0x7f, 0xd5, 0x2c, 0xbd, 0x66, 0xcf, 0x7c, 0x29, - 0x7c, 0xe7, 0x96, 0xe0, 0x3c, 0x88, 0x4e, 0x17, 0x5a, 0xef, 0x78, 0x56, 0x44, 0xd2, 0x98, 0xbe, - 0xa7, 0xd7, 0x98, 0x0d, 0x51, 0x01, 0x8f, 0x6f, 0x14, 0x51, 0x3f, 0x1d, 0x4f, 0x64, 0xb5, 0x3e, - 0xa8, 0x98, 0x64, 0x9b, 0xce, 0xb2, 0x34, 0x33, 0x11, 0x40, 0x81, 0x1e, 0x40, 0xe3, 0x38, 0x9d, - 0xa4, 0x71, 0x7a, 0x3e, 0xbf, 0xa7, 0x65, 0x04, 0x50, 0x57, 0x57, 0x83, 0x6a, 0x51, 0x4d, 0x66, - 0x44, 0xfa, 0x91, 0xac, 0xf7, 0x90, 0xc7, 0xe1, 0x34, 0xe6, 0x85, 0xc0, 0x2f, 0x02, 0x04, 0xbf, - 0x4a, 0xf9, 0x48, 0x75, 0x05, 0x7d, 0xb4, 0xe8, 0x2f, 0x75, 0x01, 0x72, 0x74, 0xc7, 0xba, 0x82, - 0x5e, 0x86, 0xf6, 0xac, 0xa5, 0x24, 0xf2, 0x63, 0x68, 0x59, 0xab, 0xed, 0x01, 0xce, 0x82, 0x99, - 0xbd, 0x86, 0xfe, 0xcd, 0x59, 0x78, 0xe7, 0xc6, 0x9d, 0xab, 0xb7, 0xba, 0x52, 0x41, 0x6a, 0x30, - 0x2d, 0x49, 0xd7, 0x0f, 0x67, 0x61, 0x3c, 0xcd, 0xa5, 0x4a, 0x5f, 0xb8, 0x25, 0x20, 0x5d, 0x97, - 0x1f, 0x7d, 0xe9, 0xd4, 0x0c, 0x37, 0x46, 0x94, 0x9f, 0x87, 0x03, 0xc1, 0x47, 0x71, 0x94, 0x08, - 0xac, 0x17, 0x8f, 0x95, 0x32, 0x79, 0xa1, 0x7a, 0xac, 0x29, 0xf4, 0xad, 0x25, 0xe2, 0xa8, 0x53, - 0x9d, 0x37, 0xa7, 0x04, 0x3a, 0xcb, 0x2a, 0xba, 0x05, 0x44, 0x55, 0xc0, 0xcb, 0xb3, 0x34, 0x33, - 0xb7, 0x2d, 0xed, 0x9b, 0xe6, 0x22, 0xa3, 0x7f, 0xdf, 0x25, 0x5e, 0x45, 0xd6, 0xb5, 0x23, 0x4b, - 0x7f, 0x01, 0x1b, 0x7a, 0xb6, 0x13, 0x19, 0x16, 0xb4, 0x0c, 0x00, 0x13, 0x61, 0x2a, 0xc7, 0x44, - 0xf3, 0x1d, 0x57, 0x01, 0xd2, 0x0e, 0x0e, 0xba, 0xe6, 0x76, 0xd2, 0x12, 0xce, 0x46, 0xd1, 0x79, - 0x22, 0x46, 0x78, 0x63, 0x78, 0x4c, 0x4b, 0xf4, 0x4f, 0x2e, 0x6c, 0xa9, 0xa1, 0x33, 0x39, 0x17, - 0x79, 0x51, 0x6d, 0x83, 0x63, 0x35, 0xf6, 0xff, 0x72, 0xac, 0xc6, 0x1b, 0xe0, 0x19, 0x6c, 0xf4, - 0x63, 0xc1, 0xb3, 0x8a, 0x83, 0xda, 0x68, 0x09, 0x95, 0xe7, 0x06, 0x11, 0x7d, 0x3d, 0xab, 0x21, - 0xd4, 0x86, 0xc8, 0x01, 0x34, 0xb4, 0x6b, 0xa6, 0x21, 0x3e, 0xc3, 0x5b, 0x6a, 0x05, 0x1b, 0x33, - 0xdf, 0xe6, 0xfa, 0xab, 0xd3, 0x88, 0x3b, 0x6f, 0xa1, 0xbd, 0xa0, 0x5a, 0xf1, 0xd5, 0xd9, 0xb3, - 0xbf, 0x3a, 0x5b, 0xfb, 0xc4, 0x1a, 0x97, 0xb5, 0x75, 0xfb, 0x4b, 0xb4, 0x0f, 0xdf, 0x5e, 0x45, - 0x20, 0x27, 0x2f, 0xc0, 0x93, 0x44, 0xd5, 0x30, 0x1c, 0xdc, 0x46, 0x94, 0xc9, 0x45, 0xf4, 0xaf, - 0x8e, 0x0e, 0xaa, 0xd0, 0x7a, 0xf3, 0xf7, 0xe0, 0x13, 0xdb, 0xc8, 0xd3, 0xd2, 0xc8, 0xd2, 0xb2, - 0xbd, 0xd2, 0x51, 0xb9, 0x7a, 0xe7, 0x6b, 0x68, 0xac, 0x72, 0xcf, 0x57, 0xee, 0xfd, 0x68, 0xd1, - 0xbd, 0xc7, 0xb7, 0x31, 0xcb, 0x6d, 0x2f, 0xf7, 0x60, 0x5b, 0xdd, 0xa6, 0x03, 0x5e, 0xf0, 0x5f, - 0x65, 0x7c, 0x2c, 0xee, 0xbc, 0x52, 0x0f, 0x3a, 0xff, 0x78, 0xbf, 0xeb, 0xfc, 0xeb, 0xfd, 0xae, - 0xf3, 0xef, 0xf7, 0xbb, 0xce, 0x9f, 0xff, 0xb3, 0xfb, 0xad, 0xb3, 0x1a, 0xfe, 0x76, 0xfb, 0xe4, - 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa4, 0xc1, 0x04, 0xc8, 0x99, 0x13, 0x00, 0x00, + // 1750 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6e, 0x24, 0x49, + 0x11, 0xa6, 0x7e, 0xdc, 0x3f, 0xd1, 0x6e, 0x4f, 0x3b, 0xd7, 0x6b, 0x6a, 0xbc, 0x83, 0xd5, 0x93, + 0xa0, 0x99, 0x66, 0x24, 0x8c, 0xf0, 0x1e, 0x16, 0xb1, 0x97, 0x1d, 0x77, 0x7b, 0x96, 0x66, 0x77, + 0x7e, 0x36, 0xed, 0x99, 0x23, 0x28, 0x5d, 0x9d, 0xd8, 0xa5, 0xa9, 0xae, 0x6a, 0xaa, 0xaa, 0x3d, + 0xdd, 0x7b, 0x40, 0x02, 0x09, 0xc1, 0x85, 0x3b, 0xe2, 0xc0, 0x5b, 0xf0, 0x0e, 0x5c, 0x90, 0x78, + 0x04, 0x34, 0xdc, 0x78, 0x0a, 0x94, 0x91, 0x99, 0x55, 0xd9, 0xed, 0xb2, 0xbd, 0x58, 0x7b, 0xab, + 0xf8, 0x22, 0x2b, 0xf2, 0x8b, 0x9f, 0x8c, 0x8c, 0x2a, 0xe8, 0xce, 0xb2, 0xe8, 0x92, 0x17, 0xe2, + 0x60, 0x96, 0xa5, 0x45, 0x4a, 0xdc, 0xd9, 0xd9, 0xde, 0xe6, 0x6c, 0x7e, 0x16, 0x47, 0xa1, 0x42, + 0x68, 0x04, 0xed, 0x71, 0x32, 0x11, 0x8b, 0xe7, 0xa2, 0xe0, 0x84, 0x80, 0xff, 0x85, 0x58, 0xe6, + 0x81, 0xd7, 0x77, 0x06, 0x2d, 0x86, 0xcf, 0xe4, 0x11, 0x6c, 0x9d, 0x66, 0x3c, 0x7c, 0x7b, 0xbc, + 0x88, 0xf2, 0x42, 0x24, 0xa1, 0x08, 0x7c, 0xd4, 0xae, 0xa1, 0xa4, 0x0f, 0x9d, 0x91, 0xc8, 0xc3, + 0x2c, 0x9a, 0x15, 0x51, 0x9a, 0x04, 0x1b, 0x7d, 0x67, 0xd0, 0x66, 0x36, 0x44, 0xff, 0xeb, 0xc1, + 0xe6, 0xb3, 0x48, 0xc4, 0x93, 0x97, 0x28, 0xe7, 0x72, 0xbb, 0xd3, 0xe5, 0x4c, 0x04, 0x2d, 0x5c, + 0x8b, 0xcf, 0xe4, 0x01, 0xb4, 0x87, 0x3c, 0xbc, 0x10, 0xa8, 0xf0, 0x50, 0x51, 0x01, 0xa5, 0xf6, + 0x24, 0xfa, 0x5a, 0xf1, 0xe8, 0xb2, 0x0a, 0x90, 0x14, 0x4e, 0xa3, 0xa9, 0xf8, 0x6a, 0xce, 0x93, + 0x62, 0x3e, 0x35, 0x14, 0x2c, 0x88, 0xec, 0x42, 0xe3, 0x65, 0x3c, 0x79, 0x1e, 0x25, 0x41, 0xbb, + 0xef, 0x0c, 0x3c, 0xa6, 0x25, 0x83, 0xf3, 0x45, 0x00, 0x15, 0xce, 0x17, 0x65, 0x40, 0x3a, 0xab, + 0x01, 0x79, 0x91, 0x9e, 0x14, 0x3c, 0x99, 0xf0, 0x6c, 0xf2, 0x26, 0x12, 0xef, 0x82, 0x4d, 0x15, + 0x90, 0x55, 0x54, 0xbe, 0x7b, 0xc4, 0x73, 0x11, 0x74, 0xd1, 0x22, 0x3e, 0x93, 0x3d, 0x68, 0x1d, + 0x45, 0xc5, 0x48, 0xcc, 0x8a, 0x8b, 0x60, 0xab, 0xef, 0x0c, 0x7c, 0x56, 0xca, 0x64, 0x07, 0x36, + 0x4e, 0x42, 0x1e, 0x8b, 0xe0, 0x1e, 0xbe, 0xa0, 0x04, 0x42, 0x61, 0xf3, 0x59, 0x9a, 0x89, 0xe8, + 0x3c, 0xc1, 0x34, 0x05, 0x3d, 0x74, 0x6a, 0x05, 0x23, 0xdf, 0x03, 0x4f, 0xba, 0xb4, 0xdd, 0x77, + 0x06, 0x9d, 0xc3, 0xce, 0xc1, 0xec, 0xec, 0x60, 0x24, 0xc2, 0x68, 0xca, 0x63, 0x26, 0x71, 0x54, + 0xf3, 0x45, 0x40, 0xea, 0xd4, 0x7c, 0x21, 0x39, 0xc9, 0x10, 0xbd, 0x4e, 0xa2, 0x22, 0xf8, 0x00, + 0xad, 0x97, 0x32, 0xe9, 0x81, 0x77, 0x7a, 0xfa, 0x65, 0xb0, 0x83, 0xb0, 0x7c, 0xac, 0x29, 0x87, + 0x0f, 0xeb, 0xca, 0x81, 0x52, 0xd8, 0x1a, 0x4f, 0x67, 0x69, 0x56, 0x30, 0x91, 0xcf, 0xd2, 0x24, + 0x17, 0xd2, 0xd6, 0x71, 0x96, 0x05, 0x8e, 0xb2, 0x75, 0x9c, 0x65, 0xf4, 0xb7, 0xd0, 0x3b, 0x8a, + 0xd3, 0xf0, 0xed, 0x88, 0x17, 0x9c, 0x89, 0xdf, 0xcc, 0x45, 0x5e, 0xc8, 0x28, 0x28, 0x47, 0xd5, + 0x3a, 0x25, 0x48, 0x14, 0x2b, 0x27, 0x70, 0x15, 0x8a, 0x82, 0x8c, 0x30, 0xc6, 0x5f, 0x25, 0x1a, + 0x9f, 0x31, 0x8a, 0x17, 0x3c, 0x9b, 0x60, 0x75, 0xf8, 0x4c, 0x09, 0x12, 0xc5, 0x9d, 0xb0, 0xa2, + 0x7c, 0xa6, 0x04, 0x3a, 0x86, 0x6d, 0x6b, 0x7f, 0x4d, 0x73, 0x17, 0x1a, 0x2c, 0x7d, 0x37, 0x1e, + 0xe5, 0x81, 0xd3, 0xf7, 0x06, 0x3e, 0xd3, 0x12, 0x96, 0x5e, 0x1a, 0xcf, 0xa7, 0x89, 0x54, 0xb9, + 0xa8, 0xaa, 0x00, 0x7a, 0x1f, 0x36, 0xb0, 0x0e, 0xa5, 0x97, 0xd5, 0xbb, 0xf2, 0x91, 0xfe, 0xce, + 0x81, 0xf6, 0x73, 0xbe, 0x40, 0x22, 0x39, 0xf9, 0x04, 0x5a, 0xa6, 0x4a, 0x70, 0x51, 0xe7, 0xf0, + 0x23, 0x99, 0x91, 0x72, 0xc1, 0x81, 0xd1, 0x1e, 0x27, 0x45, 0xb6, 0x64, 0xe5, 0xe2, 0xbd, 0x4f, + 0xa1, 0xbb, 0xa2, 0x92, 0x3b, 0xbd, 0x15, 0x4b, 0x13, 0xcf, 0xb7, 0x62, 0x29, 0xbd, 0xbc, 0xe4, + 0xf1, 0x5c, 0x60, 0x94, 0x7c, 0xa6, 0x84, 0x9f, 0xb9, 0x3f, 0x75, 0xe8, 0x1b, 0x20, 0xc3, 0x4c, + 0xf0, 0x42, 0xe0, 0x26, 0xcf, 0x45, 0x9e, 0xf3, 0x73, 0x71, 0x5b, 0xac, 0x3d, 0x3b, 0xd6, 0x65, + 0x5c, 0x5d, 0x2b, 0xae, 0xf4, 0x09, 0x90, 0x91, 0x88, 0x45, 0x21, 0x74, 0x0f, 0xb9, 0xc1, 0xae, + 0x8c, 0x83, 0x26, 0x71, 0xfb, 0x62, 0xf2, 0x10, 0x7c, 0xd9, 0x91, 0x70, 0xb7, 0xce, 0x61, 0x57, + 0x86, 0xa8, 0x6c, 0x53, 0x0c, 0x55, 0x98, 0x10, 0x34, 0x37, 0x79, 0x5a, 0x20, 0x57, 0x8f, 0x55, + 0x80, 0x34, 0xfb, 0xf2, 0x5d, 0x22, 0x32, 0x5d, 0x1c, 0x4a, 0xa0, 0x7f, 0x2d, 0x39, 0xa0, 0x57, + 0xdf, 0x30, 0x10, 0x2b, 0x45, 0xf7, 0x03, 0xcd, 0xcc, 0x43, 0x66, 0x3d, 0xc9, 0xcc, 0x6e, 0x6a, + 0x75, 0xe4, 0xfc, 0x6f, 0x46, 0xee, 0x0f, 0x0e, 0x90, 0xd7, 0xb3, 0xc9, 0x3a, 0xb9, 0x67, 0x75, + 0x94, 0x91, 0x69, 0xe7, 0x70, 0x57, 0x6e, 0x7f, 0x55, 0xcb, 0xea, 0x9c, 0x7c, 0x0c, 0x0d, 0x65, + 0x5d, 0x07, 0xf5, 0x5e, 0x49, 0x5d, 0xc1, 0x4c, 0xab, 0xe9, 0xa7, 0xd0, 0xb1, 0x60, 0xec, 0x8d, + 0xaa, 0xa7, 0xab, 0xe8, 0x68, 0x49, 0x3a, 0xf1, 0xa6, 0xac, 0xb6, 0x36, 0x53, 0x02, 0xfd, 0xcc, + 0x54, 0xc4, 0x5d, 0x03, 0x4c, 0x43, 0xf8, 0x48, 0x59, 0x78, 0x7a, 0xc9, 0xa3, 0x98, 0x9f, 0xc5, + 0xff, 0x57, 0xd1, 0xae, 0xe4, 0x2a, 0x80, 0x26, 0xbe, 0x3b, 0x1e, 0xe9, 0x83, 0x6f, 0x44, 0x3a, + 0x87, 0xaa, 0x87, 0xbc, 0xe0, 0x53, 0xa1, 0xad, 0xe1, 0x73, 0x99, 0x62, 0xf7, 0xc6, 0x14, 0x4b, + 0xff, 0x23, 0xf1, 0x4e, 0xde, 0x96, 0x1e, 0xfa, 0x2f, 0x85, 0x9b, 0x13, 0x4f, 0x7f, 0x04, 0x8d, + 0x93, 0xf0, 0x42, 0x4c, 0x39, 0xf9, 0x3e, 0x34, 0x91, 0xb9, 0xc8, 0x75, 0x1b, 0x68, 0x97, 0x35, + 0xce, 0x8c, 0x46, 0x56, 0x84, 0xf6, 0xaf, 0x8e, 0xe6, 0xca, 0x56, 0xee, 0x7a, 0x8d, 0x3d, 0x86, + 0xa6, 0xe6, 0x8b, 0x55, 0x76, 0xe5, 0x10, 0x19, 0x2d, 0x79, 0x08, 0x0d, 0xf4, 0x2e, 0x0f, 0xfc, + 0x8a, 0x08, 0x22, 0x4c, 0x2b, 0xe8, 0x31, 0x78, 0xaf, 0xd9, 0x58, 0x56, 0x02, 0xb2, 0x37, 0x34, + 0xb4, 0x24, 0xc9, 0xfd, 0x3c, 0xcd, 0x0b, 0x1d, 0x7b, 0x7c, 0x96, 0xd8, 0xab, 0x34, 0x53, 0x07, + 0xb3, 0xcb, 0xf0, 0x99, 0xfe, 0xc9, 0x01, 0xff, 0x45, 0x3a, 0x11, 0x64, 0x0b, 0xdc, 0xf1, 0x48, + 0x1b, 0x71, 0xc7, 0x23, 0x72, 0x1f, 0xed, 0xeb, 0x78, 0x37, 0xe5, 0xfe, 0xaf, 0xd9, 0x98, 0xe1, + 0x9e, 0x0f, 0xa0, 0x3d, 0xce, 0x5f, 0x65, 0xd1, 0x94, 0x67, 0x4b, 0x3d, 0x97, 0x54, 0x00, 0x76, + 0xa5, 0x42, 0x96, 0xb4, 0xaf, 0xd2, 0x8e, 0x02, 0x79, 0x08, 0xcd, 0xcf, 0xd9, 0xab, 0xa1, 0x34, + 0xb9, 0xb1, 0x6a, 0xd2, 0xe0, 0xf4, 0x33, 0xe8, 0x49, 0x26, 0xb8, 0xde, 0x54, 0xd6, 0x2e, 0x34, + 0x24, 0x56, 0x32, 0xd3, 0x52, 0xb5, 0x89, 0x6b, 0x6d, 0x42, 0x9f, 0x29, 0x0b, 0xc7, 0x97, 0x22, + 0x29, 0xac, 0xda, 0x44, 0x19, 0x0d, 0x74, 0x99, 0x12, 0xc8, 0x03, 0xe5, 0xb5, 0x76, 0xaf, 0x25, + 0xb9, 0x48, 0x99, 0x21, 0x4a, 0x97, 0x00, 0x86, 0xc9, 0x3c, 0x2f, 0xd7, 0x3a, 0x75, 0x6b, 0x09, + 0x35, 0xe5, 0xa3, 0xbb, 0x0f, 0x48, 0xbd, 0x42, 0x98, 0x29, 0xac, 0x1f, 0x56, 0x85, 0xa5, 0xf2, + 0x79, 0xaf, 0xcc, 0xbb, 0xda, 0xa3, 0x2a, 0xaf, 0x0b, 0xe8, 0x58, 0x78, 0x6d, 0x8d, 0x3d, 0x2e, + 0x8b, 0xc3, 0xad, 0x8c, 0x21, 0xa2, 0x8d, 0x69, 0xf5, 0xcd, 0xdd, 0x98, 0x46, 0xba, 0xa5, 0xdc, + 0xb0, 0xd3, 0x00, 0xee, 0xad, 0x1e, 0x78, 0x73, 0xcb, 0xae, 0xc3, 0xb7, 0x6c, 0xf5, 0x47, 0x07, + 0xba, 0xc3, 0x78, 0x9e, 0x17, 0x22, 0x2b, 0x63, 0xda, 0xd6, 0x40, 0x99, 0xda, 0x0a, 0xa8, 0xcf, + 0x2e, 0xd9, 0x87, 0x0d, 0x19, 0x71, 0x75, 0xb8, 0xed, 0x44, 0x28, 0xd8, 0xca, 0x84, 0x7f, 0x5d, + 0x26, 0xe8, 0x1b, 0x68, 0x1d, 0x9d, 0x8c, 0x3f, 0xcf, 0xd2, 0xf9, 0xac, 0xd6, 0x63, 0x33, 0xfe, + 0xba, 0xd6, 0xf8, 0xdb, 0x53, 0xa3, 0x9c, 0xf2, 0x0a, 0xa7, 0xb7, 0x9e, 0x9a, 0xde, 0x7c, 0x8d, + 0xf0, 0x05, 0x3d, 0x81, 0x6d, 0xe5, 0xae, 0xec, 0x38, 0x77, 0x69, 0x8b, 0x66, 0x6e, 0xf2, 0xaa, + 0xb9, 0x49, 0x1a, 0x55, 0x5d, 0xf7, 0xdb, 0x34, 0xfa, 0x4f, 0x17, 0xb6, 0x99, 0xc8, 0xa3, 0xaf, + 0xc5, 0x38, 0xc9, 0x8b, 0x6c, 0x1e, 0x9a, 0x8b, 0xe3, 0x17, 0xe9, 0x99, 0xce, 0x85, 0xc7, 0x94, + 0x70, 0xf3, 0x29, 0x21, 0x14, 0x9a, 0x76, 0x13, 0xb0, 0x17, 0x18, 0x05, 0x79, 0x02, 0xcd, 0x93, + 0x74, 0x9e, 0x85, 0x65, 0xe5, 0x63, 0xe7, 0x56, 0xfb, 0x2b, 0x05, 0x33, 0x0b, 0xc8, 0x17, 0x40, + 0x4e, 0x33, 0x9e, 0xe4, 0x31, 0x97, 0x94, 0xcc, 0x6b, 0xad, 0x6a, 0x20, 0xb3, 0xb4, 0x2b, 0x16, + 0x6a, 0x5e, 0x23, 0x07, 0xf6, 0x11, 0x0e, 0x9a, 0xc8, 0x6f, 0xcb, 0xf0, 0xd3, 0xe7, 0xc4, 0x3e, + 0xe4, 0x9f, 0xac, 0x55, 0x68, 0xd0, 0xc0, 0x57, 0xb6, 0xf1, 0x32, 0xb7, 0x15, 0x6c, 0x75, 0x1d, + 0xfd, 0xbd, 0x03, 0x9b, 0x36, 0x9b, 0x5b, 0xda, 0x45, 0x99, 0x3e, 0xf7, 0xf6, 0xf9, 0xce, 0xa4, + 0xcf, 0xaf, 0x9b, 0xa5, 0x37, 0xec, 0x99, 0x2f, 0x85, 0xef, 0x5e, 0x13, 0x9c, 0x3b, 0xd1, 0xe9, + 0x43, 0xe7, 0x15, 0xcf, 0x8a, 0x48, 0x1a, 0xd3, 0xf7, 0xf4, 0x06, 0xb3, 0x21, 0x2a, 0xe0, 0xfe, + 0x95, 0x22, 0x1a, 0xa6, 0xd3, 0x99, 0xac, 0xd6, 0x3b, 0x15, 0x93, 0x6c, 0xd3, 0x59, 0x96, 0x66, + 0x26, 0x02, 0x28, 0xd0, 0x23, 0x68, 0x9d, 0xa6, 0xb3, 0x34, 0x4e, 0xcf, 0x97, 0xb7, 0xb4, 0x8c, + 0x00, 0x9a, 0xea, 0x6a, 0x50, 0x2d, 0xaa, 0xcd, 0x8c, 0x48, 0x3f, 0x90, 0xf5, 0x1e, 0xf2, 0x38, + 0x9c, 0xc7, 0xbc, 0x10, 0xf8, 0x45, 0x80, 0xe0, 0x97, 0x29, 0x9f, 0xa8, 0xae, 0xa0, 0x8f, 0x16, + 0xfd, 0x95, 0x2e, 0x40, 0x8e, 0xee, 0x58, 0x57, 0xd0, 0xd3, 0xd0, 0x9e, 0xb5, 0x94, 0x44, 0x7e, + 0x02, 0x1d, 0x6b, 0xb5, 0x3d, 0xc0, 0x59, 0x30, 0xb3, 0xd7, 0xd0, 0xbf, 0x3b, 0x2b, 0xef, 0x5c, + 0xb9, 0x73, 0xf5, 0x56, 0x97, 0x2a, 0x48, 0x2d, 0xa6, 0x25, 0xe9, 0xfa, 0xf1, 0x22, 0x8c, 0xe7, + 0xb9, 0x54, 0xe9, 0x0b, 0xb7, 0x04, 0xa4, 0xeb, 0xf2, 0xe3, 0x30, 0x9d, 0x9b, 0xe1, 0xc6, 0x88, + 0xf2, 0x33, 0x72, 0x24, 0xf8, 0x24, 0x8e, 0x12, 0x81, 0xf5, 0xe2, 0xb1, 0x52, 0x26, 0x4f, 0x54, + 0x8f, 0x35, 0x85, 0xbe, 0xb3, 0x46, 0x1c, 0x75, 0xaa, 0xf3, 0xe6, 0x94, 0x40, 0x6f, 0x5d, 0x45, + 0x77, 0x80, 0xa8, 0x0a, 0x78, 0x7a, 0x96, 0x66, 0xe6, 0xb6, 0xa5, 0x43, 0xd3, 0x5c, 0x64, 0xf4, + 0x6f, 0xbb, 0xc4, 0xab, 0xc8, 0xba, 0x76, 0x64, 0xe9, 0x2f, 0x61, 0x4b, 0xcf, 0x76, 0x22, 0xc3, + 0x82, 0x96, 0x01, 0x60, 0x22, 0x4c, 0xe5, 0x98, 0x68, 0xbe, 0xe3, 0x2a, 0x40, 0xda, 0xc1, 0x41, + 0xd7, 0xdc, 0x4e, 0x5a, 0xc2, 0xd9, 0x28, 0x3a, 0x4f, 0xc4, 0x04, 0x6f, 0x0c, 0x8f, 0x69, 0x89, + 0xfe, 0xd9, 0x85, 0x1d, 0x35, 0x74, 0x26, 0xe7, 0x22, 0x2f, 0xaa, 0x6d, 0x70, 0xac, 0xc6, 0xfe, + 0x5f, 0x8e, 0xd5, 0x78, 0x03, 0x3c, 0x82, 0xad, 0x61, 0x2c, 0x78, 0x56, 0x71, 0x50, 0x1b, 0xad, + 0xa1, 0xf2, 0xdc, 0x20, 0xa2, 0xaf, 0x67, 0x35, 0x84, 0xda, 0x10, 0x39, 0x82, 0x96, 0x76, 0xcd, + 0x34, 0xc4, 0x47, 0x78, 0x4b, 0xd5, 0xb0, 0x31, 0xf3, 0x6d, 0xae, 0xbf, 0x3a, 0x8d, 0xb8, 0xf7, + 0x12, 0xba, 0x2b, 0xaa, 0x9a, 0xaf, 0xce, 0x81, 0xfd, 0xd5, 0xd9, 0x39, 0x24, 0xd6, 0xb8, 0xac, + 0xad, 0xdb, 0x5f, 0xa2, 0x43, 0xf8, 0xb0, 0x8e, 0x40, 0x4e, 0x9e, 0x80, 0x27, 0x89, 0xaa, 0x61, + 0x38, 0xb8, 0x8e, 0x28, 0x93, 0x8b, 0xe8, 0xdf, 0x1c, 0x1d, 0x54, 0xa1, 0xf5, 0xe6, 0xef, 0xc1, + 0xc7, 0xb6, 0x91, 0x87, 0xa5, 0x91, 0xb5, 0x65, 0x07, 0xa5, 0xa3, 0x72, 0xf5, 0xde, 0x57, 0xd0, + 0xaa, 0x73, 0xcf, 0x57, 0xee, 0xfd, 0x78, 0xd5, 0xbd, 0xfb, 0xd7, 0x31, 0xcb, 0x6d, 0x2f, 0x0f, + 0x60, 0x57, 0xdd, 0xa6, 0x23, 0x5e, 0xf0, 0x5f, 0x67, 0x7c, 0x2a, 0x6e, 0xbc, 0x52, 0x8f, 0x7a, + 0xff, 0x78, 0xbf, 0xef, 0xfc, 0xeb, 0xfd, 0xbe, 0xf3, 0xef, 0xf7, 0xfb, 0xce, 0x5f, 0xfe, 0xb3, + 0xff, 0x9d, 0xb3, 0x06, 0xfe, 0x9e, 0xfb, 0xf8, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x0d, + 0x5d, 0xdb, 0xc1, 0x13, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3141,6 +3150,18 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.TrackExistence { + i-- + if m.TrackExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa8 + } if len(m.TTL) > 0 { i -= len(m.TTL) copy(dAtA[i:], m.TTL) @@ -5636,6 +5657,9 @@ func (m *FieldOptions) Size() (n int) { if l > 0 { n += 2 + l + sovPrivate(uint64(l)) } + if m.TrackExistence { + n += 3 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7255,6 +7279,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } m.TTL = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 21: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TrackExistence", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.TrackExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/pb/private.proto b/pb/private.proto index a82b676ef..6822a872d 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -27,6 +27,7 @@ message FieldOptions { Decimal Max = 18; string TimeUnit = 19; string TTL = 20; + bool TrackExistence = 21; } message ImportResponse { diff --git a/pql/ast.go b/pql/ast.go index e72de72d2..362d63f67 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -1009,6 +1009,67 @@ func (c *Call) HasConditionArg() bool { return false } +// FieldEquality returns an equality test suitable for a non-BSI field. +// Given a field name, it returns an equality test from the corresponding +// argument. An equality test indicates whether the test is actually +// against null, or if not what specific uint64 value it's against, and +// whether it's an equal or non-equal test. This exists mostly to be able +// to extract `== null` and `!= null` tests consistently even for non-BSI +// fields. +func (c *Call) FieldEquality(k string) (isNull bool, value uint64, equal bool, err error) { + v := c.Args[k] + equal = true + if cond, ok := v.(*Condition); ok { + // only exactly EQ and NEQ are equality tests. + if cond.Op == EQ || cond.Op == NEQ { + equal = cond.Op == EQ + if cond.Value == nil { + return true, 0, equal, nil + } + // pick either cond.Value, or cond.Value[0] if it's + // a slice of interface{}, to be our new "v" and then + // fall through to handling we would have done for + // a non-condition. + if values, ok := cond.Value.([]interface{}); ok { + if len(values) != 1 { + return false, 0, false, fmt.Errorf("expected exactly one value for EQ/NEQ, got %d", len(values)) + } + v = values[0] + } else { + v = cond.Value + } + } else { + return false, 0, false, fmt.Errorf("only support == or != conditions, got %s", cond.Op) + } + // + } + if u, ok := v.(uint64); ok { + return false, u, equal, nil + } + if i, ok := v.(int64); ok { + return false, uint64(i), equal, nil + } + return false, 0, false, fmt.Errorf("expected integer or nil value, got %T", v) +} + +// FieldRange yields the range test corresponding to the given key, +// which means either it's a Condition, or it's just a raw equality +// to a value, which we treat as {EQ, []any{value}}. This is suitable +// for use with BSI fields, which generally get their operations as +// Conditions, and simplifies the caller side of this. +func (c *Call) FieldRange(k string) (op Token, value interface{}, err error) { + v := c.Args[k] + if cond, ok := v.(*Condition); ok { + return cond.Op, cond.Value, nil + } + // this shows up if someone wrote `foo=3`, which was + // how we spelled it for non-BSI fields. we want to handle that + // gracefully. If it had been a condition, we'd have written + // it as {Op: EQ, Value: []interface{}{v}}, so we return what + // the above would have produced in that case. Sneaky, huh. + return EQ, []interface{}{v}, nil +} + // TranslateInfo returns the relevant translation fields. func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) { switch c.Name { diff --git a/rbf.go b/rbf.go index 78081f1bc..15ecca41b 100644 --- a/rbf.go +++ b/rbf.go @@ -218,6 +218,96 @@ func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (c return tx.addOrRemove(index, field, view, shard, true, a...) } +// Removed clears the specified bits and tells you which ones it actually removed. +func (tx *RBFTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + if len(a) == 0 { + return a, nil + } + name := rbfName(index, field, view, shard) + // this special case can/should possibly go away, except that it + // turns out to be by far the most common case, and we need to know + // there's at least two items to simplify the check-sorted thing. + if len(a) == 1 { + hi, lo := highbits(a[0]), lowbits(a[0]) + rc, err := tx.tx.Container(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to retrieve container") + } + if rc.N() == 0 { + return a[:0], nil + } + rc1, chng := rc.Remove(lo) + if !chng { + return a[:0], nil + } + if rc1.N() == 0 { + err = tx.tx.RemoveContainer(name, hi) + } else { + err = tx.tx.PutContainer(name, hi, rc1) + } + if err != nil { + return a[:0], err + } + return a[:1], nil + } + + changeCount := 0 + changed = a + + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + + for i, v := range a { + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if rc == nil || rc.N() == 0 { + err = tx.tx.RemoveContainer(name, lastHi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to remove container") + } + } else { + err = tx.tx.PutContainer(name, lastHi, rc) + if err != nil { + return a[:0], errors.Wrap(err, "failed to put container") + } + } + } + // get the next container + rc, err = tx.tx.Container(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to retrieve container") + } + } // else same container, keep adding bits to rct. + chng := false + rc, chng = rc.Remove(lo) + if chng { + changed[changeCount] = v + changeCount++ + } + lastHi = hi + } + // write the last updates. + + if rc == nil || rc.N() == 0 { + err = tx.tx.RemoveContainer(name, hi) + if err != nil { + return a[:0], errors.Wrap(err, "failed to remove container") + } + } else { + err = tx.tx.PutContainer(name, hi, rc) + if err != nil { + return a[:0], errors.Wrap(err, "failed to put container") + } + } + return changed[:changeCount], nil +} + // sortedParanoia is a flag to enable a check for unsorted inputs to addOrRemove, // which is expensive in practice and only really useful occasionally. const sortedParanoia = false diff --git a/schema.go b/schema.go index 3dacfa24d..a8944dd62 100644 --- a/schema.go +++ b/schema.go @@ -264,6 +264,7 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field { TimeQuantum: timeQuantum, TTL: fo.TTL, ForeignIndex: foreignIndex, + TrackExistence: fo.TrackExistence, }, } } @@ -367,6 +368,7 @@ func FieldToFieldInfo(fld *dax.Field) *FieldInfo { TimeQuantum: TimeQuantum(fld.Options.TimeQuantum), TTL: fld.Options.TTL, ForeignIndex: fld.Options.ForeignIndex, + TrackExistence: fld.Options.TrackExistence, }, Views: nil, // TODO(tlt): do we need views populated? } @@ -410,6 +412,13 @@ func fieldToFieldType(f *dax.Field) string { } } +// FieldFromFieldOptions creates a dax.Field given a set of existing +// field options. It should possibly be unconditionally setting +// TrackExistence, because it's called in two places in SQL3 both +// of which are creating new tables, but for now I'm trying to keep +// its behavior transparent, and handle the enabling of TrackExistence +// in the code that knows it is creating a field, thus, in sql's +// create/alter table, or in api.CreateField. func FieldFromFieldOptions(fname dax.FieldName, opts ...FieldOption) (*dax.Field, error) { fo, err := newFieldOptions(opts...) if err != nil { @@ -485,6 +494,9 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) { default: return nil, errors.Errorf("unsupport field type: %s", fld.Type) } + if fld.Options.TrackExistence { + opts = append(opts, OptFieldTrackExistence()) + } return opts, nil } diff --git a/server.go b/server.go index b04feea7f..18591dd55 100644 --- a/server.go +++ b/server.go @@ -954,20 +954,37 @@ func (s *Server) ViewsRemoval(ctx context.Context) { } } } - if field.Options().NoStandardView && field.view(viewStandard) != nil { - // delete view "standard" if NoStandardView is true and view "standard" exists - for _, shard := range field.AvailableShards(true).Slice() { - err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) - if err != nil { - s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) + if field.Options().NoStandardView { + if field.view(viewStandard) != nil { + // delete view "standard" if NoStandardView is true and view "standard" exists + for _, shard := range field.AvailableShards(true).Slice() { + err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) + if err != nil { + s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) + } } - } - err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) - if err != nil { - s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) + if err != nil { + s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) + } + s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name) + } + if field.view(viewExistence) != nil { + // delete view "existence" if NoStandardView is true and view "existence" exists + for _, shard := range field.AvailableShards(true).Slice() { + err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewExistence, shard, nil) + if err != nil { + s.logger.Errorf("delete view %s from shard %d: %s", viewExistence, shard, err) + } + } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewExistence) + if err != nil { + s.logger.Errorf("view: %s, delete view: %s", viewExistence, err) + } + s.logger.Infof("view %s deleted - index: %s, field: %s ", viewExistence, index.name, field.name) } - s.logger.Infof("view %s deleted - index: %s, field: %s ", viewStandard, index.name, field.name) } } } diff --git a/server/grpc_test.go b/server/grpc_test.go index bc6e2a0b0..876570870 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -566,7 +566,7 @@ func TestQuerySQL(t *testing.T) { {[]columnResponse{uint64(5), int64(16), []string{"blue"}, int64(60), int64(-2), "2014-01-02T12:32:00Z"}}, {[]columnResponse{uint64(6), int64(34), []string{"blue"}, int64(70), int64(100), "2010-05-02T12:32:00Z"}}, {[]columnResponse{uint64(7), int64(27), []string{"blue"}, int64(80), int64(0), "2016-08-02T12:32:00Z"}}, - {[]columnResponse{uint64(8), int64(16), []string{}, int64(90), int64(-13), "2020-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(8), int64(16), []string(nil), int64(90), int64(-13), "2020-01-02T12:32:00Z"}}, {[]columnResponse{uint64(9), int64(16), []string{"red"}, int64(100), int64(80), "2000-03-02T12:32:00Z"}}, {[]columnResponse{uint64(10), int64(31), []string{"red"}, int64(110), int64(-2), "2018-01-02T12:32:00Z"}}, }, diff --git a/server/server_test.go b/server/server_test.go index 88e60d803..e46547b6e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -694,7 +694,7 @@ func TestMain_ImportTimestamp(t *testing.T) { } exp := []string{ - "standard", "standard_2018", "standard_201801", "standard_20180101", + "existence", "standard", "standard_2018", "standard_201801", "standard_20180101", "standard_2019", "standard_201912", "standard_20191231", } got := []string{} diff --git a/server_test.go b/server_test.go index 4ea5f0286..7bd8ea57f 100644 --- a/server_test.go +++ b/server_test.go @@ -42,7 +42,7 @@ func TestViewsRemovalTTL(t *testing.T) { { name: "date_old", date: "2001-02-03T04:05", - expViews: []string{"standard"}, + expViews: []string{"existence", "standard"}, /* date_old (2001-02-03T04:05), this will create these views: - standard - standard_2001 @@ -56,6 +56,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_now", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -73,6 +74,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_yesterday", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateYesterday.Year()), "standard_" + fmt.Sprintf("%d%02d", dateYesterday.Year(), dateYesterday.Month()), @@ -107,6 +109,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_last_of_month", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month(), dateLastDayOfMonth.Day(), dateLastDayOfMonth.Hour(), dateLastDayOfMonth.Minute()), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateLastDayOfMonth.Year()), "standard_" + fmt.Sprintf("%d%02d", dateLastDayOfMonth.Year(), dateLastDayOfMonth.Month()), @@ -124,6 +127,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_first_hour_day", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 0, 0), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -141,6 +145,7 @@ func TestViewsRemovalTTL(t *testing.T) { name: "date_last_hour_day", date: fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), 23, 59), expViews: []string{ + "existence", "standard", "standard_" + fmt.Sprintf("%d", dateNow.Year()), "standard_" + fmt.Sprintf("%d%02d", dateNow.Year(), dateNow.Month()), @@ -228,7 +233,7 @@ func TestViewsRemovalStandard(t *testing.T) { name: "t2_keep_standard", date: "2001-02-03T04:05", noStandardView: "false", - expViews: []string{"standard"}, + expViews: []string{"existence", "standard"}, /* date 2001-02-03T04:05, this will create these views: - standard - standard_2001 diff --git a/sql3/planner/expressionpql.go b/sql3/planner/expressionpql.go index f80500534..c6f2022b7 100644 --- a/sql3/planner/expressionpql.go +++ b/sql3/planner/expressionpql.go @@ -541,12 +541,19 @@ func (p *ExecutionPlanner) generatePQLCallFromBinaryExpr(ctx context.Context, ex pqlOp = pql.NEQ } switch typ := expr.lhs.Type().(type) { - case *parser.DataTypeID: + case *parser.DataTypeID, *parser.DataTypeString, *parser.DataTypeIDSet, *parser.DataTypeStringSet: if strings.EqualFold(lhs.columnName, string(dax.PrimaryKeyFieldName)) { return nil, sql3.NewErrInvalidColumnInFilterExpression(0, 0, string(dax.PrimaryKeyFieldName), "is/is not null") } - return nil, sql3.NewErrInvalidTypeInFilterExpression(0, 0, typ.TypeDescription(), "is/is not null") - + return &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + lhs.columnName: &pql.Condition{ + Op: pqlOp, + Value: nil, + }, + }, + }, nil case *parser.DataTypeInt, *parser.DataTypeDecimal, *parser.DataTypeTimestamp: return &pql.Call{ Name: "Row", diff --git a/sql3/planner/opaltertable.go b/sql3/planner/opaltertable.go index 3b3b362f4..6da8db0e5 100644 --- a/sql3/planner/opaltertable.go +++ b/sql3/planner/opaltertable.go @@ -94,6 +94,8 @@ func (i *alterTableRowIter) Next(ctx context.Context) (types.Row, error) { fos := i.columnDef.fos fld, err := pilosa.FieldFromFieldOptions(fname, fos...) + // all newly created fields unconditionally have TrackExistence turned on. + fld.Options.TrackExistence = true if err != nil { return nil, err } diff --git a/sql3/planner/opcreatetable.go b/sql3/planner/opcreatetable.go index fe9c877f3..25de51bbf 100644 --- a/sql3/planner/opcreatetable.go +++ b/sql3/planner/opcreatetable.go @@ -112,6 +112,8 @@ func (i *createTableRowIter) Next(ctx context.Context) (types.Row, error) { for _, f := range i.columns { fld, err := pilosa.FieldFromFieldOptions(dax.FieldName(f.name), f.fos...) + // We unconditionally turn on TrackExistence for all newly-created fields. + fld.Options.TrackExistence = true if err != nil { return nil, errors.Wrapf(err, "creating field from field options: %s", f.name) } diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index b1f0cbeff..e6267e12d 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -265,24 +265,22 @@ func (i *distinctScanRowIter) Next(ctx context.Context) (types.Row, error) { row[0] = pql.NewDecimal(val, t.Scale) case *parser.DataTypeIDSet: - //empty sets are null val, ok := result.([]uint64) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result) } - if len(val) == 0 { + if val == nil { row[0] = nil } else { row[0] = val } case *parser.DataTypeStringSet: - //empty sets are null val, ok := result.([]string) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result) } - if len(val) == 0 { + if val == nil { row[0] = nil } else { row[0] = val diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 6c876cea3..47154a65e 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -314,24 +314,22 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { } else { switch mappedColumn.dataType.(type) { case *parser.DataTypeIDSet: - //empty sets are null val, ok := result.Rows[mappedSrcColIdx].([]uint64) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result.Rows[mappedSrcColIdx]) } - if len(val) == 0 { + if val == nil { row[mappedColIdx] = nil } else { row[mappedColIdx] = val } case *parser.DataTypeStringSet: - //empty sets are null val, ok := result.Rows[mappedSrcColIdx].([]string) if !ok { return nil, sql3.NewErrInternalf("unexpected type for column value '%T'", result.Rows[mappedSrcColIdx]) } - if len(val) == 0 { + if val == nil { row[mappedColIdx] = nil } else { row[mappedColIdx] = val diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index ac007da79..5d35d2c5c 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -452,10 +452,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "int", constraints: "min 100 max 10000", expOptions: pilosa.FieldOptions{ - Type: "int", - Base: 100, - Min: pql.NewDecimal(100, 0), - Max: pql.NewDecimal(10000, 0), + Type: "int", + Base: 100, + Min: pql.NewDecimal(100, 0), + Max: pql.NewDecimal(10000, 0), + TrackExistence: true, }, }, { @@ -463,7 +464,8 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "bool", constraints: "", expOptions: pilosa.FieldOptions{ - Type: "bool", + Type: "bool", + TrackExistence: true, }, }, //test creates timestamp column without the epoch and expects the "Base" to be defaulted 0, which is the unix epoch @@ -472,11 +474,12 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "timestamp", constraints: "timeunit 'ms'", expOptions: pilosa.FieldOptions{ - Base: 0, - Type: "timestamp", - TimeUnit: "ms", - Min: pql.NewDecimal(-62135596799000, 0), - Max: pql.NewDecimal(253402300799000, 0), + Base: 0, + Type: "timestamp", + TimeUnit: "ms", + Min: pql.NewDecimal(-62135596799000, 0), + Max: pql.NewDecimal(253402300799000, 0), + TrackExistence: true, }, }, { @@ -484,10 +487,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "decimal(2)", constraints: "", expOptions: pilosa.FieldOptions{ - Type: "decimal", - Scale: 2, - Min: pql.NewDecimal(-9223372036854775808, 2), - Max: pql.NewDecimal(9223372036854775807, 2), + Type: "decimal", + Scale: 2, + Min: pql.NewDecimal(-9223372036854775808, 2), + Max: pql.NewDecimal(9223372036854775807, 2), + TrackExistence: true, }, }, { @@ -495,10 +499,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "string", constraints: "cachetype ranked size 1000", expOptions: pilosa.FieldOptions{ - Type: "mutex", - Keys: true, - CacheType: "ranked", - CacheSize: 1000, + Type: "mutex", + Keys: true, + CacheType: "ranked", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -506,10 +511,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "stringset", constraints: "cachetype lru size 1000", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: true, - CacheType: "lru", - CacheSize: 1000, + Type: "set", + Keys: true, + CacheType: "lru", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -517,12 +523,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "stringsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ - Type: "time", - Keys: true, - CacheType: "", - CacheSize: 0, - TimeQuantum: "YMD", - TTL: time.Duration(24 * time.Hour), + Type: "time", + Keys: true, + CacheType: "", + CacheSize: 0, + TimeQuantum: "YMD", + TTL: time.Duration(24 * time.Hour), + TrackExistence: true, }, }, { @@ -530,10 +537,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "id", constraints: "cachetype ranked size 1000", expOptions: pilosa.FieldOptions{ - Type: "mutex", - Keys: false, - CacheType: "ranked", - CacheSize: 1000, + Type: "mutex", + Keys: false, + CacheType: "ranked", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -541,10 +549,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idset", constraints: "cachetype lru", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: false, - CacheType: "lru", - CacheSize: pilosa.DefaultCacheSize, + Type: "set", + Keys: false, + CacheType: "lru", + CacheSize: pilosa.DefaultCacheSize, + TrackExistence: true, }, }, { @@ -552,10 +561,11 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idset", constraints: "cachetype lru size 1000", expOptions: pilosa.FieldOptions{ - Type: "set", - Keys: false, - CacheType: "lru", - CacheSize: 1000, + Type: "set", + Keys: false, + CacheType: "lru", + CacheSize: 1000, + TrackExistence: true, }, }, { @@ -563,12 +573,13 @@ func TestPlanner_CoverCreateTable(t *testing.T) { typ: "idsetq", constraints: "timequantum 'YMD' ttl '24h'", expOptions: pilosa.FieldOptions{ - Type: "time", - Keys: false, - CacheType: "", - CacheSize: 0, - TimeQuantum: "YMD", - TTL: time.Duration(24 * time.Hour), + Type: "time", + Keys: false, + CacheType: "", + CacheSize: 0, + TimeQuantum: "YMD", + TTL: time.Duration(24 * time.Hour), + TrackExistence: true, }, }, } diff --git a/sql3/test/defs/defs_set_functions.go b/sql3/test/defs/defs_set_functions.go index 893f9bfbf..c3acc23c1 100644 --- a/sql3/test/defs/defs_set_functions.go +++ b/sql3/test/defs/defs_set_functions.go @@ -97,8 +97,8 @@ var setFunctionTests = TableTest{ ), srcRows( srcRow(int64(1), int64(10), int64(100), []string{"POST"}, []int64{101}), - srcRow(int64(2), int64(20), int64(200), []string{"GET"}, nil), - srcRow(int64(3), int64(30), int64(300), []string{"GET", "POST"}, nil), + srcRow(int64(2), int64(20), int64(200), []string{"GET"}, []int64(nil)), + srcRow(int64(3), int64(30), int64(300), []string{"GET", "POST"}, []int64(nil)), ), ), SQLTests: []SQLTest{ diff --git a/sql3/test/defs/types.go b/sql3/test/defs/types.go index 9a1a0486c..bbbf493b0 100644 --- a/sql3/test/defs/types.go +++ b/sql3/test/defs/types.go @@ -226,14 +226,18 @@ func (sr sourceRows) insertTuples(t *testing.T) string { case float64: sb.WriteString(fmt.Sprintf("%.2f", v)) case []int64: - strs := make([]string, len(v)) - for i := range v { - strs[i] = fmt.Sprintf("%d", v[i]) + if v == nil { + sb.WriteString("NULL") + } else { + strs := make([]string, len(v)) + for i := range v { + strs[i] = fmt.Sprintf("%d", v[i]) + } + sb.WriteString("[" + strings.Join(strs, ",") + "]") } - sb.WriteString("[" + strings.Join(strs, ",") + "]") case []string: - if len(v) == 0 { - sb.WriteString("[]") + if v == nil { + sb.WriteString("NULL") } else { sb.WriteString("['" + strings.Join(v, "','") + "']") } diff --git a/stattx.go b/stattx.go index ec40537c0..45ff5b50b 100644 --- a/stattx.go +++ b/stattx.go @@ -149,6 +149,7 @@ const ( kRemoveContainer kAdd kRemove + kRemoved kContains kContainerIterator kCount @@ -182,6 +183,8 @@ func (k kall) String() string { return "kAdd" case kRemove: return "kRemove" + case kRemoved: + return "kRemoved" case kContains: return "kContains" case kContainerIterator: @@ -358,6 +361,23 @@ func (c *statTx) Remove(index, field, view string, shard uint64, a ...uint64) (c return c.b.Remove(index, field, view, shard, a...) } +func (c *statTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) { + me := kRemoved + + t0 := time.Now() + defer func() { + c.stats.add(me, time.Since(t0)) + }() + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Removed(index, field, view, shard, a...) +} + func (c *statTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { me := kContains diff --git a/tx.go b/tx.go index 3a4079f5b..f43967277 100644 --- a/tx.go +++ b/tx.go @@ -101,6 +101,10 @@ type Tx interface { // Remove removes the 'a' values from the Bitmap for the fragment. Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) + // Removed removes values, returning the set of values it removed. + // It may overwrite the slice passed to it. + Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) + // Contains tests if the uint64 v is stored in the fragment's Bitmap. Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) diff --git a/view.go b/view.go index d73caf530..cc1e18c1c 100644 --- a/view.go +++ b/view.go @@ -24,9 +24,12 @@ import ( // View layout modes. const ( + // standard view holds regular set/mutex data viewStandard = "standard" - + // bsig_X view holds BSI data for X viewBSIGroupPrefix = "bsig_" + // existence view holds existence bits for a specific field + viewExistence = "existence" ) // view represents a container for field data.