From cf97a0dcb8525499eab8345fc61c86eb7a28e86d Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 11 Jan 2023 12:12:04 -0600 Subject: [PATCH] overhaul: switch over to using QueryContext We switch everything to use QueryContext/QueryRead/etc instead of Qcx/Tx. We drop the short_txkey subpackage (it's now handled by either keys or querycontext). We drop all the dbshard stuff, and all the tx/txfactory stuff. We remove all the things that related to the old "Block" concept, which was mostly used by the anti-entropy code, but had one fragmentary usage left in the ImportRoaringOverwrite case of ImportRoaring. That's replaced by using a rewriter that deletes all bits (not just bits in specific columns) from an existing thing, but writes in new bits. Actually we could probably do that better with a custom "eradicate-rewriter" that doesn't try to be clever, and just eliminates things. This includes a number of minor bug fixes that were exposed by getting the testing to work. For example: * When checking whether an operation "requires write", we now consider a Delete a kind of a Write, because it is. * Several tests were relying on the fact that writes through Qcx were being committed whether or not the Qcx was ever told to finish. With QueryContext, you actually have to reach a Commit() or the writes don't happen (except for special cases in Delete). * Replaced a lot of panics with t.Fatalf in tests. There's also some minor staticcheck fixes, like deleting the unused "db" member of a boltdb transaction wrapper. --- api.go | 345 ++++---- api_directive.go | 67 +- api_test.go | 151 ++-- apimethod_string.go | 58 +- apply.go | 5 +- arrow.go | 7 +- catcher.go | 221 ------ ctl/rbf_pages.go | 16 +- ctl/restore.go | 33 +- dataframe_test.go | 9 +- dax/computer/logmessage.go | 1 - dbshard.go | 702 ---------------- dbshard_internal_test.go | 281 ------- dbshard_test.go | 90 --- delete_test.go | 43 +- encoding/proto/proto.go | 2 - executor.go | 783 +++++++----------- executor_internal_test.go | 15 +- executor_test.go | 81 +- field.go | 187 +++-- field_internal_test.go | 138 ++-- field_test.go | 21 +- fragment.go | 753 +++++++----------- fragment_internal_test.go | 1547 ++++++++++++++++-------------------- handler.go | 1 - holder.go | 356 +++++++-- holder_internal_test.go | 42 +- holder_test.go | 5 +- http_handler.go | 110 ++- index.go | 68 +- index_internal_test.go | 21 - internal_client.go | 31 +- internal_client_test.go | 20 +- pql/ast.go | 2 +- rbf.go | 537 ------------- rbf/ingest_test.go | 7 +- rbf/tx.go | 18 - rbf/util.go | 30 +- roaring/filter.go | 21 + roaring/roaring.go | 27 - server.go | 37 +- server/grpc.go | 7 +- server/handler_test.go | 44 +- server/server_test.go | 17 +- short_txkey/txkey.go | 199 ----- short_txkey/txkey_test.go | 78 -- stattx.go | 496 ------------ test/cluster.go | 43 +- test/holder.go | 81 +- test/index.go | 9 +- translate_boltdb.go | 1 - tx.go | 166 ---- tx_internal_test.go | 133 ---- tx_test.go | 244 ------ txfactory.go | 702 ---------------- txfactory_internal_test.go | 19 - txkey/txkey.go | 253 ------ txkey/txkey_test.go | 125 --- util.go | 3 - view.go | 146 ++-- view_internal_test.go | 32 - 61 files changed, 2577 insertions(+), 7110 deletions(-) delete mode 100644 catcher.go delete mode 100644 dbshard.go delete mode 100644 dbshard_internal_test.go delete mode 100644 dbshard_test.go delete mode 100644 index_internal_test.go delete mode 100644 rbf.go delete mode 100644 short_txkey/txkey.go delete mode 100644 short_txkey/txkey_test.go delete mode 100644 stattx.go delete mode 100644 tx.go delete mode 100644 tx_internal_test.go delete mode 100644 tx_test.go delete mode 100644 txfactory.go delete mode 100644 txfactory_internal_test.go delete mode 100644 txkey/txkey.go delete mode 100644 txkey/txkey_test.go diff --git a/api.go b/api.go index 0e7ad39f7..4dae006a4 100644 --- a/api.go +++ b/api.go @@ -4,7 +4,6 @@ package pilosa import ( - "bufio" "bytes" "context" "encoding/binary" @@ -13,7 +12,6 @@ import ( "io" "math" "net/url" - "os" "runtime" "sort" "strconv" @@ -26,8 +24,9 @@ import ( "github.com/molecula/featurebase/v3/dax/computer" "github.com/molecula/featurebase/v3/dax/storage" "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/logger" - "github.com/molecula/featurebase/v3/rbf" + qc "github.com/molecula/featurebase/v3/querycontext" //"github.com/molecula/featurebase/v3/pg" "github.com/molecula/featurebase/v3/pql" @@ -73,6 +72,28 @@ func (api *API) logger() logger.Logger { return api.server.logger } +// NewQueryContext requests a new read-only query context from the API's holder. +func (api *API) NewQueryContext(ctx context.Context) (qc.QueryContext, error) { + return api.holder.NewQueryContext(ctx) +} + +// NewWriteQueryContext requests a new write query context from the API's holder, +// using the provided scope. +func (api *API) NewWriteQueryContext(ctx context.Context, scope qc.QueryScope) (qc.QueryContext, error) { + return api.holder.NewWriteQueryContext(ctx, scope) +} + +// NewIndexQueryContext requests a new write query context from the API's holder, +// using the provided index. If shards are provided, it's restricted to those shards, +// otherwise it's the whole index. +func (api *API) NewIndexQueryContext(ctx context.Context, index string, shards ...uint64) (qc.QueryContext, error) { + // helpfully treat a shard of -1 as no shard + if len(shards) > 0 && shards[0] == ^uint64(0) { + shards = shards[1:] + } + return api.holder.NewIndexQueryContext(ctx, index, shards...) +} + // apiOption is a functional option type for pilosa.API type apiOption func(*API) error @@ -200,10 +221,6 @@ func (api *API) Close() error { return nil } -func (api *API) Txf() *TxFactory { - return api.holder.Txf() -} - // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { start := time.Now() @@ -474,7 +491,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { type importJob struct { ctx context.Context - qcx *Qcx + qcx qc.QueryContext req *ImportRoaringRequest shard uint64 field *Field @@ -511,18 +528,11 @@ func importWorker(importWork chan importJob) { doAction = RequestActionSet } } - if err := func() (err1 error) { - tx, finisher, err := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) - if err != nil { - return err - } - defer finisher(&err1) - var doClear bool switch doAction { case RequestActionOverwrite: - err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) + err := j.field.importRoaringOverwrite(j.ctx, j.qcx, viewData, j.shard, viewName) if err != nil { return errors.Wrap(err, "importing roaring as overwrite") } @@ -546,17 +556,19 @@ func importWorker(importWork chan importJob) { return errors.Wrap(err, "merging existence on roaring import") } - err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + err = ef.importRoaring(j.ctx, j.qcx, existence, j.shard, "standard", false) if err != nil { return errors.Wrap(err, "updating existence on roaring import") } } } - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + err := j.field.importRoaring(j.ctx, j.qcx, data, j.shard, viewName, doClear) if err != nil { return errors.Wrap(err, "importing standard roaring") } + default: + return fmt.Errorf("unexpected action type %q", doAction) } return nil }(); err != nil { @@ -574,7 +586,7 @@ func importWorker(importWork chan importJob) { } // combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment. -func combineForExistence(inputRoaringData []byte) ([]byte, error) { +func combineForExistence(inputRoaringData []byte) (a []byte, b error) { rowSize := uint64(1 << shardVsContainerExponent) rit, err := roaring.NewRoaringIterator(inputRoaringData) if err != nil { @@ -635,8 +647,11 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return newPreconditionFailedError(err) } - qcx := api.Txf().NewQcx() - defer qcx.Abort() + qcx, err := api.NewIndexQueryContext(ctx, indexName, shard) + if err != nil { + return errors.Wrap(err, "creating query context") + } + defer qcx.Release() // Create a snapshot of the cluster to use for node/partition calculations. snap := api.cluster.NewSnapshot() @@ -690,7 +705,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, Shard: shard, Clear: req.Clear, Action: req.Action, - Block: req.Block, UpdateExistence: req.UpdateExistence, Views: req.Views, } @@ -711,8 +725,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something? } } - - return qcx.Finish() + return qcx.Commit() } } } @@ -823,8 +836,15 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Obtain transaction - tx := index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Shard: shard}) - defer tx.Rollback() + qcx, err := api.holder.NewQueryContext(ctx) + if err != nil { + return err + } + defer qcx.Release() + qr, err := f.qcxRead(qcx) + if err != nil { + return err + } // Wrap writer with a CSV writer. cw := csv.NewWriter(w) @@ -860,7 +880,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return cw.Write([]string{rowStr, colStr}) } - citer, _, err := tx.ContainerIterator(indexName, fieldName, viewStandard, shard, 0) + citer, _, err := qr.ContainerIterator(0) if err != nil { return err } @@ -884,7 +904,6 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin // Ensure data is flushed. cw.Flush() span.LogKV("n", n) - tx.Rollback() return nil } @@ -918,23 +937,6 @@ func (api *API) PartitionNodes(ctx context.Context, partitionID int) ([]*disco.N return snap.PartitionNodes(partitionID), nil } -// FragmentData returns all data in the specified fragment. -func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentData") - defer span.Finish() - - if err := api.validate(apiFragmentData); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - // Retrieve fragment from holder. - f := api.holder.fragment(indexName, fieldName, viewName, shard) - if f == nil { - return nil, ErrFragmentNotFound - } - return f, nil -} - type RedirectError struct { HostPort string error string @@ -1265,35 +1267,24 @@ func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard return nil, newNotFoundError(ErrIndexNotFound, indexName) } - // Start transaction. - tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard, Write: writeTx}) - - // Ensure transaction is an RBF transaction. - rtx, ok := tx.(*RBFTx) + // check whether txStore backend supports backup + br, ok := api.holder.txStore.(qc.TxBackupRestore) if !ok { - tx.Rollback() - return nil, fmt.Errorf("snapshot not available for %q storage", tx.Type()) + return nil, errors.New("backend does not support backup/restore operations") } - r, err := rtx.SnapshotReader() + // Start transaction. + qcx, err := api.NewIndexQueryContext(ctx, indexName, shard) if err != nil { - tx.Rollback() return nil, err } - return &txReadCloser{tx: tx, Reader: r}, nil -} - -var _ io.ReadCloser = (*txReadCloser)(nil) - -// txReadCloser wraps a reader to close a tx on close. -type txReadCloser struct { - io.Reader - tx Tx -} - -func (r *txReadCloser) Close() error { - r.tx.Rollback() - return nil + // Backup either releases the QueryContext, or returns a ReadCloser which + // releases it on close. Note that we DO NOT release/close our + // QueryContext, because the entire point is to block any other writes + // to that shard from happening until the backup completes, because we + // don't want to allow any new writes to start and thus send data to the + // write log until we've snapshotted. + return br.Backup(qcx, keys.Index(indexName), keys.Shard(shard)) } // ImportOptions holds the options for the API.Import @@ -1351,7 +1342,7 @@ func OptImportOptionsSuppressLog(b bool) ImportOption { var ErrAborted = fmt.Errorf("error: update was aborted") -func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRecord, opts ...ImportOption) error { +func (api *API) ImportAtomicRecord(ctx context.Context, qcx qc.QueryContext, req *AtomicRecord, opts ...ImportOption) error { simPowerLoss := false lossAfter := -1 var opt ImportOptions @@ -1368,14 +1359,6 @@ func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRec lossAfter = opt.SimPowerLossAfter } - idx, err := api.Index(ctx, req.Index) - if err != nil { - return errors.Wrap(err, "getting index") - } - - // the whole point is to run this part of the import atomically. - // Begin that Tx now! - qcx.StartAtomicWriteTx(Txo{Write: writable, Index: idx, Shard: req.Shard}) tot := 0 options, err := setUpImportOptions(opts...) @@ -1429,7 +1412,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption { } // Import does the top-level importing. -func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) { +func (api *API) Import(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, opts ...ImportOption) (err error) { if req.Clear { opts = addClearToImportOptions(opts) } @@ -1506,7 +1489,7 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts . } // ImportWithTx bulk imports data into a particular index,field,shard. -func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { +func (api *API) ImportWithTx(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, options *ImportOptions) error { span, _ := tracing.StartSpanFromContext(ctx, "API.Import") defer span.Finish() @@ -1640,61 +1623,57 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard } // we really only need a Tx, but getting a Qcx so that there's only one path for getting a Tx - qcx := api.Txf().NewQcx() - qcx.write = true - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) + qcx, err := api.NewIndexQueryContext(ctx, indexName, shard) if err != nil { - return errors.Wrap(err, "getting Tx") + return err } - defer qcx.Finish() - var err1 error - defer finisher(&err1) + defer qcx.Release() if !req.Remote { - err1 = errors.New("forwarding unimplemented on this endpoint") - return err1 + return errors.New("forwarding unimplemented on this endpoint") } for _, viewUpdate := range req.Views { field := index.Field(viewUpdate.Field) if field == nil { - err1 = errors.Errorf("no field named '%s' found.", viewUpdate.Field) - return err1 + return errors.Errorf("no field named '%s' found.", viewUpdate.Field) } fieldType := field.Options().Type - if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil { - return err1 + if err := cleanupView(fieldType, &viewUpdate); err != nil { + return err } view, err := field.createViewIfNotExists(viewUpdate.View) if err != nil { - err1 = errors.Wrap(err, "getting view") - return err1 + return errors.Wrap(err, "getting view") } frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { - err1 = errors.Wrap(err, "getting fragment") - return err1 + return errors.Wrap(err, "getting fragment") + } + qw, err := frag.qcxWrite(qcx) + if err != nil { + return err } switch fieldType { case FieldTypeSet, FieldTypeTime: if !viewUpdate.ClearRecords { - err1 = frag.ImportRoaringClearAndSet(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + err = frag.ImportRoaringClearAndSet(ctx, qw, viewUpdate.Clear, viewUpdate.Set) } else { - err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + err = frag.ImportRoaringSingleValued(ctx, qw, viewUpdate.Clear, viewUpdate.Set) } case FieldTypeInt, FieldTypeTimestamp, FieldTypeDecimal: - err1 = frag.ImportRoaringBSI(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + err = frag.ImportRoaringBSI(ctx, qw, viewUpdate.Clear, viewUpdate.Set) case FieldTypeMutex, FieldTypeBool: - err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + err = frag.ImportRoaringSingleValued(ctx, qw, viewUpdate.Clear, viewUpdate.Set) default: - err1 = errors.Errorf("field type %s is not supported", fieldType) + err = errors.Errorf("field type %s is not supported", fieldType) } - if err1 != nil { - return err1 + if err != nil { + return err } // need to update field/bsiGroup bitDepth value if this is an int-like field. @@ -1703,10 +1682,9 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard // that we have to do this is weird and since this state isn't // in RBF might have transactional issues. if len(field.bsiGroups) > 0 { - maxRowID, _, err := frag.maxRow(tx, nil) + maxRowID, _, err := frag.maxRow(qw, nil) if err != nil { - err1 = errors.Wrapf(err, "getting fragment max row id") - return err1 + return errors.Wrapf(err, "getting fragment max row id") } var bd uint64 if maxRowID+1 > bsiOffsetBit { @@ -1741,18 +1719,16 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON) if err != nil { - err1 = errors.Wrap(err, "marshalling log message") - return err1 + return errors.Wrap(err, "marshalling log message") } resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum) - err1 = errors.Wrap(resource.Append(b), "appending shard data") - if err1 != nil { - return err1 + err = errors.Wrap(resource.Append(b), "appending shard data") + if err != nil { + return err } } - - return nil + return qcx.Commit() } func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { @@ -1778,7 +1754,7 @@ func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { // ImportValue is a wrapper around the common code in ImportValueWithTx, which // currently just translates req.Clear into a clear ImportOption. -func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { +func (api *API) ImportValue(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, opts ...ImportOption) error { if req.Clear { opts = addClearToImportOptions(opts) } @@ -1849,7 +1825,7 @@ func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueReque } // ImportValueWithTx bulk imports values into a particular field. -func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) (err0 error) { +func (api *API) ImportValueWithTx(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, options *ImportOptions) (err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue") defer span.Finish() @@ -2029,7 +2005,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { +func importExistenceColumns(qcx qc.QueryContext, index *Index, columnIDs []uint64, shard uint64) error { ef := index.existenceField() if ef == nil { return nil @@ -2176,7 +2152,7 @@ func (api *API) Info() serverInfo { CPUMHz: mhz, CPUType: si.CPUModel(), Memory: mem, - StorageBackend: api.holder.txf.TxType(), + StorageBackend: api.holder.txStore.Backend(), ReplicaN: api.cluster.ReplicaN, ShardHash: api.cluster.Hasher.Name(), KeyHash: api.cluster.Hasher.Name(), @@ -2578,91 +2554,61 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64 } idx := api.holder.Index(indexName) - // need to get a dbShard - dbs, err := api.holder.Txf().dbPerShard.GetDBShard(indexName, shard, idx) - if err != nil { - return err + br, ok := api.holder.txStore.(qc.TxBackupRestore) + if !ok { + return errors.New("backend does not support backup/restore operations") } - db := dbs.W - finalPath := db.Path() + "/data" - tempPath := finalPath + ".tmp" - o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + qcx, err := api.NewIndexQueryContext(ctx, indexName, shard) if err != nil { - return err + return errors.Wrap(err, "getting query context") } - defer o.Close() - - bw := bufio.NewWriter(o) - if _, err = io.Copy(bw, rd); err != nil { - return err - } else if err := bw.Flush(); err != nil { - return err - } else if err := o.Sync(); err != nil { - return err - } else if err := o.Close(); err != nil { - return err + defer qcx.Release() + err = br.Restore(qcx, keys.Index(indexName), keys.Shard(shard), rd) + if err != nil { + return errors.Wrap(err, "underlying restore") } + flvs, err := api.holder.txStore.ListFieldViews(keys.Index(indexName), keys.Shard(shard)) if err != nil { - _ = os.Remove(tempPath) - return err + return errors.Wrap(err, "finding field/view list") } - err = db.CloseDB() - if err != nil { - return err - } - err = os.Rename(tempPath, finalPath) - if err != nil { - _ = os.Remove(tempPath) - return err - } - err = db.OpenDB() - if err != nil { - return err - } - tx, err := db.NewTx(false, idx.name, Txo{}) - if err != nil { - return err - } - defer tx.Rollback() - // arguments idx,shard do not matter for rbf they - // are ignored - flvs, err := tx.GetSortedFieldViewList(idx, shard) - if err != nil { - return nil - } - - for _, flv := range flvs { - fld := idx.field(flv.Field) - view := fld.view(flv.View) - if view == nil { - view, err = fld.createViewIfNotExists(flv.View) + for field, views := range flvs { + fld := idx.field(string(field)) + for _, viewName := range views { + view := fld.view(string(viewName)) + if view == nil { + view, err = fld.createViewIfNotExists(string(viewName)) + if err != nil { + return err + } + } + frag, err := view.CreateFragmentIfNotExists(shard) + if err != nil { + return err + } + qr, err := frag.qcxRead(qcx) + if err != nil { + return err + } + err = frag.RebuildRankCache(ctx, qr) + if err != nil { + return err + } + bd, err := view.bitDepth(qcx, map[keys.Shard]struct{}{keys.Shard(shard): {}}) + if err != nil { + return err + } + err = fld.cacheBitDepth(bd) if err != nil { return err } - } - frag, err := view.CreateFragmentIfNotExists(shard) - if err != nil { - return err - } - err = frag.RebuildRankCache(ctx) - if err != nil { - return err - } - bd, err := view.bitDepth([]uint64{shard}) - if err != nil { - return err - } - err = fld.cacheBitDepth(bd) - if err != nil { - return err } } return nil } -func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { +func (api *API) mutexCheckThisNode(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { index := api.holder.Index(indexName) if index == nil { return nil, newNotFoundError(ErrIndexNotFound, indexName) @@ -2735,7 +2681,7 @@ func mergeKeyLists(dst []string, src []string) []string { // MutexCheckNode checks for collisions in a given mutex field. The response is // a map[shard]map[column]values, not translated. -func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { +func (api *API) MutexCheckNode(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { if err := api.validate(apiMutexCheck); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -2754,7 +2700,7 @@ func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, // details false: // []uint64 // unkeyed index // []string // keyed index -func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) { +func (api *API) MutexCheck(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) { if err = api.validate(apiMutexCheck); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -3045,21 +2991,6 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp return api.server.CompileExecutionPlan(ctx, q) } -func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo { - infos := make(map[string]*rbf.DebugInfo) - - for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap { - wrapper, ok := dbShard.W.(*RbfDBWrapper) - if !ok { - continue - } - - skey := fmt.Sprintf("%s/%d", key.index, key.shard) - infos[skey] = wrapper.db.DebugInfo() - } - return infos -} - // Directive applies the provided Directive to the local computer. func (api *API) Directive(ctx context.Context, d *dax.Directive) error { return api.ApplyDirective(ctx, d) @@ -3203,8 +3134,8 @@ const ( apiDeleteIndex apiDeleteView apiExportCSV - apiFragmentBlockData - apiFragmentBlocks + // apiFragmentBlockData + // apiFragmentBlocks apiFragmentData apiTranslateData apiFieldTranslateData @@ -3247,8 +3178,6 @@ var methodsCommon = map[apiMethod]struct{}{ var methodsDegraded = map[apiMethod]struct{}{ apiExportCSV: {}, - apiFragmentBlockData: {}, - apiFragmentBlocks: {}, apiField: {}, apiIndex: {}, apiQuery: {}, @@ -3273,8 +3202,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiDeleteIndex: {}, apiDeleteView: {}, apiExportCSV: {}, - apiFragmentBlockData: {}, - apiFragmentBlocks: {}, apiField: {}, apiFieldTranslateData: {}, apiImport: {}, diff --git a/api_directive.go b/api_directive.go index 16fdb34d5..1a6fc3f62 100644 --- a/api_directive.go +++ b/api_directive.go @@ -552,7 +552,6 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar req := &ImportRoaringRequest{ Clear: msg.Clear, Action: msg.Action, - Block: msg.Block, Views: msg.Views, UpdateExistence: msg.UpdateExistence, SuppressLog: true, @@ -574,17 +573,30 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar Clear: msg.Clear, } - qcx := api.Txf().NewQcx() - defer qcx.Abort() + // subfunc so the qcx gets released after each message is handled. + err := func() error { + qcx, err := api.NewIndexQueryContext(ctx, req.Index, req.Shard) + if err != nil { + return errors.Wrapf(err, "creating query context") + } + defer qcx.Release() - opts := []ImportOption{ - OptImportOptionsClear(msg.Clear), - OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck), - OptImportOptionsPresorted(msg.Presorted), - OptImportOptionsSuppressLog(true), - } - if err := api.Import(ctx, qcx, req, opts...); err != nil { - return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard) + opts := []ImportOption{ + OptImportOptionsClear(msg.Clear), + OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck), + OptImportOptionsPresorted(msg.Presorted), + OptImportOptionsSuppressLog(true), + } + if err := api.Import(ctx, qcx, req, opts...); err != nil { + return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard) + } + if err := qcx.Commit(); err != nil { + return errors.Wrap(err, "committing write") + } + return nil + }() + if err != nil { + return err } case *computer.ImportValueMessage: @@ -601,17 +613,30 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar Clear: msg.Clear, } - qcx := api.Txf().NewQcx() - defer qcx.Abort() + // subfunc so the qcx gets released after each message is handled. + err = func() error { + qcx, err := api.NewIndexQueryContext(ctx, req.Index, req.Shard) + if err != nil { + return errors.Wrapf(err, "creating query context") + } + defer qcx.Release() - opts := []ImportOption{ - OptImportOptionsClear(msg.Clear), - OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck), - OptImportOptionsPresorted(msg.Presorted), - OptImportOptionsSuppressLog(true), - } - if err := api.ImportValue(ctx, qcx, req, opts...); err != nil { - return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard) + opts := []ImportOption{ + OptImportOptionsClear(msg.Clear), + OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck), + OptImportOptionsPresorted(msg.Presorted), + OptImportOptionsSuppressLog(true), + } + if err := api.ImportValue(ctx, qcx, req, opts...); err != nil { + return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard) + } + if err := qcx.Commit(); err != nil { + return errors.Wrap(err, "committing write") + } + return nil + }() + if err != nil { + return err } case *computer.ImportRoaringShardMessage: req := &ImportRoaringShardRequest{ diff --git a/api_test.go b/api_test.go index 37fb0c974..6f69c47a8 100644 --- a/api_test.go +++ b/api_test.go @@ -24,15 +24,46 @@ import ( "github.com/golang-jwt/jwt" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) +// mustQueryContext gets a write query context for the API and the +// specified index, or index-and-shards, or fails +// the test. The query context will be released at the end of the test. +func mustIndexQueryContext(tb testing.TB, api *pilosa.API, index string, shards ...uint64) qc.QueryContext { + tb.Helper() + // disregard a leading ^0, because that's idiomatic for "all shards" + if len(shards) > 0 && shards[0] == ^uint64(0) { + shards = shards[1:] + } + qcx, err := api.NewIndexQueryContext(context.Background(), index, shards...) + if err != nil { + tb.Fatalf("creating query context: %v", err) + } + tb.Cleanup(qcx.Release) + return qcx +} + +// mustQueryContext gets a read-only query context for the API, or fails +// the test. The query context will be released at the end of the test. +func mustQueryContext(tb testing.TB, api *pilosa.API) qc.QueryContext { + tb.Helper() + qcx, err := api.NewQueryContext(context.Background()) + if err != nil { + tb.Fatalf("creating query context: %v", err) + } + tb.Cleanup(qcx.Release) + return qcx +} + func TestAPI_Import(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -89,12 +120,12 @@ func TestAPI_Import(t *testing.T) { ColumnKeys: colKeys, } - qcx := m0.API.Txf().NewQcx() + qcx := mustIndexQueryContext(t, m0.API, req.Index) if err := m0.API.Import(ctx, qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) pql := fmt.Sprintf("Row(%s=%d)", fieldNames[false], rowIDs[0]) @@ -154,13 +185,10 @@ func TestAPI_Import(t *testing.T) { req.RowIDs = rowIDs } err := func() error { - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() + qcx := mustIndexQueryContext(t, m0.API, req.Index) + defer qcx.Release() err := m0.API.Import(ctx, qcx, req.Clone()) - e2 := qcx.Finish() - if e2 != nil { - t.Fatalf("unexpected error committing: %v", e2) - } + require.Nil(t, qcx.Commit()) return err }() if err != nil { @@ -241,11 +269,11 @@ func TestAPI_ImportValue(t *testing.T) { Shard: 0, // inaccurate but keys override it } - qcx := coord.API.Txf().NewQcx() + qcx := mustIndexQueryContext(t, coord.API, req.Index) if err := coord.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) pql := fmt.Sprintf("Row(%s>0)", field) @@ -291,41 +319,41 @@ func TestAPI_ImportValue(t *testing.T) { Index: index, Field: field, } - qcx1 := coord.API.Txf().NewQcx() - defer qcx1.Abort() + qcx1 := mustIndexQueryContext(t, coord.API, req.Index) + defer qcx1.Release() // Import with empty request, should succeed if err := coord.API.ImportValue(ctx, qcx1, req); err != nil { t.Fatal(err) } - PanicOn(qcx1.Finish()) + require.Nil(t, qcx1.Commit()) // Import without data but with columnkeys, verify that it errors req.ColumnKeys = colKeys - qcx2 := coord.API.Txf().NewQcx() - defer qcx2.Abort() + qcx2 := mustIndexQueryContext(t, coord.API, req.Index) + defer qcx2.Release() if err := coord.API.ImportValue(ctx, qcx2, req); err == nil { t.Fatal("expected error but succeeded") } - PanicOn(qcx2.Finish()) + require.Nil(t, qcx2.Commit()) // Import with mismatch column and value lengths req.Values = values - qcx3 := coord.API.Txf().NewQcx() - defer qcx3.Abort() + qcx3 := mustIndexQueryContext(t, coord.API, req.Index) + defer qcx3.Release() if err := coord.API.ImportValue(ctx, qcx3, req); err == nil { t.Fatal("expected error but succeeded") } - PanicOn(qcx3.Finish()) + require.Nil(t, qcx3.Commit()) // Import with data but no columns req.ColumnKeys = make([]string, 0) - qcx4 := coord.API.Txf().NewQcx() - defer qcx4.Abort() + qcx4 := mustIndexQueryContext(t, coord.API, req.Index) + defer qcx4.Release() if err := coord.API.ImportValue(ctx, qcx4, req); err == nil { t.Fatal("expected error but succeeded") } - PanicOn(qcx4.Finish()) + require.Nil(t, qcx4.Commit()) }) @@ -357,11 +385,11 @@ func TestAPI_ImportValue(t *testing.T) { ColumnIDs: colIDs, FloatValues: values, } - qcx := m0.API.Txf().NewQcx() + qcx := mustIndexQueryContext(t, m0.API, req.Index) if err := m0.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) query := fmt.Sprintf("Row(%s>6)", field) // Query node0. if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil { @@ -418,11 +446,11 @@ func TestAPI_ImportValue(t *testing.T) { TimestampValues: values, } - qcx := m2.API.Txf().NewQcx() + qcx := mustIndexQueryContext(t, m2.API, req.Index) if err := m2.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) query := fmt.Sprintf("Row(%s>='1833-11-24T17:31:50Z')", field) // 6s after MinTimestamp @@ -476,11 +504,11 @@ func TestAPI_ImportValue(t *testing.T) { ColumnIDs: colIDs, StringValues: values, } - qcx := m0.API.Txf().NewQcx() + qcx := mustIndexQueryContext(t, m0.API, req.Index) if err := m0.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) pql := fmt.Sprintf(`Row(%s=="strval-110")`, field) @@ -667,14 +695,14 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { RowIDs: []uint64{iraRowID}, } - qcx := m0api.Txf().NewQcx() + qcx := mustIndexQueryContext(t, m0api, index) if err := m0api.Import(ctx, qcx, ir0.Clone()); err != nil { t.Fatal(err) } if err := m0api.ImportValue(ctx, qcx, ivr0.Clone()); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) bitIsSet := func() bool { query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) @@ -712,24 +740,24 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { } // clear the bit - qcx = m0api.Txf().NewQcx() + qcx = mustIndexQueryContext(t, m0api, index) ir0.Clear = true if err := m0api.Import(ctx, qcx, ir0); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) if bitIsSet() { PanicOn("IRA bit should have been cleared") } // clear the BSI - qcx = m0api.Txf().NewQcx() + qcx = mustIndexQueryContext(t, m0api, index) ivr0.Clear = true if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index) if bal != 0 { @@ -883,9 +911,9 @@ func TestAPI_MutexCheck(t *testing.T) { defer c.Close() m0 := c.GetNode(0) - nodesByID := make(map[string]*test.Command, 3) - qcxsByID := make(map[string]*pilosa.Qcx, 3) - for i := 0; i < 3; i++ { + nodesByID := make(map[string]*test.Command, len(c.Nodes)) + qcxsByID := make(map[string]qc.QueryContext, len(c.Nodes)) + for i := 0; i < len(c.Nodes); i++ { node := c.GetNode(i) id := node.API.NodeID() nodesByID[id] = node @@ -937,7 +965,7 @@ func TestAPI_MutexCheck(t *testing.T) { for keyedField, fieldData := range indexData.fields { t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { for id, node := range nodesByID { - qcxsByID[id] = node.API.Txf().NewQcx() + qcxsByID[id] = mustIndexQueryContext(t, node.API, indexData.indexName) } for shard := uint64(0); shard < nShards; shard++ { // restore row/col ID values which can get altered by imports @@ -977,13 +1005,10 @@ func TestAPI_MutexCheck(t *testing.T) { t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err) } pilosa.CorruptAMutex(t, field, qcxsByID[id]) - err = qcxsByID[id].Finish() - if err != nil { - t.Fatalf("closing out transaction on node %s: %v", id, err) - } + require.Nil(t, qcxsByID[id].Commit()) } - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() + qcx := mustQueryContext(t, m0.API) + defer qcx.Release() // first two shards of each group of 4 should have a collision in // position 1 @@ -1076,9 +1101,6 @@ func TestAPI_MutexCheck(t *testing.T) { indexData = indexes[true] for keyedField, fieldData := range indexData.fields { t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { - for id, node := range nodesByID { - qcxsByID[id] = node.API.Txf().NewQcx() - } req := &pilosa.ImportRequest{ Index: indexData.indexName, IndexCreatedAt: indexData.createdAt, @@ -1105,12 +1127,11 @@ func TestAPI_MutexCheck(t *testing.T) { } else { req.RowIDs = rowIDs } - var id string var node *test.Command - for id, node = range nodesByID { + for _, node = range nodesByID { break } - if err := node.API.Import(ctx, qcxsByID[id], req); err != nil { + if err := node.API.Import(ctx, nil, req); err != nil { t.Fatalf("importing data: %v", err) } expected, err := node.API.FindIndexKeys(ctx, indexData.indexName, colKeys...) @@ -1156,14 +1177,15 @@ func TestAPI_MutexCheck(t *testing.T) { if err != nil { t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err) } - pilosa.CorruptAMutex(t, field, qcxsByID[id]) - err = qcxsByID[id].Finish() - if err != nil { - t.Fatalf("closing out transaction on node %s: %v", id, err) - } + qcx := mustIndexQueryContext(t, node.API, indexData.indexName) + pilosa.CorruptAMutex(t, field, qcx) + require.Nil(t, qcx.Commit()) } - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() + qcx := mustQueryContext(t, m0.API) + if err != nil { + t.Fatalf("creating query context: %v", err) + } + defer qcx.Release() results, err := m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName, true, 0) if err != nil { @@ -1355,21 +1377,6 @@ func TestAPI_CreateField(t *testing.T) { } } -func TestAPI_RBFDebugInfo(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := test.MustRunCluster(t, 1) - defer c.Close() - - coord := c.GetPrimary() - - if _, err := coord.API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if infos := coord.API.RBFDebugInfo(); infos == nil { - t.Fatal("expected info") - } -} - // makeUser makes an authnUserInfo from groups and a name and a secret key func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.UserInfo { tkn := jwt.New(jwt.SigningMethodHS256) diff --git a/apimethod_string.go b/apimethod_string.go index da9fde7d5..8a1607271 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -16,38 +16,38 @@ func _() { _ = x[apiDeleteIndex-5] _ = x[apiDeleteView-6] _ = x[apiExportCSV-7] - _ = x[apiFragmentBlockData-8] - _ = x[apiFragmentBlocks-9] - _ = x[apiFragmentData-10] - _ = x[apiTranslateData-11] - _ = x[apiFieldTranslateData-12] - _ = x[apiField-13] - _ = x[apiImport-14] - _ = x[apiImportValue-15] - _ = x[apiIndex-16] - _ = x[apiQuery-17] - _ = x[apiRecalculateCaches-18] - _ = x[apiSchema-19] - _ = x[apiShardNodes-20] - _ = x[apiState-21] - _ = x[apiViews-22] - _ = x[apiApplySchema-23] - _ = x[apiStartTransaction-24] - _ = x[apiFinishTransaction-25] - _ = x[apiTransactions-26] - _ = x[apiGetTransaction-27] - _ = x[apiActiveQueries-28] - _ = x[apiPastQueries-29] - _ = x[apiIDReserve-30] - _ = x[apiIDCommit-31] - _ = x[apiIDReset-32] - _ = x[apiPartitionNodes-33] - _ = x[apiMutexCheck-34] + _ = x[apiFragmentData-8] + _ = x[apiTranslateData-9] + _ = x[apiFieldTranslateData-10] + _ = x[apiField-11] + _ = x[apiImport-12] + _ = x[apiImportValue-13] + _ = x[apiIndex-14] + _ = x[apiQuery-15] + _ = x[apiRecalculateCaches-16] + _ = x[apiSchema-17] + _ = x[apiShardNodes-18] + _ = x[apiState-19] + _ = x[apiViews-20] + _ = x[apiApplySchema-21] + _ = x[apiStartTransaction-22] + _ = x[apiFinishTransaction-23] + _ = x[apiTransactions-24] + _ = x[apiGetTransaction-25] + _ = x[apiActiveQueries-26] + _ = x[apiPastQueries-27] + _ = x[apiIDReserve-28] + _ = x[apiIDCommit-29] + _ = x[apiIDReset-30] + _ = x[apiPartitionNodes-31] + _ = x[apiMutexCheck-32] + _ = x[apiApplyChangeset-33] + _ = x[apiDeleteDataframe-34] } -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheckapiApplyChangesetapiDeleteDataframe" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 136, 152, 173, 181, 190, 204, 212, 220, 240, 249, 262, 270, 278, 292, 311, 331, 346, 363, 379, 393, 405, 416, 426, 443, 456, 473, 491} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/apply.go b/apply.go index cc9b769a0..12ce97bc2 100644 --- a/apply.go +++ b/apply.go @@ -15,6 +15,7 @@ import ( "github.com/apache/arrow/go/v10/arrow/memory" "github.com/gomem/gomem/pkg/dataframe" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/tracing" "github.com/molecula/featurebase/v3/vprint" "github.com/pkg/errors" @@ -117,7 +118,7 @@ func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx con } // executeApply executes a Apply() call. -func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) { +func (e *executor) executeApply(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) { if !e.dataframeEnabled { return nil, errors.New("Dataframe support not enabled") } @@ -189,7 +190,7 @@ func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter return indexResolver, nil } -func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) { +func (e *executor) executeApplyShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (value.Value, error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard") defer span.Finish() diff --git a/arrow.go b/arrow.go index 4538d8fc2..11949fb54 100644 --- a/arrow.go +++ b/arrow.go @@ -19,6 +19,7 @@ import ( "github.com/apache/arrow/go/v10/parquet/pqarrow" "github.com/gomem/gomem/pkg/dataframe" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" ) @@ -33,7 +34,7 @@ Arrow(ConstRow(columns=[2,4,6]),header=["fval"]) */ // executeApply executes a Arrow() call. -func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) { +func (e *executor) executeArrow(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) { if !e.dataframeEnabled { return nil, errors.New("Dataframe support not enabled") } @@ -353,7 +354,7 @@ func filterColumns(filters []string, table arrow.Table) arrow.Table { return array.NewTable(filterdSchema, cols, table.NumRows()) } -func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) { +func (e *executor) executeArrowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) { name := fmt.Sprintf("a. %v", shard) span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard") defer span.Finish() @@ -468,7 +469,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) return nil, err } defer rr.Close() - records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords()) + records := make([]arrow.Record, rr.NumRecords()) i := 0 for { rec, err := rr.Read() diff --git a/catcher.go b/catcher.go deleted file mode 100644 index 18c33a0d4..000000000 --- a/catcher.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "github.com/molecula/featurebase/v3/roaring" - txkey "github.com/molecula/featurebase/v3/short_txkey" - "github.com/molecula/featurebase/v3/vprint" -) - -// catcher is useful to report error locations with a -// Stack dump before the complexity -// of the executor_test swallows up -// the location of a PanicOn. -type catcherTx struct { - b Tx -} - -func newCatcherTx(b Tx) *catcherTx { - return &catcherTx{b: b} -} - -func init() { - // keep golangci-lint happy - _ = newCatcherTx -} - -var _ Tx = (*catcherTx)(nil) - -func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) -} - -func (c *catcherTx) Rollback() { - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - c.b.Rollback() -} - -func (c *catcherTx) Commit() error { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Commit() -} - -func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.RoaringBitmap(index, field, view, shard) -} - -func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Container(index, field, view, shard, key) -} - -func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.PutContainer(index, field, view, shard, key, rc) -} - -func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.RemoveContainer(index, field, view, shard, key) -} - -func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Add(index, field, view, shard, a...) -} - -func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Remove(index, field, view, shard, a...) -} - -func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Contains(index, field, view, shard, key) -} - -func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) -} - -func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Count(index, field, view, shard) -} - -func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Max(index, field, view, shard) -} - -func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.Min(index, field, view, shard) -} - -func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.CountRange(index, field, view, shard, start, end) -} - -func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { - - defer func() { - if r := recover(); r != nil { - vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack()) - vprint.PanicOn(r) - } - }() - return c.b.OffsetRange(index, field, view, shard, offset, start, end) -} - -func (c *catcherTx) Type() string { - return c.b.Type() -} - -func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} - -func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) { - return c.b.ApplyRewriter(index, field, view, shard, ckey, filter) -} - -func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { - return c.b.GetSortedFieldViewList(idx, shard) -} - -func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} diff --git a/ctl/rbf_pages.go b/ctl/rbf_pages.go index ce7cc2689..cd39d29e3 100644 --- a/ctl/rbf_pages.go +++ b/ctl/rbf_pages.go @@ -9,7 +9,6 @@ import ( "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/rbf" - "github.com/molecula/featurebase/v3/txkey" ) // RBFPagesCommand represents a command for printing a list of RBF page metadata. @@ -100,21 +99,21 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error { case *rbf.LeafPageInfo: fmt.Fprintf(cmd.stdout, "%-10s ", "leaf") if cmd.WithTree { - fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree)) + fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree)) } fmt.Fprintf(cmd.stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BranchPageInfo: fmt.Fprintf(cmd.stdout, "%-10s ", "branch") if cmd.WithTree { - fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree)) + fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree)) } fmt.Fprintf(cmd.stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BitmapPageInfo: fmt.Fprintf(cmd.stdout, "%-10s ", "bitmap") if cmd.WithTree { - fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree)) + fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree)) } fmt.Fprintf(cmd.stdout, "-\n") @@ -132,12 +131,3 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error { return nil } - -func prefixToString(s string) (ret string) { - defer func() { - if err := recover(); err != nil { - ret = s - } - }() - return txkey.PrefixToString([]byte(s)) -} diff --git a/ctl/restore.go b/ctl/restore.go index 47aa2ebf5..68ba165c8 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -2,6 +2,7 @@ package ctl import ( + "bytes" "context" "crypto/tls" "encoding/json" @@ -20,6 +21,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/server" "github.com/pkg/errors" @@ -350,34 +352,15 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er for _, node := range nodes { logger.Printf("shard %v %v", shard, indexName) - f, err := os.Open(filename) + data, err := os.ReadFile(filename) if err != nil { return err } - defer f.Close() - - url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard)) - req, err := retryablehttp.NewRequest("POST", url, f) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", "application/octet-stream") - - token, ok := authn.GetAccessToken(ctx) - if ok && token != "" { - req.Header.Set("Authorization", token) - } - - client := cmd.newClient() - resp, err := client.Do(req) - if err != nil { - return err - } else if err := resp.Body.Close(); err != nil { - return err - } else if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } + client := pilosa.NewInternalClientFromURI(&node.URI, + pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(time.Second*3)), + pilosa.WithClientRetryPeriod(cmd.RetryPeriod), + pilosa.WithSerializer(proto.Serializer{})) + return client.RestoreShard(ctx, indexName, shard, bytes.NewBuffer(data)) } return nil } diff --git a/dataframe_test.go b/dataframe_test.go index 1e041356b..2736ae9a9 100644 --- a/dataframe_test.go +++ b/dataframe_test.go @@ -12,6 +12,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" + "github.com/stretchr/testify/require" ) func TestExecutor_Apply(t *testing.T) { @@ -39,16 +40,12 @@ func TestExecutor_Apply(t *testing.T) { req.Field = fieldName req.ColumnIDs = []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} req.Values = []int64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - qcx := api.Txf().NewQcx() - defer qcx.Abort() + qcx := mustIndexQueryContext(t, api, req.Index, req.Shard) if err := api.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } + require.Nil(t, qcx.Commit()) t.Run("dataframe ingest", func(t *testing.T) { // func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetRequest, shard uint64) (map[string]interface{}, error) { diff --git a/dax/computer/logmessage.go b/dax/computer/logmessage.go index 74d33031f..7a18265ac 100644 --- a/dax/computer/logmessage.go +++ b/dax/computer/logmessage.go @@ -180,7 +180,6 @@ type ImportRoaringMessage struct { Shard uint64 `json:"shard"` Clear bool `json:"clear"` Action string `json:"action"` // [set, clear, overwrite] - Block int `json:"block"` Views map[string][]byte `json:"views"` UpdateExistence bool `json:"update-existence"` } diff --git a/dbshard.go b/dbshard.go deleted file mode 100644 index 76c483591..000000000 --- a/dbshard.go +++ /dev/null @@ -1,702 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - - rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" - txkey "github.com/molecula/featurebase/v3/short_txkey" - "github.com/molecula/featurebase/v3/storage" - "github.com/pkg/errors" - - "github.com/molecula/featurebase/v3/vprint" -) - -var _ = sort.Sort - -const ( - // backendsDir is the default backends directory used to store the - // data for each backend. - backendsDir = "backends" -) - -// types to support a database file per shard - -type DBHolder struct { - Index map[string]*DBIndex -} - -func NewDBHolder() *DBHolder { - return &DBHolder{ - Index: make(map[string]*DBIndex), - } -} - -type DBIndex struct { - Shard map[uint64]*DBShard -} - -type DBWrapper interface { - NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) - Close() error - DeleteFragment(index, field, view string, shard uint64, frag interface{}) error - DeleteField(index, field, fieldPath string) error - OpenListString() string - Path() string - HasData() (has bool, err error) - SetHolder(h *Holder) - //needed for restore - CloseDB() error - OpenDB() error -} - -type DBRegistry interface { - OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) -} - -type DBShard struct { - HolderPath string - - Index string - Shard uint64 - Open bool - - typ txtype - styp string - - W DBWrapper - ParentDBIndex *DBIndex - - idx *Index - per *DBPerShard - - closed bool -} - -func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) { - if index != dbs.Index { - return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index) - } - if shard != dbs.Shard { - return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard) - } - return dbs.W.DeleteFragment(index, field, view, shard, frag) -} - -func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - if index != dbs.Index { - return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index) - } - return dbs.W.DeleteField(index, field, fieldPath) -} - -func (dbs *DBShard) Close() (err error) { - dbs.closed = true - return dbs.W.Close() -} - -func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - if initialIndexName != dbs.Index { - return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName) - } - if o.dbs != dbs { - return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs) - } - if o.Shard != dbs.Shard { - return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)) - } - return dbs.W.NewTx(write, initialIndexName, o) -} - -type flatkey struct { - index string - shard uint64 -} - -type DBPerShard struct { - Mu sync.Mutex - - HolderDir string - - dbh *DBHolder - - // just flat, not buried within the Node heirarchy. - // Easily see how many we have. - Flatmap map[flatkey]*DBShard - - typ txtype - - txf *TxFactory - holder *Holder - - // cache the shards per index to avoid excessive - // directory scans of the index directory. - // Keep it up-to-date as we add shards to avoid doing - // a filesystem rescan on new shard creation. - // - // index -> *shardSet - index2shards map[string]*shardSet - - StorageConfig *storage.Config - RBFConfig *rbfcfg.Config -} - -func newIndex2Shards() (r map[string]*shardSet) { - r = make(map[string]*shardSet) - return -} - -type shardSet struct { - shardsMap map[uint64]struct{} - shardsVer int64 // increment with each change. - - // give out readonly to repeated consumers if - // readonlyVer == shardsVer - readonly map[uint64]struct{} - readonlyVer int64 -} - -func (a *shardSet) unionInPlace(b *shardSet) { - shards := b.CloneMaybe() - for shard := range shards { - a.add(shard) - } -} - -func (a *shardSet) equals(b *shardSet) bool { - if len(a.shardsMap) != len(b.shardsMap) { - return false - } - for shardInA := range a.shardsMap { - _, ok := b.shardsMap[shardInA] - if !ok { - return false - } - } - return true - -} - -func (a *shardSet) shards() []uint64 { - s := make([]uint64, 0, len(a.shardsMap)) - for si := range a.shardsMap { - s = append(s, si) - } - return s -} - -func (ss *shardSet) String() (r string) { - r = "[" - for k := range ss.shardsMap { - r += fmt.Sprintf("%v, ", k) - } - r += "]" - return -} - -func (ss *shardSet) add(shard uint64) { - _, already := ss.shardsMap[shard] - if !already { - ss.shardsMap[shard] = struct{}{} - ss.shardsVer++ - } -} - -// CloneMaybe maintains a re-usable readonly version -// ss.shards that can be returned to multiple goroutine -// reads as it will never change. A copy is only made -// once for each change in the shard set. -func (ss *shardSet) CloneMaybe() map[uint64]struct{} { - - if ss.readonlyVer == ss.shardsVer { - return ss.readonly - } - - // readonlyVer is out of date. - // readonly needs update. We cannot - // modify the readonly map in place; - // must make a fully new copy here. - ss.readonly = make(map[uint64]struct{}) - - for k := range ss.shardsMap { - ss.readonly[k] = struct{}{} - } - ss.readonlyVer = ss.shardsVer - return ss.readonly -} - -func newShardSet() *shardSet { - return &shardSet{ - shardsMap: make(map[uint64]struct{}), - } -} - -func (per *DBPerShard) LoadExistingDBs() (err error) { - idxs := per.holder.Indexes() - - for _, idx := range idxs { - - shardset, err := per.txf.GetShardsForIndex(idx, "", true) - if err != nil { - return err - } - for shard := range shardset { - _, err := per.GetDBShard(idx.name, shard, idx) - if err != nil { - return errors.Wrap(err, "DBPerShard.LoadExistingDBs GetDBShard()") - } - } - } - return -} - -func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) { - if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil { - vprint.PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here") - } - - d = &DBPerShard{ - typ: typ, - HolderDir: holderDir, - holder: holder, - dbh: NewDBHolder(), - Flatmap: make(map[flatkey]*DBShard), - txf: txf, - index2shards: newIndex2Shards(), - StorageConfig: holder.cfg.StorageConfig, - RBFConfig: holder.cfg.RBFConfig, - } - return -} - -func (per *DBPerShard) DeleteIndex(index string) (err error) { - - per.Mu.Lock() - defer per.Mu.Unlock() - - dbi, ok := per.dbh.Index[index] - if !ok { - // since we lazily make indexes upon use by a Tx now, we won't - // have an index for server/ TestQuerySQLUnary/test-20 to delete. - // Don't freak out. Just return nil. - return nil - } - for _, dbs := range dbi.Shard { - err = dbs.Close() - if err != nil { - return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()") - } - path := dbs.pathForType(per.typ) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) - } - delete(per.index2shards, index) - } - - // allow the index to be created again anew. - delete(per.dbh.Index, index) - - return -} - -func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - per.Mu.Lock() - defer func() { - if fieldPath != "" { - _ = os.RemoveAll(fieldPath) - } - per.Mu.Unlock() - }() - - dbi, ok := per.dbh.Index[index] - if !ok { - // TestIndex_Existence_Delete in index_internal_test.go - // will call us without having ever created a Tx or DB, - // so we can't complain here. - return nil - } - for _, dbs := range dbi.Shard { - if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil { - err = errors.Wrap(e, "DeleteFieldFromStore()") - } - } - return err -} - -func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error { - - idx := per.txf.holder.Index(index) - dbs, err := per.GetDBShard(index, shard, idx) - if err != nil { - return err - } - return dbs.DeleteFragment(index, field, view, shard, frag) -} - -// if you know the shard, you can use this -// pathForType and prefixForType must be kept in sync! -func (dbs *DBShard) pathForType(ty txtype) string { - // top level paths will end in "@@" - - // what here for roaring? well, roaringRegistrar.OpenDBWrapper() - // is a no-op anyhow. so doesn't need to be correct atm. - - path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard) - return path -} - -// if you don't know the shard, you have to use this. -// prefixForType and pathForType must be kept in sync! -func (per *DBPerShard) prefixForType(idx *Index, ty txtype) string { - // top level paths will end in "@@" - return per.HolderDir + sep + idx.name + sep + backendsDir + sep + ty.DirectoryName() + sep -} - -var ErrNoData = fmt.Errorf("no data") - -// keep our cache of shards up-to-date in memory; after the initial -// directory scan, this is all we should we need. Prevents us from -// doing additional, expensive, directory scans. -// -// Caller must hold per.Mu.Lock() already. -func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) { - shardset, ok := per.index2shards[dbs.Index] - if !ok { - shardset = newShardSet() - per.index2shards[dbs.Index] = shardset - } - // INVAR: shardset is present, not nil; a map that can be added to. - shardset.add(dbs.Shard) -} - -func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { - per.Mu.Lock() - defer per.Mu.Unlock() - return per.unprotectedGetDBShard(index, shard, idx) -} - -func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { - - dbi, ok := per.dbh.Index[index] - if !ok { - dbi = &DBIndex{ - Shard: make(map[uint64]*DBShard), - } - per.dbh.Index[index] = dbi - } - dbs, ok = dbi.Shard[shard] - if dbs != nil && dbs.closed { - vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) - } - if !ok { - dbs = &DBShard{ - typ: per.typ, - ParentDBIndex: dbi, - Index: index, - Shard: shard, - HolderPath: per.HolderDir, - idx: idx, - per: per, - } - dbs.styp = per.typ.String() - dbi.Shard[shard] = dbs - per.updateIndex2ShardCacheWithNewShard(dbs) - } - if !dbs.Open { - var registry DBRegistry - switch dbs.typ { - case rbfTxn: - registry = globalRbfDBReg - registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) - default: - vprint.PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ)) - } - path := dbs.pathForType(dbs.typ) - w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) - vprint.PanicOn(err) - h := idx.Holder() - w.SetHolder(h) - dbs.Open = true - per.Flatmap[flatkey{index: index, shard: shard}] = dbs - dbs.W = w - } - return dbs, nil -} - -func (per *DBPerShard) Close() (err error) { - per.Mu.Lock() - defer per.Mu.Unlock() - - for _, dbi := range per.dbh.Index { - for _, dbs := range dbi.Shard { - err = dbs.Close() - vprint.PanicOn(err) - } - } - return -} - -// DBPerShardGetShardsForIndex returns the shards for idx. -// If requireData, we open the database and see that it has a key, rather -// than assume that the database file presence is enough. -func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]struct{}, error) { - return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData) -} - -// requireData means open the database file and verify that at least one key is set. -// The returned sliceOfShards should not be modified. We will cache it for subsequent -// queries. -// -// when a new DBShard is made, we will update the list of shards then. Thus -// the per.index2shard should always be up to date AFTER the first call here. -func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]struct{}, err error) { - - // use the cache, always - per.Mu.Lock() - defer per.Mu.Unlock() - - i2ss := per.index2shards - - ss, ok := i2ss[idx.name] - if ok { - return ss.CloneMaybe(), nil - } - // INVAR: cache miss, and index2shards[ty] exists. - - // gotta read shards from disk directory layout. - setOfShards := newShardSet() - per.index2shards[idx.name] = setOfShards - - // Upon return, cache the setOfShards value and reuse it next time - - path := per.prefixForType(idx, ty) - - ignoreEmpty := false - includeRoot := true - dbf, err := listDirUnderDir(path, includeRoot, ignoreEmpty) - vprint.PanicOn(err) - - for _, nm := range dbf { - base := filepath.Base(nm) - - // We're only interested in "shard.*" files, so skip everything else. - const shardPrefix = "shard." - const lenOfShardPrefix = len(shardPrefix) - if !strings.HasPrefix(base, shardPrefix) { - continue - } - - // Parse filename into integer. - shard, err := strconv.ParseUint(base[lenOfShardPrefix:], 10, 64) - if err != nil { - vprint.PanicOn(err) - continue - } - - // exclude those without data? - hasData := false - - if requireData { - hasData, err = per.unprotectedTypedIndexShardHasData(ty, idx, shard) - if err != nil { - return nil, err - } - if hasData { - setOfShards.add(shard) - } - } else { - // file presence is enough - setOfShards.add(shard) - } - } - return setOfShards.CloneMaybe(), nil -} - -func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) { - if ty != per.typ { - return - } - - // make the dbs if it doesn't get exist - dbs, err := per.unprotectedGetDBShard(idx.name, shard, idx) - if err != nil { - return false, errors.Wrap(err, fmt.Sprintf("DBPerShard.TypedIndexShardHasData() "+ - "per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String())) - } - - return dbs.W.HasData() -} - -func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) { - if !dirExists(root) { - return - } - - n := len(root) + 1 - if includeRoot { - n = 0 - } - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if len(path) < n { - // ignore - } else { - if info == nil { - // re-opening an RBF database hit this, racing with a directory rename. - // Don't freak out. - return nil - } - if !info.IsDir() { - // ignore files - } else { - if ignoreEmpty && info.Size() == 0 { - return nil - } - files = append(files, path[n:]) - } - } - return nil - }) - return -} - -type FieldView2Shards struct { - // field -> view -> *shardSet - m map[string]map[string]*shardSet -} - -func (vs *FieldView2Shards) getViewsForField(field string) map[string]*shardSet { - return vs.m[field] -} - -func (vs *FieldView2Shards) addViewShardSet(fv txkey.FieldView, ss *shardSet) { - - f, ok := vs.m[fv.Field] - if !ok { - f = make(map[string]*shardSet) - vs.m[fv.Field] = f - } - // INVAR: f is ready to take ss. - - // existing stuff to merge with? - prior, ok := f[fv.View] - if !ok { - f[fv.View] = ss - return - } - // merge ss and prior. No need to put the union back into f[fv.View] - // because prior is a pointer. - prior.unionInPlace(ss) -} - -func (a *FieldView2Shards) equals(b *FieldView2Shards) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - if len(a.m) != len(b.m) { - return false - } - for field, viewmapA := range a.m { - viewmapB, ok := b.m[field] - if !ok { - return false - } - if len(viewmapB) != len(viewmapA) { - return false - } - for k, va := range viewmapA { - vb, ok := viewmapB[k] - if !ok { - return false - } - if !va.equals(vb) { - return false - } - } - } - return true -} - -func NewFieldView2Shards() *FieldView2Shards { - return &FieldView2Shards{ - m: make(map[string]map[string]*shardSet), // expected response from GetView2ShardMapForIndex - } -} - -func (vs *FieldView2Shards) addShard(fv txkey.FieldView, shard uint64) { - viewmap, ok := vs.m[fv.Field] - if !ok { - viewmap = make(map[string]*shardSet) - vs.m[fv.Field] = viewmap - } - ss, ok := viewmap[fv.View] - if !ok { - ss = newShardSet() - viewmap[fv.View] = ss - } - ss.add(shard) -} - -func (vs *FieldView2Shards) String() (r string) { - r = "\n" - for field, viewmap := range vs.m { - for view, shards := range viewmap { - r += fmt.Sprintf("field '%v' view:'%v' shards:%v\n", field, view, shards) - } - } - r += "\n" - return -} - -func (vs *FieldView2Shards) removeField(name string) { - delete(vs.m, name) -} - -func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) { - ty := per.typ - - switch ty { - default: - vs = NewFieldView2Shards() - - shardMap, err := per.TypedDBPerShardGetShardsForIndex(ty, idx, "", true) - if err != nil { - return nil, err - } - - for shard := range shardMap { - dbs, err := per.GetDBShard(idx.name, shard, idx) - if err != nil { - return nil, errors.Wrap(err, "DBPerShard.GetFieldView2ShardsMapForIndex GetDBShard()") - } - fieldviews, err := dbs.AllFieldViews() - if err != nil { - return nil, errors.Wrap(err, "DBPerShard.GetFieldView2ShardsMapForIndex dbs.AllFieldViews()") - } - for _, fv := range fieldviews { - vs.addShard(fv, shard) - } - } - } - - return -} - -func (dbs *DBShard) AllFieldViews() (fvs []txkey.FieldView, err error) { - - tx, err := dbs.NewTx(!writable, dbs.idx.name, Txo{Write: !writable, Shard: dbs.Shard, Index: dbs.idx, dbs: dbs}) - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("dbshard.NewTx for index '%v', shard %v", dbs.idx.name, dbs.Shard)) - } - defer tx.Rollback() - return tx.GetSortedFieldViewList(dbs.idx, dbs.Shard) -} diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go deleted file mode 100644 index bf2dae261..000000000 --- a/dbshard_internal_test.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/molecula/featurebase/v3/rbf" - "github.com/molecula/featurebase/v3/shardwidth" - txkey "github.com/molecula/featurebase/v3/short_txkey" - "github.com/molecula/featurebase/v3/testhook" - . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck -) - -// Shard per db evaluation -func TestShardPerDB_SetBit(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx - defer f.Clean(t) - - // Set bits on the fragment. - if _, err := f.setBit(tx, 120, 1); err != nil { - t.Fatal(err) - } else if _, err := f.setBit(tx, 120, 6); err != nil { - t.Fatal(err) - } else if _, err := f.setBit(tx, 121, 0); err != nil { - t.Fatal(err) - } - // should have two containers set in the fragment. - - // Verify counts on rows. - if n := f.mustRow(tx, 120).Count(); n != 2 { - t.Fatalf("unexpected count: %d", n) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - t.Fatalf("unexpected count: %d", n) - } - - // commit the change, and verify it is still there - PanicOn(tx.Commit()) - - // Close and reopen the fragment & verify the data. - err := f.Reopen() // roaring data not being flushed? red on roaring - if err != nil { - t.Fatal(err) - } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - if n := f.mustRow(tx, 120).Count(); n != 2 { - t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - t.Fatalf("unexpected count (reopen): %d", n) - } -} - -// test that we find all *local* shards -func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { - tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly") - PanicOn(err) - defer os.RemoveAll(tmpdir) - - v2s := NewFieldView2Shards() - stdShardSet := newShardSet() - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - stdShardSet.add(shard) - } - for _, field := range []string{"f", "_exists"} { - v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) - } - - for _, src := range []string{"rbf"} { - holder := newTestHolder(t) - - index := "rick" - idx := makeSampleRoaringDir(t, tmpdir, index, src, 1, holder, v2s) - if idx == nil { - idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index) - PanicOn(err) - } - std := "rick/fields/f/views/standard" - - shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false) - PanicOn(err) - - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - if _, ok := shards[shard]; !ok { - t.Fatalf("missing shard=%v from shards='%#v'", shard, shards) - } - } - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) - fvs, err := tx.GetSortedFieldViewList(idx, shard) - PanicOn(err) - // expect these same two field/views for all 6 shards - expect0 := txkey.FieldView{Field: "_exists", View: "standard"} - expect1 := txkey.FieldView{Field: "f", View: "standard"} - if len(fvs) != 2 { - t.Fatalf("fvs should be len 2, got '%#v' (%s)", fvs, src) - } - if fvs[0] != expect0 { - t.Fatalf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]) - } - if fvs[1] != expect1 { - t.Fatalf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]) - } - tx.Rollback() - } - } -} - -// data for Test_DBPerShard_GetShardsForIndex -var sampleRoaringDirList = map[string]string{"roaring": ` -rick/fields/f/views/standard/fragments/215.cache -rick/fields/f/views/standard/fragments/221.cache -rick/fields/f/views/standard/fragments/223.cache -rick/fields/f/views/standard/fragments/93.cache -rick/fields/f/views/standard/fragments/217.cache -rick/fields/f/views/standard/fragments/219.cache -rick/fields/f/views/standard/fragments/217 -rick/fields/f/views/standard/fragments/219 -rick/fields/f/views/standard/fragments/215 -rick/fields/f/views/standard/fragments/221 -rick/fields/f/views/standard/fragments/223 -rick/fields/f/views/standard/fragments/93 -rick/fields/_exists/views/standard/fragments/221 -rick/fields/_exists/views/standard/fragments/215 -rick/fields/_exists/views/standard/fragments/217 -rick/fields/_exists/views/standard/fragments/93 -rick/fields/_exists/views/standard/fragments/219 -rick/fields/_exists/views/standard/fragments/223 -`, - "rbf": ` -rick/backends/backend-rbf/shard.0093-rbf -rick/backends/backend-rbf/shard.0215-rbf -rick/backends/backend-rbf/shard.0217-rbf -rick/backends/backend-rbf/shard.0219-rbf -rick/backends/backend-rbf/shard.0221-rbf -rick/backends/backend-rbf/shard.0223-rbf -`, -} - -func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) { - shards := []uint64{0, 93, 215, 217, 219, 221, 223} - fns := strings.Split(sampleRoaringDirList[backend], "\n") - firstDone := false - - for i, fn := range fns { - // This check is here because in sampleRoaringDirList, the first entry - // of each map value is a line feed, so the strings.Split() above - // results in a blank entry for the first item. This means that the - // slice of shards above has an initial entry "0" which is not used. - if fn == "" { - continue - } - var shard uint64 - switch backend { - case "rbf": - shard = shards[i] - - idx = helperCreateDBShard(h, index, shard) - - // first time only, we'll actually make all the shards at this point because - // view2shards has them all anyway. - if !firstDone { - firstDone = true - makeTxTestDBWithViewsShards(t, h, idx, view2shards) - } - continue - case "roaring": - default: - t.Fatalf("invalid backend: %s", backend) - } - - path := root + sep + filepath.Dir(fn) - PanicOn(os.MkdirAll(path, 0755)) - fd, err := os.Create(root + sep + fn) - PanicOn(err) - if minBytes > 0 { - _, err := fd.Write(make([]byte, minBytes)) - PanicOn(err) - } - fd.Close() - } - return -} - -func helperCreateDBShard(h *Holder, index string, shard uint64) *Index { - idx, err := h.CreateIndexIfNotExists(index, "", IndexOptions{}) - PanicOn(err) - // TODO: It's not clear that this is actually doing anything. - dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx) - PanicOn(err) - _ = dbs - return idx -} - -// keep the ocd linter happy -var _ = makeRBFtestDB - -func makeRBFtestDB(path string, h *Holder, shard uint64) { - i := uint64(1) - - db := rbf.NewDB(path, nil) - err := db.Open() - PanicOn(err) - defer db.Close() - - tx, err := db.Begin(true) - PanicOn(err) - - err = tx.CreateBitmap("x") - PanicOn(err) - - _, err = tx.Add("x", i) - PanicOn(err) - - err = tx.Commit() - PanicOn(err) -} - -func makeTxTestDBWithViewsShards(tb testing.TB, holder *Holder, idx *Index, exp *FieldView2Shards) { - - // TODO(jea): need date time quantum views!! - for field, viewmap := range exp.m { - for view, shset := range viewmap { - - ss := shset.CloneMaybe() - for shard := range ss { - - // simply write 1 bit to each shard to force its creation. - bits := []uint64{(shard << shardwidth.Exponent) + 1} - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) - changeCount, err := tx.Add(idx.name, field, view, shard, bits...) - PanicOn(err) - if changeCount != len(bits) { - tb.Fatalf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount) - } - - PanicOn(tx.Commit()) - } - } - } - -} - -// test that rbf can give us a map[view]*shardSet -func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) { - holder := newTestHolder(t) - - index := "rick" - field := "f" - - idx, err := holder.CreateIndex(index, "", IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - - exp := NewFieldView2Shards() - - stdShardSet := newShardSet() - stdShardSet.add(12) - stdShardSet.add(15) - exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) - - hrShardSet := newShardSet() - hrShardSet.add(7) - exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard_2019092416"}, hrShardSet) - - makeTxTestDBWithViewsShards(t, holder, idx, exp) - - // setup is done - view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx) - PanicOn(err) - - // compare against setup - if !view2shard.equals(exp) { - t.Fatalf("expected '%v' but got view2shard '%v'", exp, view2shard) - } -} diff --git a/dbshard_test.go b/dbshard_test.go deleted file mode 100644 index 533d9e9f2..000000000 --- a/dbshard_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa_test - -import ( - "context" - "fmt" - "testing" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/test" - . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck -) - -func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) { - - c := test.MustRunCluster(t, 1) - defer c.Close() - - m0 := c.GetNode(0) - - t.Run("RowIDColumnKey", func(t *testing.T) { - ctx := context.Background() - indexName := c.Idx() - fieldName := "f" - - index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - if index.CreatedAt() == 0 { - t.Fatal("index createdAt is empty") - } - - field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) - if err != nil { - t.Fatalf("creating field: %v", err) - } - if field.CreatedAt() == 0 { - t.Fatal("field createdAt is empty") - } - - rowID := uint64(1) - timestamp := int64(0) - - // Generate some keyed records. - rowIDs := []uint64{} - timestamps := []int64{} - for i := 1; i <= 10; i++ { - rowIDs = append(rowIDs, rowID) - timestamps = append(timestamps, timestamp) - } - - // Keys are sharded so ordering is not guaranteed. - colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} - - // Import data with keys to the primary and verify that it gets - // translated and forwarded to the owner of shard 0 - req := &pilosa.ImportRequest{ - Index: indexName, - IndexCreatedAt: index.CreatedAt(), - Field: fieldName, - FieldCreatedAt: field.CreatedAt(), - Shard: 0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored. - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, - } - - qcx := m0.API.Txf().NewQcx() - if err := m0.API.Import(ctx, qcx, req); err != nil { - t.Fatal(err) - } - PanicOn(qcx.Finish()) - - //select {} - - pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) - - // Query node0. - res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) - if err != nil { - t.Fatal(err) - } - keys := res.Results[0].(*pilosa.Row).Keys - if !sameStringSlice(keys, colKeys) { - t.Fatalf("unexpected column keys: %#v", keys) - } - }) - -} diff --git a/delete_test.go b/delete_test.go index b92bc192b..7c088d97e 100644 --- a/delete_test.go +++ b/delete_test.go @@ -113,7 +113,48 @@ func TestExecutor_DeleteRecords(t *testing.T) { } require := require.New(t) t.Run("DeleteRecords", func(t *testing.T) { - + t.Run("DeleteRace", func(t *testing.T) { + setup(t, require, c) + defer tearDown(t, require, c) + for i := 0; i < 100; i++ { + // imagine that we start with bsi set to 3 in column 0. + // We then execute two operations: + // (1) we set bsi to 1 in column 0 + // (2) we delete everything with bsi > 2 + // No matter which order these happen in, we should see + // bsi set to 1 in column 0. + // If the set happens first, the delete doesn't touch it. + // If the set happens second, the delete deletes the previous + // value, then the set happens. + // Let's find out... + c.ImportIntID(t, indexName, "bsi", []test.IntID{ + {ID: 0, Val: 3}, + }) + ch := make(chan error) + go func() { + // Make sure we close the channel even if we're failing out of the test. + defer close(ch) + // We don't actually care whether the delete succeeds or fails... + _ = c.Query(t, indexName, `Delete(Row(bsi>2))`) + // we don't try to handle an error from that, at this time. + }() + c.ImportIntID(t, indexName, "bsi", []test.IntID{ + {ID: 0, Val: 1}, + }) + // wait for the async delete + <-ch + resp := c.Query(t, indexName, `Row(bsi<2)`) + // we expect to always find 0 in this row + row, ok := resp.Results[0].(*pilosa.Row) + if !ok { + t.Fatalf("expected row return") + } + cols := row.Columns() + if len(cols) < 1 || cols[0] != 0 { + t.Fatalf("expected columns including 0, got %d on try %d", cols, i) + } + } + }) t.Run("Delete", func(t *testing.T) { setup(t, require, c) defer tearDown(t, require, c) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 2f8017a3a..129237d83 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -450,7 +450,6 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * FieldCreatedAt: m.FieldCreatedAt, Clear: m.Clear, Action: m.Action, - Block: uint64(m.Block), Views: views, UpdateExistence: m.UpdateExistence, } @@ -1191,7 +1190,6 @@ func (s Serializer) decodeImportRoaringRequest(pb *pb.ImportRoaringRequest, m *p } m.Clear = pb.Clear m.Action = pb.Action - m.Block = int(pb.Block) m.Views = views m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt diff --git a/executor.go b/executor.go index 4488ce628..a1636d58e 100644 --- a/executor.go +++ b/executor.go @@ -22,6 +22,7 @@ import ( "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/proto" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/task" @@ -235,13 +236,17 @@ func (e *executor) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q *pq // Can't do NewTx() this high up, because we need a specific shard. // So start a qcx with a TxGroup and pass it down. - var qcx *Qcx + var qcx qc.QueryContext + var err error if needWriteTxn { - qcx = idx.holder.txf.NewWritableQcx() + qcx, err = e.Holder.NewIndexQueryContext(ctx, index) } else { - qcx = idx.holder.txf.NewQcx() + qcx, err = e.Holder.NewQueryContext(ctx) } - defer qcx.Abort() + if err != nil { + return resp, err + } + defer qcx.Release() results, err := e.execute(ctx, qcx, index, q, shards, opt) if err != nil { @@ -272,9 +277,10 @@ func (e *executor) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q *pq // Must copy out of Tx data before Commiting, because it will become invalid afterwards. respSafeNoTxData := safeCopy(resp) - // Commit transactions if writing; else let the defer grp.Abort do the rollbacks. + // Commit transactions if writing. (Non-writes have no writes to + // commit, and we already deferred a Release) if needWriteTxn { - if err := qcx.Finish(); err != nil { + if err := qcx.Commit(); err != nil { return respSafeNoTxData, err } } @@ -359,7 +365,7 @@ func safeCopy(resp QueryResponse) (out QueryResponse) { // handlePreCalls traverses the call tree looking for calls that need // precomputed values (e.g. Distinct, UnionRows, ConstRow...). -func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) error { +func (e *executor) handlePreCalls(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) error { if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) if idx >= 0 && idx < int64(len(opt.EmbeddedData)) { @@ -458,7 +464,7 @@ func (e *executor) dumpPrecomputedCalls(ctx context.Context, c *pql.Call) { } // handlePreCallChildren handles any pre-calls in the children of a given call. -func (e *executor) handlePreCallChildren(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) error { +func (e *executor) handlePreCallChildren(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) error { for i := range c.Children { if err := ctx.Err(); err != nil { return err @@ -485,7 +491,7 @@ func (e *executor) handlePreCallChildren(ctx context.Context, qcx *Qcx, index st return nil } -func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { +func (e *executor) execute(ctx context.Context, qcx qc.QueryContext, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.execute") defer span.Finish() @@ -627,7 +633,7 @@ func (vc *ValCount) Cleanup() { } // preprocessQuery expands any calls that need preprocessing. -func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*pql.Call, error) { +func (e *executor) preprocessQuery(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*pql.Call, error) { switch c.Name { case "All": _, hasLimit, err := c.UintArg("limit") @@ -674,7 +680,7 @@ func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, } // executeCall executes a call. -func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeCall") defer span.Finish() @@ -870,7 +876,7 @@ func (e *executor) validateTimeCallArgs(c *pql.Call, indexName string) error { return nil } -func (e *executor) executeOptionsCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeOptionsCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeOptionsCall") defer span.Finish() @@ -894,7 +900,7 @@ func (e *executor) executeOptionsCall(ctx context.Context, qcx *Qcx, index strin } // executeIncludesColumnCall executes an IncludesColumn() call. -func (e *executor) executeIncludesColumnCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { +func (e *executor) executeIncludesColumnCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { // Get the shard containing the column, since that's the only // shard that needs to execute this query. var shard uint64 @@ -930,7 +936,7 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, qcx *Qcx, inde } // executeFieldValueCall executes a FieldValue() call. -func (e *executor) executeFieldValueCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { +func (e *executor) executeFieldValueCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { fieldName, ok := c.Args["field"].(string) if !ok || fieldName == "" { return ValCount{}, ErrFieldRequired @@ -983,7 +989,7 @@ func (e *executor) executeFieldValueCall(ctx context.Context, qcx *Qcx, index st return other, nil } -func (e *executor) executeFieldValueCallShard(ctx context.Context, qcx *Qcx, field *Field, col uint64, shard uint64) (_ ValCount, err0 error) { +func (e *executor) executeFieldValueCallShard(ctx context.Context, qcx qc.QueryContext, field *Field, col uint64, shard uint64) (_ ValCount, err0 error) { value, exists, err := field.Value(qcx, col) if err != nil { return ValCount{}, errors.Wrap(err, "getting field value") @@ -1014,7 +1020,7 @@ func (e *executor) executeFieldValueCallShard(ctx context.Context, qcx *Qcx, fie } // executeLimitCall executes a Limit() call. -func (e *executor) executeLimitCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { +func (e *executor) executeLimitCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { bitmapCall := c.Children[0] limit, hasLimit, err := c.UintArg("limit") @@ -1090,7 +1096,7 @@ func (e *executor) executeLimitCall(ctx context.Context, qcx *Qcx, index string, } // executeIncludesColumnCallShard -func (e *executor) executeIncludesColumnCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, column uint64) (_ bool, err error) { +func (e *executor) executeIncludesColumnCallShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64, column uint64) (_ bool, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeIncludesColumnCallShard") defer span.Finish() @@ -1106,7 +1112,7 @@ func (e *executor) executeIncludesColumnCallShard(ctx context.Context, qcx *Qcx, } // executeSum executes a Sum() call. -func (e *executor) executeSum(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { +func (e *executor) executeSum(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeSum") defer span.Finish() @@ -1160,7 +1166,7 @@ func (e *executor) executeSum(ctx context.Context, qcx *Qcx, index string, c *pq // executeDistinct executes a Distinct call on a field. It returns a // SignedRow for int fields and a *Row for set/mutex/time fields. -func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeDistinct(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeDistinct") defer span.Finish() @@ -1212,7 +1218,7 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, } // executeMin executes a Min() call. -func (e *executor) executeMin(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { +func (e *executor) executeMin(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeMin") defer span.Finish() @@ -1248,7 +1254,7 @@ func (e *executor) executeMin(ctx context.Context, qcx *Qcx, index string, c *pq } // executeMax executes a Max() call. -func (e *executor) executeMax(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { +func (e *executor) executeMax(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeMax") defer span.Finish() @@ -1284,7 +1290,7 @@ func (e *executor) executeMax(ctx context.Context, qcx *Qcx, index string, c *pq } // executePercentile executes a Percentile() call. -func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { +func (e *executor) executePercentile(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executePercentile") defer span.Finish() @@ -1410,7 +1416,7 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string } // executeMinRow executes a MinRow() call. -func (e *executor) executeMinRow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ interface{}, err error) { +func (e *executor) executeMinRow(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeMinRow") defer span.Finish() @@ -1449,7 +1455,7 @@ func (e *executor) executeMinRow(ctx context.Context, qcx *Qcx, index string, c } // executeMaxRow executes a MaxRow() call. -func (e *executor) executeMaxRow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ interface{}, err error) { +func (e *executor) executeMaxRow(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeMaxRow") defer span.Finish() @@ -1488,7 +1494,7 @@ func (e *executor) executeMaxRow(ctx context.Context, qcx *Qcx, index string, c } // executePrecomputedCall pretends to execute a call that we have a precomputed value for. -func (e *executor) executePrecomputedCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ *Row, err error) { +func (e *executor) executePrecomputedCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ *Row, err error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executePrecomputedCall") defer span.Finish() result := NewRow() @@ -1500,7 +1506,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, qcx *Qcx, index s } // executeBitmapCall executes a call that returns a bitmap. -func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ *Row, err error) { +func (e *executor) executeBitmapCall(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeBitmapCall") span.LogKV("pqlCallName", c.Name) defer span.Finish() @@ -1544,7 +1550,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string } // executeBitmapCallShard executes a bitmap call for a single shard. -func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeBitmapCallShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { if err := validateQueryContext(ctx); err != nil { return nil, err } @@ -1582,26 +1588,43 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s // executeDistinctShard executes a Distinct call on a single shard, yielding // a SignedRow of the values found. -func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result interface{}, err error) { +func (e *executor) executeDistinctShard(ctx context.Context, qcx qc.QueryContext, index string, fieldName string, c *pql.Call, shard uint64) (result interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeDistinctShard") defer span.Finish() + var isTimestamp bool - idx := e.Holder.Index(index) field := e.Holder.Field(index, fieldName) if field == nil { return nil, ErrFieldNotFound } + // determine result type: + // SignedRow for BSI fields (which can have negative values) + // Row for set fields (which can only have positive values) + // DistinctTimestamp for FieldTypeTimestamp, which is a BSI field + // with special interpretation rules. bsig := field.bsiGroup(fieldName) + var frag *fragment if bsig == nil { + frag = e.Holder.fragment(index, fieldName, viewStandard, shard) result = &Row{ Index: index, Field: fieldName, } - } else if field.Options().Type == FieldTypeTimestamp { - result = DistinctTimestamp{Name: fieldName} } else { - result = SignedRow{} + // we'll be using the BSI fragment... + frag = e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) + // ... but how we interpret it might vary + if field.Options().Type == FieldTypeTimestamp { + isTimestamp = true + result = DistinctTimestamp{Name: fieldName} + } else { + result = SignedRow{} + } + } + + if frag == nil { + return result, nil } var filter *Row @@ -1627,33 +1650,68 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } } - if bsig == nil { - return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) + // Now we want to do the actual op. The only operation the executeDistinct + // functions need is OffsetRange, or alternatively, frag.row. We used to + // use OffsetRange directly, because it avoided cluttering a row cache, + // but we don't use that anymore anyway. + qr, err := frag.qcxRead(qcx) + if err != nil { + return result, err } - if field.Options().Type == FieldTypeTimestamp { - r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) - if err != nil { - return nil, err + if bsig == nil { + resultData, err := executeDistinctShardSet(ctx, qr, filterBitmap) + if resultData != nil { + resultData.Index = index + resultData.Field = fieldName } - // If we have a filter, or there's just no content for this shard, we - // can end up with empty results. Rather than trying to synthesize - // a result from this empty set, we just go ahead and use that. - if r.Pos == nil { - return result, nil + return resultData, err + } + resultData, err := executeDistinctShardBSI(ctx, frag, qr, bsig, filterBitmap) + if err != nil { + // return our empty result and the error + return result, err + } + + if !isTimestamp { + if resultData.Pos != nil { + resultData.Pos.Index = index + resultData.Pos.Field = fieldName } - cols := r.Pos.Columns() - results := make([]string, len(cols)) - for i, val := range cols { - t, err := ValToTimestamp(field.options.TimeUnit, int64(val)+bsig.Base) - if err != nil { - return nil, errors.Wrap(err, "translating value to timestamp") - } - results[i] = t.Format(time.RFC3339Nano) + if resultData.Neg != nil { + resultData.Neg.Index = index + resultData.Neg.Field = fieldName } - result = DistinctTimestamp{Name: fieldName, Values: results} + return resultData, nil + } + + // We have a bunch of timestamps. We need to coalesce them. + // Row.Count() is defined on nil rows, it yields 0. + expected := resultData.Pos.Count() + resultData.Neg.Count() + if expected == 0 { return result, nil } - return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + allResults := make([]string, expected) + posCols := resultData.Pos.Columns() + results := allResults[:len(posCols)] + for i, val := range posCols { + t, err := ValToTimestamp(field.options.TimeUnit, int64(val)+bsig.Base) + if err != nil { + return nil, errors.Wrap(err, "translating value to timestamp") + } + results[i] = t.Format(time.RFC3339Nano) + } + negCols := resultData.Neg.Columns() + // slice into the rest of it, and... + results = allResults[len(posCols):] + for i, val := range negCols { + // note the negative val here. + t, err := ValToTimestamp(field.options.TimeUnit, int64(-val)+bsig.Base) + if err != nil { + return nil, errors.Wrap(err, "translating value to timestamp") + } + results[i] = t.Format(time.RFC3339Nano) + } + return DistinctTimestamp{Name: fieldName, Values: allResults}, nil } type DistinctTimestamp struct { @@ -1715,15 +1773,10 @@ const ( FragmentNotFound = Error("fragment not found") ) -func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { - index := idx.Name() - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(&err0) - - fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) +// executeDistinctShardSet returns a Row without populated Index or Field +// because it does not concern itself with such trivialities. +func executeDistinctShardSet(ctx context.Context, qr qc.QueryRead, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { + fragData, _, err := qr.ContainerIterator(0) switch errors.Cause(err) { case ViewNotFound, FragmentNotFound: // It may seem reasonable to return `nil` here in the case where the @@ -1737,7 +1790,7 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam // after the union will have blank Index and Field values. Here, we // ensure that we send a non-nil Row with valid Index and Field values // so that the union step doesn't cause problems. - return &Row{Index: index, Field: fieldName}, nil + return &Row{}, nil case nil: default: return nil, errors.Wrap(err, "getting fragment data") @@ -1791,25 +1844,17 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam seenThisRow = true } } - result = NewRowFromBitmap(rows) - result.Index = idx.Name() - result.Field = fieldName - return result, nil + return NewRowFromBitmap(rows), nil } -func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, bsig *bsiGroup, filterBitmap *roaring.Bitmap) (result SignedRow, err0 error) { - view := viewBSIGroupPrefix + fieldName - index := idx.Name() +// executeDistinctShardBSI returns a signedRow in which the Row members +// don't have Index/Field computed, because it does not concern itself +// with such trivialities. +func executeDistinctShardBSI(ctx context.Context, frag *fragment, qr qc.QueryRead, bsig *bsiGroup, filterBitmap *roaring.Bitmap) (result SignedRow, err0 error) { depth := uint64(bsig.BitDepth) offset := bsig.Base - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return SignedRow{}, err - } - defer finisher(&err0) - - existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1) + existsBitmap, err := frag.rowAsBitmap(qr, 0) if err != nil { switch errors.Cause(err) { case ViewNotFound, FragmentNotFound: @@ -1824,7 +1869,7 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam return result, nil } - signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2) + signBitmap, err := frag.rowAsBitmap(qr, 1) if err != nil { return result, errors.Wrap(err, "getting sign bitmap") } @@ -1832,7 +1877,7 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam dataBitmaps := make([]*roaring.Bitmap, depth) for i := uint64(0); i < depth; i++ { - dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*(i+2), ShardWidth*(i+3)) + dataBitmaps[i], err = frag.rowAsBitmap(qr, i+2) if err != nil { return result, err } @@ -1912,20 +1957,14 @@ func executeDistinctShardBSI(ctx context.Context, qcx *Qcx, idx *Index, fieldNam Neg: NewRowFromBitmap(negBitmap), Pos: NewRowFromBitmap(posBitmap), } - result.Neg.Index, result.Pos.Index = idx.Name(), idx.Name() - result.Neg.Field, result.Pos.Field = fieldName, fieldName return result, nil } // executeSumCountShard calculates the sum and count for bsiGroups on a shard. -func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *Row, shard uint64) (_ ValCount, err0 error) { +func (e *executor) executeSumCountShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, filter *Row, shard uint64) (_ ValCount, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeSumCountShard") defer span.Finish() - // use tx to keep consistency between - // the filter and the later count. - idx := e.Holder.Index(index) - // Only calculate the filter if it doesn't exist and a child call as been passed in. if filter == nil && len(c.Children) == 1 { @@ -1956,15 +1995,14 @@ func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index str return ValCount{}, nil } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: fragment, Shard: shard}) + qr, err := fragment.qcxRead(qcx) if err != nil { return ValCount{}, err } - defer finisher(&err0) sumspan, _ := tracing.StartSpanFromContext(ctx, "executor.executeSumCountShard_fragment.sum") defer sumspan.Finish() - vsum, vcount, err := fragment.sum(tx, filter, bsig.BitDepth) + vsum, vcount, err := fragment.sum(qr, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } @@ -1981,7 +2019,7 @@ func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index str } // executeMinShard calculates the min for bsiGroups on a shard. -func (e *executor) executeMinShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ ValCount, err0 error) { +func (e *executor) executeMinShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ ValCount, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeMinShard") defer span.Finish() @@ -2007,7 +2045,7 @@ func (e *executor) executeMinShard(ctx context.Context, qcx *Qcx, index string, } // executeMaxShard calculates the max for bsiGroups on a shard. -func (e *executor) executeMaxShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ ValCount, err0 error) { +func (e *executor) executeMaxShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ ValCount, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxShard") defer span.Finish() @@ -2033,7 +2071,7 @@ func (e *executor) executeMaxShard(ctx context.Context, qcx *Qcx, index string, } // executeMinRowShard returns the minimum row ID for a shard. -func (e *executor) executeMinRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ PairField, err0 error) { +func (e *executor) executeMinRowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ PairField, err0 error) { var filter *Row if len(c.Children) == 1 { @@ -2055,15 +2093,12 @@ func (e *executor) executeMinRowShard(ctx context.Context, qcx *Qcx, index strin return PairField{}, nil } - idx := e.Holder.Index(index) - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: fragment, Shard: fragment.shard}) + qr, err := fragment.qcxRead(qcx) if err != nil { return PairField{}, err } - defer finisher(&err0) - - minRowID, count, err := fragment.minRow(tx, filter) + minRowID, count, err := fragment.minRow(qr, filter) if err != nil { return PairField{}, err } @@ -2078,7 +2113,7 @@ func (e *executor) executeMinRowShard(ctx context.Context, qcx *Qcx, index strin } // executeMaxRowShard returns the maximum row ID for a shard. -func (e *executor) executeMaxRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ PairField, err0 error) { +func (e *executor) executeMaxRowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ PairField, err0 error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) @@ -2098,15 +2133,12 @@ func (e *executor) executeMaxRowShard(ctx context.Context, qcx *Qcx, index strin if fragment == nil { return PairField{}, nil } - - idx := e.Holder.Index(index) - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) + qr, err := fragment.qcxRead(qcx) if err != nil { - return PairField{}, ErrQcxDone + return PairField{}, err } - defer finisher(&err0) - maxRowID, count, err := fragment.maxRow(tx, filter) + maxRowID, count, err := fragment.maxRow(qr, filter) if err != nil { return PairField{}, nil } @@ -2120,7 +2152,7 @@ func (e *executor) executeMaxRowShard(ctx context.Context, qcx *Qcx, index strin }, nil } -func (e *executor) executeTopK(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeTopK(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopK") defer span.Finish() @@ -2178,7 +2210,7 @@ func (e *executor) executeTopK(ctx context.Context, qcx *Qcx, index string, c *p } // executeTopKShard builds a perpendicular BSI bitmap of a shard for TopK. -func (e *executor) executeTopKShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ []*Row, err0 error) { +func (e *executor) executeTopKShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ []*Row, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopKShard") defer span.Finish() @@ -2230,28 +2262,22 @@ func (e *executor) executeTopKShard(ctx context.Context, qcx *Qcx, index string, } } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(&err0) - ftype := f.Type() switch ftype { case FieldTypeTime: if !(fromTime.IsZero() && toTime.IsZero()) { - return e.executeTopKShardTime(ctx, tx, filterBitmap, index, fieldName, shard, fromTime, toTime) + return e.executeTopKShardTime(ctx, qcx, filterBitmap, index, fieldName, shard, fromTime, toTime) } fallthrough case FieldTypeSet, FieldTypeMutex: - return e.executeTopKShardSet(ctx, tx, filterBitmap, index, fieldName, shard) + return e.executeTopKShardSet(ctx, qcx, filterBitmap, index, fieldName, shard) default: return nil, errors.Errorf("field type %q is not yet supported by TopK", ftype) } } // executeTopKShardSet builds a perpendicular BSI bitmap of a set field within a shard. -func (e *executor) executeTopKShardSet(ctx context.Context, tx Tx, filter *Row, index, field string, shard uint64) ([]*Row, error) { +func (e *executor) executeTopKShardSet(ctx context.Context, qcx qc.QueryContext, filter *Row, index, field string, shard uint64) ([]*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopKShardSet") defer span.Finish() @@ -2260,11 +2286,11 @@ func (e *executor) executeTopKShardSet(ctx context.Context, tx Tx, filter *Row, return nil, nil } - return topKFragments(ctx, tx, filter, f) + return topKFragments(ctx, qcx, filter, f) } // executeTopKShardTime builds a perpendicular BSI bitmap of a time field within a shard. -func (e *executor) executeTopKShardTime(ctx context.Context, tx Tx, filter *Row, index, field string, shard uint64, from, to time.Time) ([]*Row, error) { +func (e *executor) executeTopKShardTime(ctx context.Context, qcx qc.QueryContext, filter *Row, index, field string, shard uint64, from, to time.Time) ([]*Row, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -2293,19 +2319,23 @@ func (e *executor) executeTopKShardTime(ctx context.Context, tx Tx, filter *Row, fragments = append(fragments, f) } - return topKFragments(ctx, tx, filter, fragments...) + return topKFragments(ctx, qcx, filter, fragments...) } // topKFragments builds a perpendicular BSI bitmap from fragments. // The fragments are expected to be from set fields. -func topKFragments(ctx context.Context, tx Tx, filter *Row, fragments ...*fragment) (BSIData, error) { +func topKFragments(ctx context.Context, qcx qc.QueryContext, filter *Row, fragments ...*fragment) (BSIData, error) { // Acquire fragment container iterators. iters := make([]roaring.ContainerIterator, len(fragments)) for i, f := range fragments { f.mu.RLock() defer f.mu.RUnlock() - iter, _, err := tx.ContainerIterator(f.index(), f.field(), f.view(), f.shard, 0) + qr, err := f.qcxRead(qcx) + if err != nil { + return nil, err + } + iter, _, err := qr.ContainerIterator(0) if err != nil { return nil, err } @@ -2542,7 +2572,7 @@ func (f *topKFilter) fillIt(it roaring.ContainerIterator) { // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *executor) executeTopN(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*PairsField, error) { +func (e *executor) executeTopN(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopN") defer span.Finish() @@ -2594,7 +2624,7 @@ func (e *executor) executeTopN(ctx context.Context, qcx *Qcx, index string, c *p }, nil } -func (e *executor) executeTopNShards(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*PairsField, error) { +func (e *executor) executeTopNShards(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopNShards") defer span.Finish() @@ -2632,7 +2662,7 @@ func (e *executor) executeTopNShards(ctx context.Context, qcx *Qcx, index string } // executeTopNShard executes a TopN call for a single shard. -func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *PairsField, err0 error) { +func (e *executor) executeTopNShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *PairsField, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeTopNShard") defer span.Finish() @@ -2689,14 +2719,12 @@ func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string, return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - idx := e.Holder.Index(index) - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Fragment: f, Index: idx, Shard: shard}) + qr, err := f.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err0) - pairs, err := f.top(tx, topOptions{ + pairs, err := f.top(qr, topOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -2713,7 +2741,7 @@ func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string, } // executeDifferenceShard executes a difference() call for a local shard. -func (e *executor) executeDifferenceShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeDifferenceShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeDifferenceShard") defer span.Finish() @@ -2939,7 +2967,7 @@ func findGroupCounts(v interface{}) []GroupCount { return nil } -func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*GroupCounts, error) { +func (e *executor) executeGroupBy(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*GroupCounts, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeGroupBy") defer span.Finish() // validate call @@ -3681,7 +3709,7 @@ func ApplyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64, ignoreLimit bool) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64, ignoreLimit bool) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeGroupByShard") defer span.Finish() @@ -3750,7 +3778,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri return results, nil } -func (e *executor) executeRows(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (RowIDs, error) { +func (e *executor) executeRows(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (RowIDs, error) { // Fetch field name from argument. // Check "field" first for backwards compatibility. // TODO: remove at Pilosa 2.0 @@ -3840,7 +3868,7 @@ func (e *executor) executeRows(ctx context.Context, qcx *Qcx, index string, c *p return results, nil } -func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (_ RowIDs, err0 error) { +func (e *executor) executeRowsShard(ctx context.Context, qcx qc.QueryContext, index string, fieldName string, c *pql.Call, shard uint64) (_ RowIDs, err0 error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -3920,11 +3948,6 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, limit = int(lim) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(&err0) for _, view := range views { if err := ctx.Err(); err != nil { return nil, err @@ -3933,8 +3956,12 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, if frag == nil { continue } + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } - viewRows, err := frag.rows(ctx, tx, start, filters...) + viewRows, err := frag.rows(ctx, qr, start, filters...) if err != nil { return nil, err } @@ -4120,7 +4147,7 @@ var ( typeSQLNullInt64 = reflect.TypeOf(sql.NullInt64{}) ) -func (e *executor) executeExternalLookup(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (res ExtractedTable, err error) { +func (e *executor) executeExternalLookup(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (res ExtractedTable, err error) { if e.Holder.lookupDB == nil { return ExtractedTable{}, errors.New("external DB connection is not configured") } @@ -4474,7 +4501,7 @@ func handleExtractResults(other interface{}, filter *pql.Call, opt *ExecOptions) } } -func (e *executor) executeExtract(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeExtract(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { // Extract the column filter call. if len(c.Children) < 1 { return ExtractedIDMatrix{}, errors.New("missing column filter in Extract") @@ -4521,7 +4548,7 @@ var ( falseRowFakeID = []uint64{0} ) -func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index string, fields []string, filter *pql.Call, shard uint64, mopt *mapOptions, timeArgs []TimeArgs) (_ interface{}, err0 error) { +func (e *executor) executeExtractShard(ctx context.Context, qcx qc.QueryContext, index string, fields []string, filter *pql.Call, shard uint64, mopt *mapOptions, timeArgs []TimeArgs) (_ interface{}, err0 error) { var colsBitmap *Row var cols []uint64 var sortedResult *SortedRow @@ -4550,12 +4577,6 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return ExtractedIDMatrix{}, newNotFoundError(ErrIndexNotFound, index) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return ExtractedIDMatrix{}, err - } - defer finisher(&err0) - // Generate a matrix to stuff the results into. m := make([]ExtractedIDColumn, len(cols)) { @@ -4596,9 +4617,13 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // There is nothing here. continue } + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ExtractedIDMatrix{}, err + } // List all rows in the standard view. - rows, err := fragment.rows(ctx, tx, 0) + rows, err := fragment.rows(ctx, qr, 0) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "listing rows in set field") } @@ -4606,7 +4631,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // Loop over each row and scan the intersection with the filter. for _, rowID := range rows { // Load row from fragment. - row, err := fragment.row(tx, rowID) + row, err := fragment.row(qr, rowID) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading row from fragment") } @@ -4636,8 +4661,12 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // There is nothing here. continue } + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ExtractedIDMatrix{}, err + } - rows, err := fragment.rows(ctx, tx, 0) + rows, err := fragment.rows(ctx, qr, 0) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "listing rows in set field") } @@ -4645,7 +4674,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // Loop over each row and scan the intersection with the filter. for _, rowID := range rows { // Load row from fragment. - row, err := fragment.row(tx, rowID) + row, err := fragment.row(qr, rowID) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading row from fragment") } @@ -4689,13 +4718,16 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // There is nothing here. continue } - + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ExtractedIDMatrix{}, err + } // Fetch true and false rows. - trueRow, err := fragment.row(tx, trueRowID) + trueRow, err := fragment.row(qr, trueRowID) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading true row from fragment") } - falseRow, err := fragment.row(tx, falseRowID) + falseRow, err := fragment.row(qr, falseRowID) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading true row from fragment") } @@ -4720,6 +4752,10 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // There is nothing here. continue } + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ExtractedIDMatrix{}, err + } // Load the BSI group. bsig := field.bsiGroup(name) @@ -4728,7 +4764,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri } // Load the BSI exists bit. - exists, err := fragment.row(tx, bsiExistsBit) + exists, err := fragment.row(qr, bsiExistsBit) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading BSI exists bit from fragment") } @@ -4745,7 +4781,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri mergeBits(exists, 0, data) // Copy in the sign bit. - sign, err := fragment.row(tx, bsiSignBit) + sign, err := fragment.row(qr, bsiSignBit) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading BSI sign bit from fragment") } @@ -4754,7 +4790,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri // Copy in the significand. for i := uint64(0); i < bsig.BitDepth; i++ { - bits, err := fragment.row(tx, bsiOffsetBit+uint64(i)) + bits, err := fragment.row(qr, bsiOffsetBit+uint64(i)) if err != nil { return ExtractedIDMatrix{}, errors.Wrap(err, "loading BSI significand bit from fragment") } @@ -4790,7 +4826,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri return matrix, nil } -func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { +func (e *executor) executeRowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowShard") defer span.Finish() @@ -4850,16 +4886,11 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, if frag == nil { return NewRow(), nil } - - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Fragment: frag, Index: idx, Shard: shard}) + qr, err := frag.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err0) - row, err := frag.row(tx, rowID) - if qcx.write && err == nil { - row = row.Clone() - } + row, err := frag.row(qr, rowID) return row, err } @@ -4870,18 +4901,16 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, // Union bitmaps across all time-based views. rows := make([]*Row, 0, len(views)) - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - defer finisher(&err0) for _, view := range views { f := e.Holder.fragment(index, fieldName, view, shard) if f == nil { continue } + qr, err := f.qcxRead(qcx) if err != nil { return nil, err } - - row, err := f.row(tx, rowID) + row, err := f.row(qr, rowID) if err != nil { return nil, err } @@ -4890,20 +4919,14 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, if len(rows) == 0 { return &Row{}, nil } else if len(rows) == 1 { - if qcx.write { - return rows[0].Clone(), nil - } return rows[0], nil } row := rows[0].Union(rows[1:]...) - if qcx.write { - row = row.Clone() - } return row, nil } // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { +func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (cloneable *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeRowBSIGroupShard") defer span.Finish() @@ -4930,17 +4953,6 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index return nil, newNotFoundError(ErrFieldNotFound, fieldName) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: f.idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(&err0) - defer func() { - if qcx.write && cloneable != nil { - cloneable = cloneable.Clone() - } - }() - // EQ null _exists - frag.NotNull() // NEQ null frag.NotNull() // BETWEEN a,b(in) BETWEEN/frag.RowBetween() @@ -4955,7 +4967,11 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index if frag == nil { return NewRow(), nil } - return frag.notNull(tx) + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } + return frag.notNull(qr) } else if cond.Op == pql.EQ && cond.Value == nil { // Make sure the index supports existence tracking. @@ -4971,17 +4987,24 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index if existenceFrag == nil { existenceRow = NewRow() } else { - if existenceRow, err0 = existenceFrag.row(tx, 0); err0 != nil { + qr, err := existenceFrag.qcxRead(qcx) + if err != nil { + return nil, err + } + if existenceRow, err0 = existenceFrag.row(qr, 0); err0 != nil { return nil, err0 } } var notNull *Row - var err error // Retrieve notNull from fragment if it exists. if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { - if notNull, err = frag.notNull(tx); err != nil { + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } + if notNull, err = frag.notNull(qr); err != nil { return nil, errors.Wrap(err, "getting fragment not null") } } else { @@ -5022,14 +5045,18 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index if frag == nil { return NewRow(), nil } + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.notNull(tx) + return frag.notNull(qr) } - return frag.rangeBetween(tx, bsig.BitDepth, baseValueMin, baseValueMax) + return frag.rangeBetween(qr, bsig.BitDepth, baseValueMin, baseValueMax) } else { value, err := getScaledInt(f, cond.Value) @@ -5053,24 +5080,28 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, qcx *Qcx, index if frag == nil { return NewRow(), nil } + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.notNull(tx) + return frag.notNull(qr) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.notNull(tx) + return frag.notNull(qr) } - return frag.rangeOp(tx, cond.Op, bsig.BitDepth, baseValue) + return frag.rangeOp(qr, cond.Op, bsig.BitDepth, baseValue) } } // executeIntersectShard executes a intersect() call for a local shard. -func (e *executor) executeIntersectShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeIntersectShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeIntersectShard") defer span.Finish() @@ -5095,7 +5126,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, qcx *Qcx, index st } // executeUnionShard executes a union() call for a local shard. -func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (out *Row, err error) { +func (e *executor) executeUnionShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (out *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeUnionShard") defer span.Finish() @@ -5119,7 +5150,7 @@ func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string // executeInnerUnionRowsShard executes a special magical call which is actually // more like Row() than Union(), and takes a call plus a []uint64 of rows, and // generates the union of the rows in the []uint64. -func (e *executor) executeInnerUnionRowsShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (out *Row, err0 error) { +func (e *executor) executeInnerUnionRowsShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (out *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeInnerUnionRowsShard") defer span.Finish() @@ -5173,16 +5204,11 @@ func (e *executor) executeInnerUnionRowsShard(ctx context.Context, qcx *Qcx, ind if frag == nil { return NewRow(), nil } - - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Fragment: frag, Index: idx, Shard: shard}) + qr, err := frag.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err0) - row, err := frag.unionRows(ctx, tx, rowIDs) - if qcx.write && err == nil { - row = row.Clone() - } + row, err := frag.unionRows(ctx, qr, rowIDs) return row, err } @@ -5193,18 +5219,17 @@ func (e *executor) executeInnerUnionRowsShard(ctx context.Context, qcx *Qcx, ind // Union bitmaps across all time-based views. rows := make([]*Row, 0, len(views)) - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - defer finisher(&err0) for _, view := range views { f := e.Holder.fragment(index, fieldName, view, shard) if f == nil { continue } + qr, err := f.qcxRead(qcx) if err != nil { return nil, err } - row, err := f.unionRows(ctx, tx, rowIDs) + row, err := f.unionRows(ctx, qr, rowIDs) if err != nil { return nil, err } @@ -5213,20 +5238,14 @@ func (e *executor) executeInnerUnionRowsShard(ctx context.Context, qcx *Qcx, ind if len(rows) == 0 { return &Row{}, nil } else if len(rows) == 1 { - if qcx.write { - return rows[0].Clone(), nil - } return rows[0], nil } row := rows[0].Union(rows[1:]...) - if qcx.write { - row = row.Clone() - } return row, nil } // executeXorShard executes a xor() call for a local shard. -func (e *executor) executeXorShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeXorShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeXorShard") defer span.Finish() @@ -5248,7 +5267,7 @@ func (e *executor) executeXorShard(ctx context.Context, qcx *Qcx, index string, } // executePrecomputedCallShard pretends to execute a precomputed call for a local shard. -func (e *executor) executePrecomputedCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executePrecomputedCallShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { if c.Precomputed != nil { v := c.Precomputed[shard] if v == nil { @@ -5267,7 +5286,7 @@ func (e *executor) executePrecomputedCallShard(ctx context.Context, qcx *Qcx, in } // executeNotShard executes a Not() call for a local shard. -func (e *executor) executeNotShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { +func (e *executor) executeNotShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeNotShard") defer span.Finish() @@ -5285,29 +5304,19 @@ func (e *executor) executeNotShard(ctx context.Context, qcx *Qcx, index string, return nil, errors.Errorf("index does not support existence tracking: %s", index) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(nil) - var existenceRow *Row existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) if existenceFrag == nil { existenceRow = NewRow() } else { - if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + qr, err := existenceFrag.qcxRead(qcx) + if err != nil { return nil, err } - if qcx.write { - existenceRow = existenceRow.Clone() + if existenceRow, err = existenceFrag.row(qr, 0); err != nil { + return nil, err } } - // the finishers returned by a write tx, which we might be in if there's - // a higher-level write in this call OR ANY OTHER CALL, are safe to - // double-call, but we have to be sure of finishing before starting a - // bitmap call, or we lock against ourselves. - finisher(nil) row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) if err != nil { @@ -5327,7 +5336,7 @@ func (e *executor) executeConstRow(ctx context.Context, index string, c *pql.Cal return NewRow(ids...), nil } -func (e *executor) executeUnionRows(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { +func (e *executor) executeUnionRows(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { // Turn UnionRows(Rows(...)) into Union(Row(...), ...). var rows []*pql.Call for _, child := range c.Children { @@ -5412,7 +5421,7 @@ func (e *executor) executeUnionRows(ctx context.Context, qcx *Qcx, index string, } // executeAllCallShard executes an All() call for a local shard. -func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (res *Row, err0 error) { +func (e *executor) executeAllCallShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (res *Row, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeAllCallShard") defer span.Finish() @@ -5433,14 +5442,12 @@ func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index stri if existenceFrag == nil { existenceRow = NewRow() } else { - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: existenceFrag, Shard: shard}) + qr, err := existenceFrag.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err0) - - if existenceRow, err = existenceFrag.row(tx, 0); err != nil { + if existenceRow, err = existenceFrag.row(qr, 0); err != nil { return nil, err } } @@ -5449,7 +5456,7 @@ func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index stri } // executeShiftShard executes a shift() call for a local shard. -func (e *executor) executeShiftShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeShiftShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ *Row, err error) { n, _, err := c.IntArg("n") if err != nil { return nil, fmt.Errorf("executeShiftShard: %v", err) @@ -5470,7 +5477,7 @@ func (e *executor) executeShiftShard(ctx context.Context, qcx *Qcx, index string } // executeCount executes a count() call. -func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (uint64, error) { +func (e *executor) executeCount(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeCount") defer span.Finish() @@ -5526,7 +5533,7 @@ func (e *executor) executeCount(ctx context.Context, qcx *Qcx, index string, c * } // executeClearBit executes a Clear() call. -func (e *executor) executeClearBit(ctx context.Context, qcx *Qcx, index string, c *pql.Call, opt *ExecOptions) (bool, error) { +func (e *executor) executeClearBit(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, opt *ExecOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeClearBit") defer span.Finish() @@ -5570,7 +5577,7 @@ func (e *executor) executeClearBit(ctx context.Context, qcx *Qcx, index string, } // executeClearBitField executes a Clear() call for a field. -func (e *executor) executeClearBitField(ctx context.Context, qcx *Qcx, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (_ bool, err0 error) { +func (e *executor) executeClearBitField(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (_ bool, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeClearBitField") defer span.Finish() @@ -5607,7 +5614,7 @@ func (e *executor) executeClearBitField(ctx context.Context, qcx *Qcx, index str } // executeClearRow executes a ClearRow() call. -func (e *executor) executeClearRow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ bool, err error) { +func (e *executor) executeClearRow(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (_ bool, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeClearRow") defer span.Finish() @@ -5658,7 +5665,7 @@ func (e *executor) executeClearRow(ctx context.Context, qcx *Qcx, index string, } // executeClearRowShard executes a ClearRow() call for a single shard. -func (e *executor) executeClearRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ bool, err0 error) { +func (e *executor) executeClearRowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ bool, err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "executor.executeClearRowShard") defer span.Finish() @@ -5682,13 +5689,6 @@ func (e *executor) executeClearRowShard(ctx context.Context, qcx *Qcx, index str return false, newNotFoundError(ErrFieldNotFound, fieldName) } - idx := e.Holder.Index(index) - tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Shard: shard}) - if err != nil { - return false, err - } - defer finisher(&err0) - // Remove the row from all views. changed := false for _, view := range field.views() { @@ -5696,7 +5696,11 @@ func (e *executor) executeClearRowShard(ctx context.Context, qcx *Qcx, index str if fragment == nil { continue } - cleared, err := fragment.clearRow(tx, rowID) + qw, err := fragment.qcxWrite(qcx) + if err != nil { + return false, err + } + cleared, err := fragment.clearRow(qw, rowID) if err != nil { return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard) } @@ -5708,7 +5712,7 @@ func (e *executor) executeClearRowShard(ctx context.Context, qcx *Qcx, index str // executeSetRow executes a Store() call. -func (e *executor) executeSetRow(ctx context.Context, qcx *Qcx, indexName string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { +func (e *executor) executeSetRow(ctx context.Context, qcx qc.QueryContext, indexName string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { // Parse arguments. fieldName, err := c.FieldArg() if err != nil { @@ -5759,7 +5763,7 @@ func (e *executor) executeSetRow(ctx context.Context, qcx *Qcx, indexName string } // executeSetRowShard executes a SetRow() call for a single shard. -func (e *executor) executeSetRowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ bool, err0 error) { +func (e *executor) executeSetRowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (_ bool, err0 error) { fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Store() argument required: field") @@ -5807,15 +5811,12 @@ func (e *executor) executeSetRowShard(ctx context.Context, qcx *Qcx, index strin } } - idx := e.Holder.Index(index) - tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Shard: shard}) + qw, err := fragment.qcxWrite(qcx) if err != nil { return false, err } - defer finisher(&err0) - - set, err := fragment.setRow(tx, src, rowID) + set, err := fragment.setRow(qw, src, rowID) if err != nil { return false, errors.Wrapf(err, "storing row %d on view %s shard %d", rowID, viewStandard, shard) } @@ -5825,7 +5826,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, qcx *Qcx, index strin } // executeSet executes a Set() call. -func (e *executor) executeSet(ctx context.Context, qcx *Qcx, index string, c *pql.Call, opt *ExecOptions) (_ bool, err0 error) { +func (e *executor) executeSet(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, opt *ExecOptions) (_ bool, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeSet") defer span.Finish() @@ -5909,7 +5910,7 @@ func (e *executor) executeSet(ctx context.Context, qcx *Qcx, index string, c *pq } // executeSetBitField executes a Set() call for a specific field. -func (e *executor) executeSetBitField(ctx context.Context, qcx *Qcx, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (_ bool, err0 error) { +func (e *executor) executeSetBitField(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (_ bool, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeSetBitField") defer span.Finish() @@ -5947,7 +5948,7 @@ func (e *executor) executeSetBitField(ctx context.Context, qcx *Qcx, index strin } // executeSetValueField executes a Set() call for a specific int field. -func (e *executor) executeSetValueField(ctx context.Context, qcx *Qcx, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *ExecOptions) (_ bool, err0 error) { +func (e *executor) executeSetValueField(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *ExecOptions) (_ bool, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeSetValueField") defer span.Finish() @@ -5985,7 +5986,7 @@ func (e *executor) executeSetValueField(ctx context.Context, qcx *Qcx, index str } // executeClearValueField removes value for colID if present -func (e *executor) executeClearValueField(ctx context.Context, qcx *Qcx, index string, c *pql.Call, f *Field, colID uint64, opt *ExecOptions) (_ bool, err0 error) { +func (e *executor) executeClearValueField(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, f *Field, colID uint64, opt *ExecOptions) (_ bool, err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeClearValueField") defer span.Finish() @@ -8250,7 +8251,7 @@ func callArgString(call *pql.Call, key string) string { // calls). type groupByIterator struct { executor *executor - qcx *Qcx + qcx qc.QueryContext index string shard uint64 @@ -8282,7 +8283,7 @@ type groupByIterator struct { } // newGroupByIterator initializes a new groupByIterator. -func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children []*pql.Call, aggregate *pql.Call, filter *Row, index string, shard uint64, holder *Holder) (_ *groupByIterator, err0 error) { +func newGroupByIterator(executor *executor, qcx qc.QueryContext, rowIDs []RowIDs, children []*pql.Call, aggregate *pql.Call, filter *Row, index string, shard uint64, holder *Holder) (_ *groupByIterator, err0 error) { gbi := &groupByIterator{ executor: executor, qcx: qcx, @@ -8298,7 +8299,6 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children aggregate: aggregate, fields: make([]FieldRow, len(children)), } - idx := holder.Index(index) var ( fieldName string @@ -8375,12 +8375,7 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children filters = append(filters, roaring.NewBitmapRowsFilter(rowIDs[i])) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, err - } - defer finisher(&err0) - + var err error // Fetch fragment(s), get rowIterator if isTimeField { var fragments []*fragment @@ -8395,7 +8390,7 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, nil } - gbi.rowIters[i], err = timeFragmentsRowIterator(fragments, tx, i != 0, filters...) + gbi.rowIters[i], err = timeFragmentsRowIterator(fragments, qcx, i != 0, filters...) if err != nil { return nil, err } @@ -8404,8 +8399,11 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children if frag == nil { // this means this whole shard doesn't have all it needs to continue return nil, nil } - - gbi.rowIters[i], err = frag.rowIterator(tx, i != 0, filters...) + qr, err := frag.qcxRead(qcx) + if err != nil { + return nil, err + } + gbi.rowIters[i], err = frag.rowIterator(qr, i != 0, filters...) if err != nil { return nil, err } @@ -8681,7 +8679,7 @@ func decimalToInt64(dec pql.Decimal, opt FieldOptions) int64 { } // executeDeleteRecords executes a delete() call. -func (e *executor) executeDeleteRecords(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { +func (e *executor) executeDeleteRecords(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "executor.executeDelete") defer span.Finish() @@ -8690,12 +8688,10 @@ func (e *executor) executeDeleteRecords(ctx context.Context, qcx *Qcx, index str } else if len(c.Children) > 1 { return false, errors.New("Delete() only accepts a single bitmap input") } - qcx.Abort() - qcx.Reset() // release the qcx to allow for rbf checkpoint // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) { - return e.executeDeleteRecordFromShard(ctx, index, c.Children[0], shard) + return e.executeDeleteRecordFromShard(ctx, qcx, index, c.Children[0], shard) } // Merge returned results at coordinating node. @@ -8713,29 +8709,7 @@ func (e *executor) executeDeleteRecords(ctx context.Context, qcx *Qcx, index str return n, nil } -func transactExistRow(ctx context.Context, idx *Index, shard uint64, frag *fragment, src *Row) (uint64, error) { - holder := idx.Holder() - tx := holder.Txf().NewTx(Txo{Write: writable, Index: idx, Shard: shard}) - rows, err := frag.rows(ctx, tx, 1) - if err != nil { - tx.Rollback() - return 0, err - - } - // obtain a rowID which is higher than any currently present row ID. - rowID := uint64(1) - if len(rows) > 0 { - rowID = rows[len(rows)-1] + 1 - } - _, err = frag.setRow(tx, src, rowID) - if err != nil { - tx.Rollback() - return 0, err - } - return rowID, tx.Commit() -} - -func (e *executor) executeDeleteRecordFromShard(ctx context.Context, index string, bmCall *pql.Call, shard uint64) (changed bool, err error) { +func (e *executor) executeDeleteRecordFromShard(ctx context.Context, qcx qc.QueryContext, index string, bmCall *pql.Call, shard uint64) (changed bool, err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeDeleteRecordFromShard") defer span.Finish() // Fetch index. @@ -8744,10 +8718,8 @@ func (e *executor) executeDeleteRecordFromShard(ctx context.Context, index strin err = newNotFoundError(ErrIndexNotFound, index) return } - qcx := e.Holder.Txf().NewQcx() // bmCall is a bitmap row, err := e.executeBitmapCallShard(ctx, qcx, index, bmCall, shard) - qcx.Abort() if err != nil { return false, err } @@ -8759,178 +8731,7 @@ func (e *executor) executeDeleteRecordFromShard(ctx context.Context, index strin return } src := NewRowFromBitmap(columns) - return DeleteRowsWithFlow(ctx, src, idx, shard, true) -} - -func DeleteRows(ctx context.Context, src *Row, idx *Index, shard uint64) (bool, error) { - return DeleteRowsWithFlow(ctx, src, idx, shard, false) -} - -func DeleteRowsWithFlowWithKeys(ctx context.Context, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (bool, error) { - var existenceFragment *fragment - var deletedRowID uint64 - var commitor Commitor = &NopCommitor{} - var err error // store columns in exits field ToBeDelete row commited - holder := idx.Holder() - if normalFlow { // normalFlow is the standard path, "not normal" is recoverory - existenceFragment = holder.fragment(idx.Name(), existenceFieldName, viewStandard, shard) - if existenceFragment == nil { - // no exists field - return false, errors.New("can't bulk delete without existence field") - } - src := NewRowFromBitmap(columns) - deletedRowID, err = transactExistRow(ctx, idx, shard, existenceFragment, src) - if err != nil { - return false, err - } - } - commitor, err = deleteKeyTranslation(ctx, idx, shard, columns) - if err != nil { - return false, err - } - writeTx := holder.Txf().NewTx(Txo{Write: writable, Index: idx, Shard: shard}) - if err != nil { - return false, err - } - defer writeTx.Rollback() - changed := false - defer func() { - // if there is an error on the bit clearing rollback the keys - if err != nil { - changed = false - commitor.Rollback() - return - } - // if there is an error in the key commit, then rollback the delete - // write records before keys to remove possiblity of unmatch keys=records - err = writeTx.Commit() - if err != nil { - changed = false - commitor.Rollback() - return - } - if er := commitor.Commit(); er != nil { - err = er - } - if err != nil { - holder.Logger.Errorf("problems committing delete in rbf %v shard %v", err, shard) - } - }() - - for _, field := range idx.Fields() { - for _, view := range field.views() { - frag := view.Fragment(shard) - if frag == nil { - continue - } - c, err := frag.clearRecordsByBitmap(writeTx, columns) - if err != nil { - return false, err - } - if c { - changed = true - } - } - } - if existenceFragment != nil { // a string keys have been deleted and the deleteRow was created - if normalFlow { - existenceFragment.clearRow(writeTx, deletedRowID) - } else { - // this is if we are recovering from failure and cleaning up - rows, err := existenceFragment.rows(ctx, writeTx, 1) - if err != nil { - return false, err - } - for _, rowId := range rows { - existenceFragment.clearRow(writeTx, rowId) - } - } - } - return changed, nil -} - -func DeleteRowsWithOutKeysFlow(ctx context.Context, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (changed bool, err error) { - var existenceFragment *fragment - var deletedRowID uint64 - var commitor Commitor = &NopCommitor{} - holder := idx.Holder() - writeTx := holder.Txf().NewTx(Txo{Write: writable, Index: idx, Shard: shard}) - defer writeTx.Rollback() - defer func() { - // if there is an error in the key commit, then rollback the delete - // write records before keys to remove possiblity of unmatch keys=records - err := writeTx.Commit() - if err != nil { - changed = false - commitor.Rollback() - return - } - }() - for _, field := range idx.Fields() { - for _, view := range field.views() { - - frag := view.Fragment(shard) - if frag == nil { - continue - } - c, err := frag.clearRecordsByBitmap(writeTx, columns) - if err != nil { - return false, err - } - if c { - changed = true - } - - } - } - if existenceFragment == nil { // a string keys have been deleted and the deleteRow was created - return changed, nil - } - - if normalFlow { - existenceFragment.clearRow(writeTx, deletedRowID) - return changed, nil - } - - // this is if we are recovering from failure and cleaning up - rows, err := existenceFragment.rows(ctx, writeTx, 1) - if err != nil { - return false, err - } - for _, rowId := range rows { - existenceFragment.clearRow(writeTx, rowId) - } - return changed, nil -} - -func DeleteRowsWithFlow(ctx context.Context, src *Row, idx *Index, shard uint64, normalFlow bool) (change bool, err error) { - if len(src.Segments) == 0 { // nothing to remove - return false, nil - } - columns := src.Segments[0].data // should only be one segment - if columns.Count() == 0 { - return false, nil - } - bits := src.Segments[0].data.Slice() - min := func(a, b int) int { - if a <= b { - return a - } - return b - } - limit := idx.holder.cfg.RBFConfig.MaxDelete - for i := 0; i < len(bits); i += limit { - batch := roaring.NewBitmap(bits[i:min(i+limit, len(bits))]...) - if idx.Keys() { - change, err = DeleteRowsWithFlowWithKeys(ctx, batch, idx, shard, normalFlow) - } else { - change, err = DeleteRowsWithOutKeysFlow(ctx, batch, idx, shard, normalFlow) - } - if err != nil { - return change, err - } - } - return change, err + return e.Holder.deleteRowsWithFlow(ctx, qcx, src, idx, shard, true) } type Commitor interface { @@ -8953,7 +8754,7 @@ func deleteKeyTranslation(ctx context.Context, idx *Index, shard uint64, records return idx.TranslateStore(paritionID).Delete(records) } -func (e *executor) executeSort(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*SortedRow, error) { +func (e *executor) executeSort(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*SortedRow, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSort") defer span.Finish() @@ -9019,7 +8820,7 @@ func (e *executor) executeSort(ctx context.Context, qcx *Qcx, index string, c *p return result, nil } -func (e *executor) executeSortShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (*SortedRow, error) { +func (e *executor) executeSortShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (*SortedRow, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) @@ -9043,12 +8844,6 @@ func (e *executor) executeSortShard(ctx context.Context, qcx *Qcx, index string, return nil, newNotFoundError(ErrFieldNotFound, fieldName) } - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return nil, ErrQcxDone - } - defer finisher(&err) - sort_desc, _, err := c.BoolArg("sort-desc") if err != nil { return nil, errors.Wrap(err, " getting sort-desc") @@ -9060,11 +8855,15 @@ func (e *executor) executeSortShard(ctx context.Context, qcx *Qcx, index string, if fragment == nil { return nil, errors.New("bool fragment not found") } - falses, err := fragment.row(tx, falseRowID) + qr, err := fragment.qcxRead(qcx) + if err != nil { + return nil, err + } + falses, err := fragment.row(qr, falseRowID) if err != nil { return nil, errors.New("error loading false from fragment") } - trues, err := fragment.row(tx, trueRowID) + trues, err := fragment.row(qr, trueRowID) if err != nil { return nil, errors.New("error loading true from fragment") } @@ -9108,20 +8907,26 @@ func (e *executor) executeSortShard(ctx context.Context, qcx *Qcx, index string, RowKVs: rowKVs, }, nil case FieldTypeDecimal, FieldTypeInt, FieldTypeTimestamp: - return f.SortShardRow(tx, shard, filter, sort_desc) + return f.SortShardRow(qcx, shard, filter, sort_desc) case FieldTypeMutex: fragment := e.Holder.fragment(index, f.name, viewStandard, shard) if fragment == nil { return nil, errors.Errorf("fragment not found for field %s", f.name) } - rows, err := fragment.rows(ctx, tx, 0) + qr, err := fragment.qcxRead(qcx) + if err != nil { + return nil, err + } + // It feels like we should be able to do this in one pass, just + // accumulating entries for every row intersected with filter. + rows, err := fragment.rows(ctx, qr, 0) if err != nil { return nil, errors.Wrap(err, " ggettign rows error") } rowKVs := make([]RowKV, filter.Count()) i := 0 for _, rowID := range rows { - row, err := fragment.row(tx, rowID) + row, err := fragment.row(qr, rowID) if err != nil { return nil, errors.Wrap(err, "couldn't load row from fragment") } diff --git a/executor_internal_test.go b/executor_internal_test.go index 0f709d3f6..cf04636d2 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/molecula/featurebase/v3/pql" + "github.com/stretchr/testify/require" ) // AssertEqual checks a given RowIdentifiers against expected values. @@ -51,8 +52,7 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) { t.Fatalf("creating index: %v", err) } - qcx := holder.Txf().NewWritableQcx() - defer qcx.Abort() + qcx := holder.MustIndexQueryContext(t, "i") fb, errb := idx.CreateField("b", "", OptFieldTypeBool()) _, errbk := idx.CreateField("bk", "", OptFieldTypeBool(), OptFieldKeys()) @@ -569,8 +569,7 @@ func TestExecutor_DeleteRows(t *testing.T) { t.Fatalf("creating field: %v", err) } - qcx := holder.Txf().NewWritableQcx() - defer qcx.Abort() + qcx := holder.MustIndexQueryContext(t, "i") if _, err = f.SetBit(qcx, 1, 1, nil); err != nil { t.Fatalf("setting bit: %v", err) } @@ -580,14 +579,18 @@ func TestExecutor_DeleteRows(t *testing.T) { if err != nil { t.Fatalf("failed to read row: %v", err) } + require.Nil(t, qcx.Commit()) ctx := context.Background() - changed, err := DeleteRows(ctx, row, idx, 0) + // make a new query context for the deletes. we use the same context + // for both, because deleteRows is supposed to handle this. + qcx = holder.MustIndexQueryContext(t, "i") + changed, err := holder.deleteRows(ctx, qcx, row, idx, 0) if !changed || err != nil { t.Fatalf("failed to delete row: %v", err) } - changed, err = DeleteRows(ctx, row, idx, 0) + changed, err = holder.deleteRows(ctx, qcx, row, idx, 0) if err != nil { t.Fatalf("deleting rows: %v", err) } diff --git a/executor_test.go b/executor_test.go index 1f80a54eb..cd7ab2197 100644 --- a/executor_test.go +++ b/executor_test.go @@ -35,6 +35,8 @@ import ( . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var ( @@ -57,6 +59,54 @@ func TestExecutor(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() // Ensure a row query can be executed. + t.Run("QcxError", func(t *testing.T) { + n := c.GetNode(0) + a := n.API + h := c.GetHolder(0) + i, err := h.CreateIndex("i", "", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = i.CreateField("f", "", pilosa.OptFieldTypeInt(0, 1000)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + eg, ctx := errgroup.WithContext(ctx) + defer cancel() + eg.Go(func() error { + for i := 0; i < 1000; i++ { + _, err = a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"}) + if err != nil { + return err + } + _, err = a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=3)"}) + if err != nil { + return err + } + } + return nil + }) + for i := 0; i < 1000; i++ { + res, err := a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Intersect(Row(f>2),Row(f<2))\nSet(1, f=2)"}) + if err != nil { + t.Fatal(err) + } + var row *pilosa.Row + var ok bool + if row, ok = res.Results[0].(*pilosa.Row); !ok { + t.Fatalf("expected row, got %T", res.Results[0]) + } + cols := row.Columns() + if len(cols) != 0 { + t.Fatalf("expected empty row, got %d", cols) + } + } + err = eg.Wait() + if err != nil { + t.Fatalf("background errored") + } + }) t.Run("ExecuteRow", func(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { writeQuery := `` + @@ -1645,9 +1695,8 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } - // Obtain transaction. - qcx := hldr.Txf().NewQcx() - defer qcx.Abort() + qcx := hldr.MustIndexQueryContext(t, c.Idx()) + defer qcx.Release() f := hldr.Field(c.Idx(), "f") if value, exists, err := f.Value(qcx, 10); err != nil { @@ -1716,9 +1765,8 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Fatal(err) } - // Obtain transaction. - qcx := hldr.Txf().NewQcx() - defer qcx.Abort() + qcx := hldr.MustIndexQueryContext(t, c.Idx()) + defer qcx.Release() f := hldr.Field(c.Idx(), "f") if value, exists, err := f.Value(qcx, 10); err != nil { @@ -4371,7 +4419,11 @@ func TestExecutor_Execute_All(t *testing.T) { m0 := c.GetNode(0) // the request gets altered by the Import operation now... reqs := req.Clone().SortToShards() - qcx := m0.API.Txf().NewQcx() + shards := make([]uint64, 0, len(reqs)) + for shardID := range reqs { + shards = append(shards, shardID) + } + qcx := mustIndexQueryContext(t, m0.API, req.Index, shards...) for _, r := range reqs { // we can ignore the key (which is the shard) because each req // also got its internal key set. @@ -4379,7 +4431,7 @@ func TestExecutor_Execute_All(t *testing.T) { t.Fatal(err) } } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) i0, err := m0.API.Index(context.Background(), c.Idx()) PanicOn(err) @@ -4452,11 +4504,12 @@ func TestExecutor_Execute_All(t *testing.T) { req.ColumnKeys[i] = fmt.Sprintf("c%d", i) } - qcx := c.GetNode(0).API.Txf().NewQcx() + // note, the shard specified in the import request is wrong. + qcx := mustIndexQueryContext(t, c.GetNode(0).API, req.Index) if err := c.GetNode(0).API.Import(context.Background(), qcx, req); err != nil { t.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(t, qcx.Commit()) tests := []struct { qry string @@ -4850,11 +4903,11 @@ func benchmarkExistence(nn bool, b *testing.B) { b.ResetTimer() nodeAPI := c.GetNode(0).API for i := 0; i < b.N; i++ { - qcx := nodeAPI.Txf().NewQcx() + qcx := mustIndexQueryContext(b, nodeAPI, req.Index, req.Shard) if err := nodeAPI.Import(context.Background(), qcx, req); err != nil { b.Fatal(err) } - PanicOn(qcx.Finish()) + require.Nil(b, qcx.Commit()) } } @@ -5390,8 +5443,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { t.Fatalf("importing: %v", err) } m0 := c.GetNode(0) - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() + qcx := mustIndexQueryContext(t, m0.API, c.Idx()) var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4 @@ -5446,6 +5498,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { }); err != nil { t.Fatalf("importing: %v", err) } + require.Nil(t, qcx.Commit()) tests := []struct { query string diff --git a/field.go b/field.go index 55d6b3543..5fdea345d 100644 --- a/field.go +++ b/field.go @@ -15,7 +15,9 @@ import ( "sync" "time" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" @@ -719,17 +721,17 @@ func (f *Field) TTL() time.Duration { return f.options.TTL } -func (f *Field) bitDepth() (uint64, error) { +func (f *Field) bitDepth(qcx qc.QueryContext) (uint64, error) { var maxBitDepth uint64 - view2shards := f.idx.fieldView2shard.getViewsForField(f.name) + view2shards := f.holder.dbContents[keys.Index(f.index)][keys.Field(f.name)] for name, shardset := range view2shards { - view := f.view(name) + view := f.view(string(name)) if view == nil { continue } - bd, err := view.bitDepth(shardset.shards()) + bd, err := view.bitDepth(qcx, shardset.Shards()) if err != nil { return 0, errors.Wrapf(err, "getting view(%s) bit depth", name) } @@ -763,14 +765,14 @@ func (f *Field) cacheBitDepth(bd uint64) error { // openViews opens and initializes the views inside the field. func (f *Field) openViews() error { - view2shards := f.idx.fieldView2shard.getViewsForField(f.name) - if view2shards == nil { + viewShards := f.holder.dbContents[keys.Index(f.index)][keys.Field(f.name)] + if viewShards == nil { // no data return nil } - for name, shardset := range view2shards { - view := f.newView(f.viewPath(name), name) + for name, shardset := range viewShards { + view := f.newView(f.viewPath(string(name)), string(name)) if err := view.openWithShardSet(shardset); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } @@ -1052,7 +1054,7 @@ func (f *Field) viewsByTimeRange(from, to time.Time) (views []string, err error) // RowTime gets the row at the particular time with the granularity specified by // the quantum. -func (f *Field) RowTime(qcx *Qcx, rowID uint64, time time.Time, quantum string) (*Row, error) { +func (f *Field) RowTime(qcx qc.QueryContext, rowID uint64, time time.Time, quantum string) (*Row, error) { if !TimeQuantum(quantum).Valid() { return nil, ErrInvalidTimeQuantum } @@ -1203,7 +1205,7 @@ func (f *Field) deleteView(name string) error { // package, and the fact that it's only allowed on // `set`,`mutex`, and `bool` fields is odd. This may // be considered for deprecation in a future version. -func (f *Field) Row(qcx *Qcx, rowID uint64) (*Row, error) { +func (f *Field) Row(qcx qc.QueryContext, rowID uint64) (*Row, error) { switch f.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: view := f.view(viewStandard) @@ -1218,7 +1220,7 @@ func (f *Field) Row(qcx *Qcx, rowID uint64) (*Row, error) { // mutexCheck performs a sanity-check on the available fragments for a // field. The return is map[column]map[shard][]values for collisions only. -func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { +func (f *Field) MutexCheck(ctx context.Context, qcx qc.QueryContext, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { if f.Type() != FieldTypeMutex { return nil, errors.New("mutex check only valid for mutex fields") } @@ -1240,7 +1242,7 @@ func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit in } // SetBit sets a bit on a view within the field. -func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed bool, err error) { +func (f *Field) SetBit(qcx qc.QueryContext, rowID, colID uint64, t *time.Time) (changed bool, err error) { viewName := viewStandard if !f.options.NoStandardView { // Retrieve view. Exit if it doesn't exist. @@ -1280,7 +1282,7 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo } // ClearBit clears a bit within the field. -func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error) { +func (f *Field) ClearBit(qcx qc.QueryContext, rowID, colID uint64) (changed bool, err error) { viewName := viewStandard // Retrieve view. Exit if it doesn't exist. @@ -1325,29 +1327,31 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error return changed, nil } -// ClearBits clears all bits corresponding to the given record IDs in standard -// or BSI views. It does not delete bits from time quantum views. -func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error { - bsig := f.bsiGroup(f.name) - var v *view - if bsig != nil { - // looks like we're a BSI field? - v = f.view(viewBSIGroupPrefix + f.name) - } else { - v = f.view(viewStandard) - } - // it's fine if we never actually created the view, that means the - // bits are all clear! - if v == nil { - return nil - } - frag := v.Fragment(shard) - if frag == nil { - return nil - } - _, err := frag.ClearRecords(tx, recordIDs) - return err -} +// ClearBits is probably unused. Leaving it commented out for now but if things +// still work without it it should go away. +// // ClearBits clears all bits corresponding to the given record IDs in standard +// // or BSI views. It does not delete bits from time quantum views. +// func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error { +// bsig := f.bsiGroup(f.name) +// var v *view +// if bsig != nil { +// // looks like we're a BSI field? +// v = f.view(viewBSIGroupPrefix + f.name) +// } else { +// v = f.view(viewStandard) +// } +// // it's fine if we never actually created the view, that means the +// // bits are all clear! +// if v == nil { +// return nil +// } +// frag := v.Fragment(shard) +// if frag == nil { +// return nil +// } +// _, err := frag.ClearRecords(tx, recordIDs) +// return err +// } func groupCompare(a, b string, offset int) (lt, eq bool) { if len(a) > offset { @@ -1392,7 +1396,7 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { // StringValue reads an integer field value for a column, and converts // it to a string based on a foreign index string key. -func (f *Field) StringValue(qcx *Qcx, columnID uint64) (value string, exists bool, err error) { +func (f *Field) StringValue(qcx qc.QueryContext, columnID uint64) (value string, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return value, false, ErrBSIGroupNotFound @@ -1406,7 +1410,7 @@ func (f *Field) StringValue(qcx *Qcx, columnID uint64) (value string, exists boo } // Value reads a field value for a column. -func (f *Field) Value(qcx *Qcx, columnID uint64) (value int64, exists bool, err error) { +func (f *Field) Value(qcx qc.QueryContext, columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return 0, false, ErrBSIGroupNotFound @@ -1428,7 +1432,7 @@ func (f *Field) Value(qcx *Qcx, columnID uint64) (value int64, exists bool, err } // SetValue sets a field value for a column. -func (f *Field) SetValue(qcx *Qcx, columnID uint64, value int64) (changed bool, err error) { +func (f *Field) SetValue(qcx qc.QueryContext, columnID uint64, value int64) (changed bool, err error) { // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) if bsig == nil { @@ -1480,7 +1484,7 @@ func (f *Field) SetValue(qcx *Qcx, columnID uint64, value int64) (changed bool, } // ClearValue removes a field value for a column. -func (f *Field) ClearValue(qcx *Qcx, columnID uint64) (changed bool, err error) { +func (f *Field) ClearValue(qcx qc.QueryContext, columnID uint64) (changed bool, err error) { bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound @@ -1500,9 +1504,7 @@ func (f *Field) ClearValue(qcx *Qcx, columnID uint64) (changed bool, err error) return false, nil } -func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, error) { - tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: f.idx, Shard: shard}) - defer finisher(&err) +func (f *Field) MaxForShard(qcx qc.QueryContext, shard uint64, filter *Row) (ValCount, error) { bsig := f.bsiGroup(f.name) if bsig == nil { return ValCount{}, ErrBSIGroupNotFound @@ -1518,7 +1520,11 @@ func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro return ValCount{}, nil } - max, cnt, err := fragment.max(tx, filter, bsig.BitDepth) + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ValCount{}, err + } + max, cnt, err := fragment.max(qr, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "calling fragment.max") } @@ -1530,9 +1536,7 @@ func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro // MinForShard returns the minimum value which appears in this shard // (this field must be an Int or Decimal field). It also returns the // number of times the minimum value appears. -func (f *Field) MinForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, error) { - tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: f.idx, Shard: shard}) - defer finisher(&err) +func (f *Field) MinForShard(qcx qc.QueryContext, shard uint64, filter *Row) (ValCount, error) { bsig := f.bsiGroup(f.name) if bsig == nil { return ValCount{}, ErrBSIGroupNotFound @@ -1548,7 +1552,11 @@ func (f *Field) MinForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro return ValCount{}, nil } - min, cnt, err := fragment.min(tx, filter, bsig.BitDepth) + qr, err := fragment.qcxRead(qcx) + if err != nil { + return ValCount{}, err + } + min, cnt, err := fragment.min(qr, filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "calling fragment.min") } @@ -1590,7 +1598,7 @@ func (f *Field) valCountize(val int64, cnt uint64, bsig *bsiGroup) (ValCount, er } // Range performs a conditional operation on Field. -func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Row, error) { +func (f *Field) Range(qcx qc.QueryContext, name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) if bsig == nil { @@ -1614,7 +1622,7 @@ func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Ro } // Import bulk imports data. -func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) { +func (f *Field) Import(qcx qc.QueryContext, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) { // Determine quantum if timestamps are set. q := f.TimeQuantum() if len(timestamps) > 0 { @@ -1636,12 +1644,6 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, } } } - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: f.idx, Shard: shard}) - if err != nil { - return errors.Wrap(err, "qcx.GetTx") - } - var err1 error - defer finisher(&err1) view, err := f.createViewIfNotExists(viewStandard) if err != nil { return errors.Wrapf(err, "creating view %s", viewStandard) @@ -1652,8 +1654,11 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return errors.Wrap(err, "creating fragment") } - err1 = frag.bulkImport(tx, rowIDs, columnIDs, options) - return err1 + qw, err := frag.qcxWrite(qcx) + if err != nil { + return err + } + return frag.bulkImport(qw, rowIDs, columnIDs, options) } fieldType := f.Type() @@ -1717,12 +1722,10 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, } } } - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: f.idx, Shard: shard}) - if err != nil { - return errors.Wrap(err, "qcx.GetTx") - } - var err1 error - defer finisher(&err1) + + // in the Qcx/Tx era, we grabbed a single top-level Tx, because we secretly + // knew that Tx were shard-based, and didn't care about views. Now we request + // a QueryWrite per frag. for viewName, data := range views { view, err := f.createViewIfNotExists(viewName) if err != nil { @@ -1734,9 +1737,13 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, return errors.Wrap(err, "creating fragment") } - err1 = frag.bulkImport(tx, data.RowIDs, data.ColumnIDs, options) - if err1 != nil { - return err1 + qw, err := frag.qcxWrite(qcx) + if err != nil { + return err + } + err = frag.bulkImport(qw, data.RowIDs, data.ColumnIDs, options) + if err != nil { + return err } } return nil @@ -1745,7 +1752,7 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, // importFloatValue imports floating point values. In current usage, this // should only ever be called with data for a single shard; the API calls // around this are splitting it up per shard. -func (f *Field) importFloatValue(qcx *Qcx, columnIDs []uint64, values []float64, shard uint64, options *ImportOptions) error { +func (f *Field) importFloatValue(qcx qc.QueryContext, columnIDs []uint64, values []float64, shard uint64, options *ImportOptions) error { // convert values to int64 values based on scale ivalues := make([]int64, len(values)) bsig := f.bsiGroup(f.name) @@ -1763,7 +1770,7 @@ func (f *Field) importFloatValue(qcx *Qcx, columnIDs []uint64, values []float64, // importTimestampValue imports timestamp values. In current usage, this // should only ever be called with data for a single shard; the API calls // around this are splitting it up per shard. -func (f *Field) importTimestampValue(qcx *Qcx, columnIDs []uint64, values []time.Time, shard uint64, options *ImportOptions) error { +func (f *Field) importTimestampValue(qcx qc.QueryContext, columnIDs []uint64, values []time.Time, shard uint64, options *ImportOptions) error { ivalues := make([]int64, len(values)) bsig := f.bsiGroup(f.name) if bsig == nil { @@ -1779,7 +1786,7 @@ func (f *Field) importTimestampValue(qcx *Qcx, columnIDs []uint64, values []time // importValue bulk imports range-encoded value data. This function should // only be called with data for a single shard; the API calls that wrap // this handle splitting the data up per-shard. -func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, shard uint64, options *ImportOptions) (err0 error) { +func (f *Field) importValue(qcx qc.QueryContext, columnIDs []uint64, values []int64, shard uint64, options *ImportOptions) error { // no data to import if len(columnIDs) == 0 { return nil @@ -1870,19 +1877,15 @@ func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, shard } } - // now we know which shard we discovered. - tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: f.idx, Shard: frag.shard}) + // request a QueryWrite and write to it + qw, err := frag.qcxWrite(qcx) if err != nil { return err } - // defer the finisher, so it will check the error returned and - // possibly rollback. - defer finisher(&err0) - - return frag.importValue(tx, columnIDs, values, requiredDepth, options.Clear) + return frag.importValue(qw, columnIDs, values, requiredDepth, options.Clear) } -func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, clear bool) error { +func (f *Field) importRoaring(ctx context.Context, qcx qc.QueryContext, data []byte, shard uint64, viewName string, clear bool) error { span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaring") defer span.Finish() @@ -1899,7 +1902,11 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin if err != nil { return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaring(ctx, tx, data, clear); err != nil { + qw, err := frag.qcxWrite(qcx) + if err != nil { + return err + } + if err := frag.importRoaring(ctx, qw, data, clear); err != nil { return err } @@ -1910,7 +1917,7 @@ func (f *Field) GetIndex() *Index { return f.idx } -func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error { +func (f *Field) importRoaringOverwrite(ctx context.Context, qcx qc.QueryContext, data []byte, shard uint64, viewName string) error { span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite") defer span.Finish() @@ -1927,7 +1934,11 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, if err != nil { return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaringOverwrite(ctx, tx, data, block); err != nil { + qw, err := frag.qcxWrite(qcx) + if err != nil { + return errors.Wrap(err, "creating writer") + } + if err := frag.importRoaringOverwrite(ctx, qw, data); err != nil { return err } @@ -1935,8 +1946,12 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, // field.options.BitDepth and bsiGroup.BitDepth based on the imported data. switch f.Options().Type { case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: + qr, err := frag.qcxRead(qcx) + if err != nil { + return err + } frag.mu.Lock() - maxRowID, _, err := frag.maxRow(tx, nil) + maxRowID, _, err := frag.maxRow(qr, nil) frag.mu.Unlock() if err != nil { return err @@ -2368,7 +2383,7 @@ func CheckEpochOutOfRange(epoch, min, max time.Time) error { return nil } -func (f *Field) SortShardRow(tx Tx, shard uint64, filter *Row, sort_desc bool) (*SortedRow, error) { +func (f *Field) SortShardRow(qcx qc.QueryContext, shard uint64, filter *Row, sort_desc bool) (*SortedRow, error) { bsig := f.bsiGroup(f.name) if bsig == nil { return nil, errors.New("bsig is nil") @@ -2384,5 +2399,9 @@ func (f *Field) SortShardRow(tx Tx, shard uint64, filter *Row, sort_desc bool) ( return nil, errors.New("fragment is nil") } - return fragment.sortBsiData(tx, filter, bsig.BitDepth, sort_desc) + qr, err := fragment.qcxRead(qcx) + if err != nil { + return nil, err + } + return fragment.sortBsiData(qr, filter, bsig.BitDepth, sort_desc) } diff --git a/field_internal_test.go b/field_internal_test.go index d09843998..3c7fef38a 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -2,7 +2,6 @@ package pilosa import ( - "context" "fmt" "math" "reflect" @@ -12,9 +11,10 @@ import ( "time" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/shardwidth" - . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck + "github.com/stretchr/testify/require" ) // CorruptAMutex breaks a mutex in order to test the mutex-corruption stuff. @@ -25,7 +25,7 @@ import ( // // This always sets row 3 in column 0 of each shard it finds. Populate the // field with existing shards first. -func CorruptAMutex(tb testing.TB, field *Field, qcx *Qcx) { +func CorruptAMutex(tb testing.TB, field *Field, qcx qc.QueryContext) { v := field.view(viewStandard) if v == nil { tb.Fatalf("creating view failed") @@ -33,14 +33,13 @@ func CorruptAMutex(tb testing.TB, field *Field, qcx *Qcx) { frags := v.allFragments() for _, frag := range frags { func() { - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: field.idx, Shard: frag.shard}) - defer finisher(&err) + qw, err := frag.qcxWrite(qcx) if err != nil { tb.Fatalf("getting tx: %v", err) } // set a bonus bit, bypassing the mutex handling frag.mu.Lock() - _, err = frag.unprotectedSetBit(tx, 3, (frag.shard< 0 if err != nil { return false, errors.Wrap(err, "writing") @@ -391,13 +399,10 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo return changed, nil } - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) - // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { - n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth) + n, err := qw.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) if err != nil { return false, err } @@ -411,15 +416,15 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo // clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) clearBit(qw qc.QueryWrite, rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.unprotectedClearBit(tx, rowID, columnID) + return f.unprotectedClearBit(qw, rowID, columnID) } // unprotectedClearBit TODO should be replaced by an invocation of // importPositions with a single bit to clear. -func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) unprotectedClearBit(qw qc.QueryWrite, rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -429,7 +434,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b // Write to storage. changeCount := 0 - if changeCount, err = tx.Remove(f.index(), f.field(), f.view(), f.shard, pos); err != nil { + if changeCount, err = qw.Remove(pos); err != nil { return false, errors.Wrap(err, "writing") } @@ -440,13 +445,10 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b changed = true } - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) - // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { - n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth) + n, err := qw.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) if err != nil { return changed, err } @@ -460,13 +462,13 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b // setRow replaces an existing row (specified by rowID) with the given // Row. This updates both the on-disk storage and the in-cache bitmap. -func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) { +func (f *fragment) setRow(qw qc.QueryWrite, row *Row, rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.unprotectedSetRow(tx, row, rowID) + return f.unprotectedSetRow(qw, row, rowID) } -func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) { +func (f *fragment) unprotectedSetRow(qw qc.QueryWrite, row *Row, rowID uint64) (changed bool, err error) { // TODO: In order to return `changed`, we need to first compare // the existing row with the given row. Determine if the overhead // of this is worth having `changed`. @@ -478,7 +480,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo // Remove every existing container in the row. for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { - if err := tx.RemoveContainer(f.index(), f.field(), f.view(), f.shard, headContainerKey+i); err != nil { + if err := qw.RemoveContainer(headContainerKey + i); err != nil { return changed, err } } @@ -490,14 +492,14 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) for citer.Next() { k, c := citer.Value() - if err := tx.PutContainer(f.index(), f.field(), f.view(), f.shard, headContainerKey+(k%(1<= 0 || clear { - if c, err := f.unprotectedClearBit(tx, uint64(bsiSignBit), columnID); err != nil { + if c, err := f.unprotectedClearBit(qw, uint64(bsiSignBit), columnID); err != nil { return errors.Wrap(err, "clearing sign") } else if c { changed = true } } else { - if c, err := f.unprotectedSetBit(tx, uint64(bsiSignBit), columnID); err != nil { + if c, err := f.unprotectedSetBit(qw, uint64(bsiSignBit), columnID); err != nil { return errors.Wrap(err, "marking sign") } else if c { changed = true @@ -726,7 +706,7 @@ func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint64, value i // sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count uint64, err error) { +func (f *fragment) sum(qr qc.QueryRead, filter *Row, bitDepth uint64) (sum int64, count uint64, err error) { // If there's a provided filter, but it has no contents for this particular // shard, we're done and can return early. If there's no provided filter, // though, we want to run with no-filter, as opposed to an empty filter. @@ -744,7 +724,7 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count ui } } bsiFilt := roaring.NewBitmapBSICountFilter(filterData) - err = tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, 0, bsiFilt) + err = qr.ApplyFilter(0, bsiFilt) if err != nil && err != io.EOF { return sum, count, errors.Wrap(err, "finding existing positions") } @@ -756,8 +736,8 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count ui // min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) min(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) { - consider, err := f.row(tx, bsiExistsBit) +func (f *fragment) min(qr qc.QueryRead, filter *Row, bitDepth uint64) (min int64, count uint64, err error) { + consider, err := f.row(qr, bsiExistsBit) if err != nil { return min, count, err } else if filter != nil { @@ -773,22 +753,22 @@ func (f *fragment) min(tx Tx, filter *Row, bitDepth uint64) (min int64, count ui // from that set, then negate it, and return it. For example, if values // (-1, -2) exist, they are stored unsigned (1,2) with a negative sign bit // set. We take the highest of that set (2) and negate it and return it. - if row, err := f.row(tx, bsiSignBit); err != nil { + if row, err := f.row(qr, bsiSignBit); err != nil { return min, count, err } else if row = row.Intersect(consider); row.Any() { - min, count, err := f.maxUnsigned(tx, row, bitDepth) + min, count, err := f.maxUnsigned(qr, row, bitDepth) return -min, count, err } // Otherwise find lowest positive number. - return f.minUnsigned(tx, consider, bitDepth) + return f.minUnsigned(qr, consider, bitDepth) } // minUnsigned the lowest value without considering the sign bit. Filter is required. -func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) { +func (f *fragment) minUnsigned(qr qc.QueryRead, filter *Row, bitDepth uint64) (min int64, count uint64, err error) { count = filter.Count() for i := int(bitDepth - 1); i >= 0; i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return min, count, err } @@ -808,8 +788,8 @@ func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint64) (min int64, // max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) max(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) { - consider, err := f.row(tx, bsiExistsBit) +func (f *fragment) max(qr qc.QueryRead, filter *Row, bitDepth uint64) (max int64, count uint64, err error) { + consider, err := f.row(qr, bsiExistsBit) if err != nil { return max, count, err } else if filter != nil { @@ -822,25 +802,25 @@ func (f *fragment) max(tx Tx, filter *Row, bitDepth uint64) (max int64, count ui } // Find lowest negative number w/o sign and negate, if no positives are available. - row, err := f.row(tx, bsiSignBit) + row, err := f.row(qr, bsiSignBit) if err != nil { return max, count, err } pos := consider.Difference(row) if !pos.Any() { - max, count, err = f.minUnsigned(tx, consider, bitDepth) + max, count, err = f.minUnsigned(qr, consider, bitDepth) return -max, count, err } // Otherwise find highest positive number. - return f.maxUnsigned(tx, pos, bitDepth) + return f.maxUnsigned(qr, pos, bitDepth) } // maxUnsigned the highest value without considering the sign bit. Filter is required. -func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) { +func (f *fragment) maxUnsigned(qr qc.QueryRead, filter *Row, bitDepth uint64) (max int64, count uint64, err error) { count = filter.Count() for i := int(bitDepth - 1); i >= 0; i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return max, count, err } @@ -860,8 +840,8 @@ func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint64) (max int64, // minRow returns minRowID of the rows in the filter and its count. // if filter is nil, it returns fragment.minRowID, 1 // if fragment has no rows, it returns 0, 0 -func (f *fragment) minRow(tx Tx, filter *Row) (uint64, uint64, error) { - minRowID, hasRowID, err := f.minRowID(tx) +func (f *fragment) minRow(qr qc.QueryRead, filter *Row) (uint64, uint64, error) { + minRowID, hasRowID, err := f.minRowID(qr) if err != nil { return 0, 0, err } @@ -871,14 +851,14 @@ func (f *fragment) minRow(tx Tx, filter *Row) (uint64, uint64, error) { } // Read last bit to determine max row. - maxRowID, err := f.maxRowID(tx) + maxRowID, err := f.maxRowID(qr) if err != nil { return 0, 0, err } // iterate from min row ID and return the first that intersects with filter. for i := minRowID; i <= maxRowID; i++ { - row, err := f.row(tx, i) + row, err := f.row(qr, i) if err != nil { return 0, 0, err } @@ -896,13 +876,13 @@ func (f *fragment) minRow(tx Tx, filter *Row) (uint64, uint64, error) { // maxRow returns maxRowID of the rows in the filter and its count. // if filter is nil, it returns fragment.maxRowID, 1 // if fragment has no rows, it returns 0, 0 -func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) { - minRowID, hasRowID, err := f.minRowID(tx) +func (f *fragment) maxRow(qr qc.QueryRead, filter *Row) (uint64, uint64, error) { + minRowID, hasRowID, err := f.minRowID(qr) if err != nil { return 0, 0, err } if hasRowID { - maxRowID, err := f.maxRowID(tx) + maxRowID, err := f.maxRowID(qr) if err != nil { return 0, 0, err } @@ -913,7 +893,7 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) { // iterate back from max row ID and return the first that intersects with filter. // TODO: implement reverse container iteration to improve performance here for sparse data. --Jaffee for i := maxRowID; i >= minRowID; i-- { - row, err := f.row(tx, i) + row, err := f.row(qr, i) if err != nil { return 0, 0, err } @@ -930,8 +910,8 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) { // maxRowID determines the field's maxRowID value based // on the contents of its storage, and sets the struct argument. -func (f *fragment) maxRowID(tx Tx) (_ uint64, err error) { - max, err := tx.Max(f.index(), f.field(), f.view(), f.shard) +func (f *fragment) maxRowID(qr qc.QueryRead) (_ uint64, err error) { + max, err := qr.Max() if err != nil { return 0, err } @@ -939,16 +919,16 @@ func (f *fragment) maxRowID(tx Tx) (_ uint64, err error) { } // rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) { +func (f *fragment) rangeOp(qr qc.QueryRead, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) { switch op { case pql.EQ: - return f.rangeEQ(tx, bitDepth, predicate) + return f.rangeEQ(qr, bitDepth, predicate) case pql.NEQ: - return f.rangeNEQ(tx, bitDepth, predicate) + return f.rangeNEQ(qr, bitDepth, predicate) case pql.LT, pql.LTE: - return f.rangeLT(tx, bitDepth, predicate, op == pql.LTE) + return f.rangeLT(qr, bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.rangeGT(tx, bitDepth, predicate, op == pql.GTE) + return f.rangeGT(qr, bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } @@ -965,9 +945,9 @@ func absInt64(v int64) uint64 { } } -func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) { +func (f *fragment) rangeEQ(qr qc.QueryRead, bitDepth uint64, predicate int64) (*Row, error) { // Start with set of columns with values set. - b, err := f.row(tx, bsiExistsBit) + b, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } @@ -979,7 +959,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error } // Filter to only positive/negative numbers. - r, err := f.row(tx, bsiSignBit) + r, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } @@ -991,7 +971,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return nil, err } @@ -1007,15 +987,15 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error return b, nil } -func (f *fragment) rangeNEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) { +func (f *fragment) rangeNEQ(qr qc.QueryRead, bitDepth uint64, predicate int64) (*Row, error) { // Start with set of columns with values set. - b, err := f.row(tx, bsiExistsBit) + b, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } // Get the equal bitmap. - eq, err := f.rangeEQ(tx, bitDepth, predicate) + eq, err := f.rangeEQ(qr, bitDepth, predicate) if err != nil { return nil, err } @@ -1026,19 +1006,19 @@ func (f *fragment) rangeNEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, erro return b, nil } -func (f *fragment) rangeLT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeLT(qr qc.QueryRead, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) { if predicate == 1 && !allowEquality { predicate, allowEquality = 0, true } // Start with set of columns with values set. - b, err := f.row(tx, bsiExistsBit) + b, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } // Get the sign bit row. - sign, err := f.row(tx, bsiSignBit) + sign, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } @@ -1052,17 +1032,17 @@ func (f *fragment) rangeLT(tx Tx, bitDepth uint64, predicate int64, allowEqualit return b.Intersect(sign), nil case predicate == 0 && allowEquality: // Match all integers that are either negative or 0. - zeroes, err := f.rangeEQ(tx, bitDepth, 0) + zeroes, err := f.rangeEQ(qr, bitDepth, 0) if err != nil { return nil, err } return b.Intersect(sign).Union(zeroes), nil case predicate < 0: // Match all every negative number beyond the predicate. - return f.rangeGTUnsigned(tx, b.Intersect(sign), bitDepth, upredicate, allowEquality) + return f.rangeGTUnsigned(qr, b.Intersect(sign), bitDepth, upredicate, allowEquality) default: // Match positive numbers less than the predicate, and all negatives. - pos, err := f.rangeLTUnsigned(tx, b.Difference(sign), bitDepth, upredicate, allowEquality) + pos, err := f.rangeLTUnsigned(qr, b.Difference(sign), bitDepth, upredicate, allowEquality) if err != nil { return nil, err } @@ -1072,7 +1052,7 @@ func (f *fragment) rangeLT(tx Tx, bitDepth uint64, predicate int64, allowEqualit } // rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. -func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeLTUnsigned(qr qc.QueryRead, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) { switch { case uint64(bits.Len64(predicate)) > bitDepth: fallthrough @@ -1083,7 +1063,7 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicat // This query matches everything that is not (1<= 0 && predicate > 0 && remaining.Any(); i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return nil, err } @@ -1117,26 +1097,26 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicat return matched, nil } -func (f *fragment) rangeGT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeGT(qr qc.QueryRead, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) { if predicate == -1 && !allowEquality { predicate, allowEquality = 0, true } - b, err := f.row(tx, bsiExistsBit) + b, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } // Create predicate without sign bit. upredicate := absInt64(predicate) - sign, err := f.row(tx, bsiSignBit) + sign, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } switch { case predicate == 0 && !allowEquality: // Match all positive numbers except zero. - nonzero, err := f.rangeNEQ(tx, bitDepth, 0) + nonzero, err := f.rangeNEQ(qr, bitDepth, 0) if err != nil { return nil, err } @@ -1147,10 +1127,10 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint64, predicate int64, allowEqualit return b.Difference(sign), nil case predicate >= 0: // Match all positive numbers greater than the predicate. - return f.rangeGTUnsigned(tx, b.Difference(sign), bitDepth, upredicate, allowEquality) + return f.rangeGTUnsigned(qr, b.Difference(sign), bitDepth, upredicate, allowEquality) default: // Match all positives and greater negatives. - neg, err := f.rangeLTUnsigned(tx, b.Intersect(sign), bitDepth, upredicate, allowEquality) + neg, err := f.rangeLTUnsigned(qr, b.Intersect(sign), bitDepth, upredicate, allowEquality) if err != nil { return nil, err } @@ -1159,7 +1139,7 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint64, predicate int64, allowEqualit } } -func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeGTUnsigned(qr qc.QueryRead, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) { prep: switch { case predicate == 0 && allowEquality: @@ -1169,7 +1149,7 @@ prep: // This query matches everything that is not 0. matches := NewRow() for i := uint64(0); i < bitDepth; i++ { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return nil, err } @@ -1190,7 +1170,7 @@ prep: remaining := filter predicate |= (^uint64(0)) << bitDepth for i := int(bitDepth - 1); i >= 0 && predicate < ^uint64(0) && remaining.Any(); i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return nil, err } @@ -1210,13 +1190,13 @@ prep: } // notNull returns the exists row. -func (f *fragment) notNull(tx Tx) (*Row, error) { - return f.row(tx, bsiExistsBit) +func (f *fragment) notNull(qr qc.QueryRead) (*Row, error) { + return f.row(qr, bsiExistsBit) } // rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *fragment) rangeBetween(tx Tx, bitDepth uint64, predicateMin, predicateMax int64) (*Row, error) { - b, err := f.row(tx, bsiExistsBit) +func (f *fragment) rangeBetween(qr qc.QueryRead, bitDepth uint64, predicateMin, predicateMax int64) (*Row, error) { + b, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } @@ -1226,36 +1206,36 @@ func (f *fragment) rangeBetween(tx Tx, bitDepth uint64, predicateMin, predicateM switch { case predicateMin == predicateMax: - return f.rangeEQ(tx, bitDepth, predicateMin) + return f.rangeEQ(qr, bitDepth, predicateMin) case predicateMin >= 0: // Handle positive-only values. - r, err := f.row(tx, bsiSignBit) + r, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } - return f.rangeBetweenUnsigned(tx, b.Difference(r), bitDepth, upredicateMin, upredicateMax) + return f.rangeBetweenUnsigned(qr, b.Difference(r), bitDepth, upredicateMin, upredicateMax) case predicateMax < 0: // Handle negative-only values. Swap unsigned min/max predicates. - r, err := f.row(tx, bsiSignBit) + r, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } - return f.rangeBetweenUnsigned(tx, b.Intersect(r), bitDepth, upredicateMax, upredicateMin) + return f.rangeBetweenUnsigned(qr, b.Intersect(r), bitDepth, upredicateMax, upredicateMin) default: // If predicate crosses positive/negative boundary then handle separately and union. - r0, err := f.row(tx, bsiSignBit) + r0, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } - pos, err := f.rangeLTUnsigned(tx, b.Difference(r0), bitDepth, upredicateMax, true) + pos, err := f.rangeLTUnsigned(qr, b.Difference(r0), bitDepth, upredicateMax, true) if err != nil { return nil, err } - r1, err := f.row(tx, bsiSignBit) + r1, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } - neg, err := f.rangeLTUnsigned(tx, b.Intersect(r1), bitDepth, upredicateMin, true) + neg, err := f.rangeLTUnsigned(qr, b.Intersect(r1), bitDepth, upredicateMin, true) if err != nil { return nil, err } @@ -1264,21 +1244,21 @@ func (f *fragment) rangeBetween(tx Tx, bitDepth uint64, predicateMin, predicateM } // rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. -func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint64, predicateMin, predicateMax uint64) (*Row, error) { +func (f *fragment) rangeBetweenUnsigned(qr qc.QueryRead, filter *Row, bitDepth uint64, predicateMin, predicateMax uint64) (*Row, error) { switch { case predicateMax > (1<= diffLen; i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return nil, err } @@ -1296,11 +1276,11 @@ func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint64, pre predicateMax &^= equalMask var err error - remaining, err = f.rangeGTUnsigned(tx, remaining, uint64(diffLen), predicateMin, true) + remaining, err = f.rangeGTUnsigned(qr, remaining, uint64(diffLen), predicateMin, true) if err != nil { return nil, err } - remaining, err = f.rangeLTUnsigned(tx, remaining, uint64(diffLen), predicateMax, true) + remaining, err = f.rangeLTUnsigned(qr, remaining, uint64(diffLen), predicateMax, true) if err != nil { return nil, err } @@ -1319,9 +1299,9 @@ func (f *fragment) pos(rowID, columnID uint64) (uint64, error) { // top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. -func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) { +func (f *fragment) top(qr qc.QueryRead, opt topOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. - pairs, err := f.topBitmapPairs(tx, opt.RowIDs) + pairs, err := f.topBitmapPairs(qr, opt.RowIDs) if err != nil { return nil, err } @@ -1370,7 +1350,7 @@ func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - r, err := f.row(tx, rowID) + r, err := f.row(qr, rowID) if err != nil { return nil, err } @@ -1418,7 +1398,7 @@ func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - r, err := f.row(tx, rowID) + r, err := f.row(qr, rowID) if err != nil { return nil, err } @@ -1441,7 +1421,7 @@ func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) { return r, nil } -func (f *fragment) topBitmapPairs(tx Tx, rowIDs []uint64) ([]bitmapPair, error) { +func (f *fragment) topBitmapPairs(qr qc.QueryRead, rowIDs []uint64) ([]bitmapPair, error) { // Don't retrieve from storage if CacheTypeNone. if f.CacheType == CacheTypeNone { return f.cache.Top(), nil @@ -1466,7 +1446,7 @@ func (f *fragment) topBitmapPairs(tx Tx, rowIDs []uint64) ([]bitmapPair, error) continue } - row, err := f.row(tx, rowID) + row, err := f.row(qr, rowID) if err != nil { return nil, err } @@ -1500,16 +1480,16 @@ type topOptions struct { // bulkImport bulk imports a set of bits. // The cache is updated to reflect the new data. -func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) error { +func (f *fragment) bulkImport(qw qc.QueryWrite, rowIDs, columnIDs []uint64, options *ImportOptions) error { // Verify that there are an equal number of row ids and column ids. if len(rowIDs) != len(columnIDs) { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } if f.mutexVector != nil && !options.Clear { - return f.bulkImportMutex(tx, rowIDs, columnIDs, options) + return f.bulkImportMutex(qw, rowIDs, columnIDs, options) } - return f.bulkImportStandard(tx, rowIDs, columnIDs, options) + return f.bulkImportStandard(qw, rowIDs, columnIDs, options) } // rowColumnSet is a sortable set of row and column IDs which @@ -1542,7 +1522,7 @@ func (r rowColumnSet) Less(i, j int) bool { // bulkImportStandard performs a bulk import on a standard fragment. May mutate // its rowIDs and columnIDs arguments. -func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) (err error) { +func (f *fragment) bulkImportStandard(qw qc.QueryWrite, rowIDs, columnIDs []uint64, options *ImportOptions) (err error) { // rowSet maintains the set of rowIDs present in this import. It allows the // cache to be updated once per row, instead of once per bit. TODO: consider // sorting by rowID/columnID first and avoiding the map allocation here. (we @@ -1581,9 +1561,9 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options f.mu.Lock() defer f.mu.Unlock() if options.Clear { - err = f.importPositions(tx, nil, positions, rowSet) + err = f.importPositions(qw, nil, positions, rowSet) } else { - err = f.importPositions(tx, positions, nil, rowSet) + err = f.importPositions(qw, positions, nil, rowSet) } return errors.Wrap(err, "bulkImportStandard") } @@ -1710,12 +1690,12 @@ func (p parallelSlices) Swap(i, j int) { // importPositions tries to intelligently decide whether or not to do a full // snapshot of the fragment or just do in-memory updates while appending // operations to the op log. -func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error { +func (f *fragment) importPositions(qw qc.QueryWrite, set, clear []uint64, rowSet map[uint64]struct{}) error { if len(set) > 0 { f.stats.Count(MetricImportingN, int64(len(set)), 1) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions - changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...) + changedN, err := qw.Add(set...) if err != nil { return errors.Wrap(err, "adding positions") } @@ -1724,30 +1704,28 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 if len(clear) > 0 { f.stats.Count(MetricClearingN, int64(len(clear)), 1) - changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...) + changedN, err := qw.Remove(clear...) if err != nil { return errors.Wrap(err, "clearing positions") } f.stats.Count(MetricClearedN, int64(changedN), 1) } - return f.updateCaching(tx, rowSet) + return f.updateCaching(qw, rowSet) } -// updateCaching clears checksums for rows, and clears any existing TopN -// cache for them, and marks the cache for needing updates. I'm not sure +// updateCaching clears any existing TopN +// cache for rows, and marks the cache for needing updates. I'm not sure // that's correct. This was originally the tail end of importPositions, but // we want to be able to access the same logic from elsewhere. -func (f *fragment) updateCaching(tx Tx, rowSet map[uint64]struct{}) error { +// it used to clear block checksums but we don't maintain those anymore. +func (f *fragment) updateCaching(qr qc.QueryRead, rowSet map[uint64]struct{}) error { // Update cache counts for all affected rows. for rowID := range rowSet { - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) - if f.CacheType != CacheTypeNone { start := rowID * ShardWidth end := (rowID + 1) * ShardWidth - n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, start, end) + n, err := qr.CountRange(start, end) if err != nil { return errors.Wrap(err, "CountRange") } @@ -1795,7 +1773,7 @@ func sliceDifference(original, remove []uint64) []uint64 { // mutex restrictions. Because the mutex requirements must be checked // against storage, this method must acquire a write lock on the fragment // during the entire process, and it handles every bit independently. -func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) error { +func (f *fragment) bulkImportMutex(qw qc.QueryWrite, rowIDs, columnIDs []uint64, options *ImportOptions) error { f.mu.Lock() defer f.mu.Unlock() @@ -1913,30 +1891,30 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64, options *I return nil }) - err := tx.ApplyRewriter(f.index(), f.field(), f.view(), f.shard, 0, rewriteExisting) + err := qw.ApplyRewriter(0, rewriteExisting) if err != nil { return err } - return f.updateCaching(tx, rowSet) + return f.updateCaching(qw, rowSet) } // ClearRecords deletes all bits for the given records. It's basically // the remove-only part of setting a mutex. -func (f *fragment) ClearRecords(tx Tx, recordIDs []uint64) (bool, error) { +func (f *fragment) ClearRecords(qw qc.QueryWrite, recordIDs []uint64) (bool, error) { // create a mask of columns we care about columns := roaring.NewSliceBitmap(recordIDs...) - return f.clearRecordsByBitmap(tx, columns) + return f.clearRecordsByBitmap(qw, columns) } -func (f *fragment) clearRecordsByBitmap(tx Tx, columns *roaring.Bitmap) (changed bool, err error) { +func (f *fragment) clearRecordsByBitmap(qw qc.QueryWrite, columns *roaring.Bitmap) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.unprotectedClearRecordsByBitmap(tx, columns) + return f.unprotectedClearRecordsByBitmap(qw, columns) } // clearRecordsByBitmap clears bits in a fragment that correspond to those // positions within the bitmap. -func (f *fragment) unprotectedClearRecordsByBitmap(tx Tx, columns *roaring.Bitmap) (changed bool, err error) { +func (f *fragment) unprotectedClearRecordsByBitmap(qw qc.QueryWrite, columns *roaring.Bitmap) (changed bool, err error) { rowSet := make(map[uint64]struct{}) rewriteExisting := roaring.NewBitmapBitmapTrimmer(columns, func(key roaring.FilterKey, data *roaring.Container, filter *roaring.Container, writeback roaring.ContainerWriteback) error { if filter.N() == 0 { @@ -1957,15 +1935,15 @@ func (f *fragment) unprotectedClearRecordsByBitmap(tx Tx, columns *roaring.Bitma return nil }) - err = tx.ApplyRewriter(f.index(), f.field(), f.view(), f.shard, 0, rewriteExisting) + err = qw.ApplyRewriter(0, rewriteExisting) if err != nil { return false, err } - return changed, f.updateCaching(tx, rowSet) + return changed, f.updateCaching(qw, rowSet) } // importValue bulk imports a set of range-encoded values. -func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error { +func (f *fragment) importValue(qw qc.QueryWrite, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -2042,7 +2020,7 @@ func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDep } for i := range positionsByDepth { - err := f.importPositions(tx, positionsByDepth[i][:toSetByDepth[i]], positionsByDepth[i][toClearByDepth[i]:], nil) + err := f.importPositions(qw, positionsByDepth[i][:toSetByDepth[i]], positionsByDepth[i][toClearByDepth[i]:], nil) if err != nil { return errors.Wrap(err, "importing positions") } @@ -2056,11 +2034,11 @@ func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDep // importRoaring imports from the official roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. -func (f *fragment) importRoaring(ctx context.Context, tx Tx, data []byte, clear bool) error { +func (f *fragment) importRoaring(ctx context.Context, qw qc.QueryWrite, data []byte, clear bool) error { span, ctx := tracing.StartSpanFromContext(ctx, "fragment.importRoaring") defer span.Finish() - rowSet, updateCache, err := f.doImportRoaring(ctx, tx, data, clear) + rowSet, updateCache, err := f.doImportRoaring(ctx, qw, data, clear) if err != nil { return errors.Wrap(err, "doImportRoaring") } @@ -2071,7 +2049,7 @@ func (f *fragment) importRoaring(ctx context.Context, tx Tx, data []byte, clear } // ImportRoaringClearAndSet simply clears the bits in clear and sets the bits in set. -func (f *fragment) ImportRoaringClearAndSet(ctx context.Context, tx Tx, clear, set []byte) error { +func (f *fragment) ImportRoaringClearAndSet(ctx context.Context, qw qc.QueryWrite, clear, set []byte) error { clearIter, err := roaring.NewContainerIterator(clear) if err != nil { return errors.Wrap(err, "getting clear iterator") @@ -2086,7 +2064,7 @@ func (f *fragment) ImportRoaringClearAndSet(ctx context.Context, tx Tx, clear, s return errors.Wrap(err, "getting rewriter") } - err = tx.ApplyRewriter(f.index(), f.field(), f.view(), f.shard, 0, rewriter) + err = qw.ApplyRewriter(0, rewriter) if err != nil { return fmt.Errorf("pilosa.ImportRoaringClearAndSet: %s", err) } @@ -2097,7 +2075,7 @@ func (f *fragment) ImportRoaringClearAndSet(ctx context.Context, tx Tx, clear, s // significant changes to the Rewriter API. f.mu.Lock() defer f.mu.Unlock() - return f.rebuildRankCache(ctx, tx) + return f.rebuildRankCache(ctx, qw) } return nil } @@ -2105,7 +2083,7 @@ func (f *fragment) ImportRoaringClearAndSet(ctx context.Context, tx Tx, clear, s // ImportRoaringBSI interprets "clear" as a single row specifying // records to be cleared, and "set" as specifying the values to be set // which implies clearing any other values in those columns. -func (f *fragment) ImportRoaringBSI(ctx context.Context, tx Tx, clear, set []byte) error { +func (f *fragment) ImportRoaringBSI(ctx context.Context, qw qc.QueryWrite, clear, set []byte) error { // In this first block, we take the first row of clear as records // we want to unconditionally clear, and the first row of set as // records we also want to clear because they're going to get set @@ -2130,8 +2108,8 @@ func (f *fragment) ImportRoaringBSI(ctx context.Context, tx Tx, clear, set []byt return errors.Wrap(err, "getting rewriter") } - err = tx.ApplyRewriter(f.index(), f.field(), f.view(), f.shard, 0, rewriter) - return errors.Wrap(err, "pilosa.ImportRoaringBSI: ") + err = qw.ApplyRewriter(0, rewriter) + return errors.Wrap(err, "applying rewriter in ImportRoaringBSI") } // ImportRoaringSingleValued treats "clear" as a single row and clears @@ -2139,7 +2117,7 @@ func (f *fragment) ImportRoaringBSI(ctx context.Context, tx Tx, clear, set []byt // similar to ImportRoaringBSI, but doesn't treate the first row of // "set" as the existence row to also be cleared. Essentially it's for // FieldTypeMutex. -func (f *fragment) ImportRoaringSingleValued(ctx context.Context, tx Tx, clear, set []byte) error { +func (f *fragment) ImportRoaringSingleValued(ctx context.Context, qw qc.QueryWrite, clear, set []byte) error { clearIter, err := roaring.NewRepeatedRowIteratorFromBytes(clear) if err != nil { return errors.Wrap(err, "getting cleariterator") @@ -2154,11 +2132,11 @@ func (f *fragment) ImportRoaringSingleValued(ctx context.Context, tx Tx, clear, return errors.Wrap(err, "getting rewriter") } - err = tx.ApplyRewriter(f.index(), f.field(), f.view(), f.shard, 0, rewriter) - return errors.Wrap(err, "pilosa.ImportRoaringSingleValued: ") + err = qw.ApplyRewriter(0, rewriter) + return errors.Wrap(err, "applying rewriter in ImportRoaringSingleValued") } -func (f *fragment) doImportRoaring(ctx context.Context, tx Tx, data []byte, clear bool) (map[uint64]int, bool, error) { +func (f *fragment) doImportRoaring(ctx context.Context, qw qc.QueryWrite, data []byte, clear bool) (map[uint64]int, bool, error) { f.mu.RLock() defer f.mu.RUnlock() rowSize := uint64(1 << shardVsContainerExponent) @@ -2173,7 +2151,7 @@ func (f *fragment) doImportRoaring(ctx context.Context, tx Tx, data []byte, clea return err } - _, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize) + _, rowSet, err = qw.ImportRoaringBits(rit, clear, rowSize) return err }() if err != nil { @@ -2214,14 +2192,23 @@ func (f *fragment) updateCachePostImport(ctx context.Context, rowSet map[uint64] } // importRoaringOverwrite overwrites the specified block with the provided data. -func (f *fragment) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, block int) error { - // Clear the existing data from fragment block. - if _, err := f.clearBlock(tx, block); err != nil { - return errors.Wrapf(err, "clearing block: %d", block) +func (f *fragment) importRoaringOverwrite(ctx context.Context, qw qc.QueryWrite, data []byte) error { + // This is just like ImportRoaringBSI, except we just use a virtual + // iterator that has an infinite stream of full containers. + clearIter := roaring.NewInfiniteOnesIterator() + + // Then we get the set iterator and create the rewriter. + setIter, err := roaring.NewContainerIterator(data) + if err != nil { + return errors.Wrap(err, "getting set iterator") + } + rewriter, err := roaring.NewClearAndSetRewriter(clearIter, setIter) + if err != nil { + return errors.Wrap(err, "getting rewriter") } - // Union the new block data with the fragment data. - return f.importRoaring(ctx, tx, data, false) + err = qw.ApplyRewriter(0, rewriter) + return errors.Wrap(err, "applying rewriter") } // RecalculateCache rebuilds the cache regardless of invalidate time delay. @@ -2238,18 +2225,18 @@ func (f *fragment) FlushCache() error { return f.flushCache() } -func (f *fragment) rebuildRankCache(ctx context.Context, tx Tx) error { +func (f *fragment) rebuildRankCache(ctx context.Context, qr qc.QueryRead) error { if f.CacheType != CacheTypeRanked { return nil // only rebuild ranked caches } f.cache.Clear() - rows, err := f.unprotectedRows(ctx, tx, uint64(0)) + rows, err := f.unprotectedRows(ctx, qr, uint64(0)) if err != nil { return err } for _, id := range rows { - n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, id*ShardWidth, (id+1)*ShardWidth) + n, err := qr.CountRange(id*ShardWidth, (id+1)*ShardWidth) if err != nil { return errors.Wrap(err, "CountRange") } @@ -2259,18 +2246,13 @@ func (f *fragment) rebuildRankCache(ctx context.Context, tx Tx) error { return nil } -func (f *fragment) RebuildRankCache(ctx context.Context) error { +func (f *fragment) RebuildRankCache(ctx context.Context, qr qc.QueryRead) error { if f.CacheType != CacheTypeRanked { return nil //only rebuild ranked caches } f.mu.Lock() defer f.mu.Unlock() - tx, err := f.holder.BeginTx(false, f.idx, f.shard) - if err != nil { - return err - } - defer tx.Rollback() - return f.rebuildRankCache(ctx, tx) + return f.rebuildRankCache(ctx, qr) } func (f *fragment) flushCache() error { @@ -2302,175 +2284,8 @@ func (f *fragment) flushCache() error { return nil } -// WriteTo writes the fragment's data to w. -func (f *fragment) WriteTo(w io.Writer) (n int64, err error) { - // Force cache flush. - if err := f.FlushCache(); err != nil { - return 0, errors.Wrap(err, "flushing cache") - } - - // Write out data and cache to a tar archive. - tw := tar.NewWriter(w) - if err := f.writeStorageToArchive(tw); err != nil { - return 0, fmt.Errorf("write storage: %s", err) - } - if err := f.writeCacheToArchive(tw); err != nil { - return 0, fmt.Errorf("write cache: %s", err) - } - return 0, nil -} - -// used in shipping the slices across the network for a resize. -func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { - - tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) - defer tx.Rollback() - rbm, err := tx.RoaringBitmap(f.index(), f.field(), f.view(), f.shard) - if err != nil { - return errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err := rbm.WriteTo(&buf) - if err != nil { - return errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - - // Write archive header. - if err := tw.WriteHeader(&tar.Header{ - Name: "data", - Mode: 0600, - Size: sz, - ModTime: time.Now(), - }); err != nil { - return errors.Wrap(err, "writing header") - } - - // Copy the file up to the last known size. - // This is done outside the lock because the storage format is append-only. - if _, err := io.CopyN(tw, &buf, sz); err != nil { - return errors.Wrap(err, "copying") - } - return nil -} - -func (f *fragment) writeCacheToArchive(tw *tar.Writer) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Read cache into buffer. - buf, err := os.ReadFile(f.cachePath()) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "reading cache") - } - - // Write archive header. - if err := tw.WriteHeader(&tar.Header{ - Name: "cache", - Mode: 0600, - Size: int64(len(buf)), - ModTime: time.Now(), - }); err != nil { - return errors.Wrap(err, "writing header") - } - - // Write data to archive. - if _, err := tw.Write(buf); err != nil { - return errors.Wrap(err, "writing") - } - return nil -} - -// ReadFrom reads a data file from r and loads it into the fragment. -func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { - f.mu.Lock() - defer f.mu.Unlock() - - tr := tar.NewReader(r) - for { - // Read next tar header. - hdr, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return 0, errors.Wrap(err, "opening") - } - - // Process file based on file name. - switch hdr.Name { - case "data": - tx := f.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - if err := f.fillFragmentFromArchive(tx, tr); err != nil { - return 0, errors.Wrap(err, "reading storage") - } - if err := tx.Commit(); err != nil { - return 0, errors.Wrap(err, "Commit after tx.ReadFragmentFromArchive") - } - case "cache": - if err := f.readCacheFromArchive(tr); err != nil { - return 0, errors.Wrap(err, "reading cache") - } - default: - return 0, fmt.Errorf("invalid fragment archive file: %s", hdr.Name) - } - } - - return 0, nil -} - -// should be morally equivalent to fragment.readStorageFromArchive() -// below for RoaringTx, but also work on any Tx because it uses -// tx.ImportRoaringBits(). -func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { - - // this is reading from inside a tarball, so definitely no need - // to close it here. - data, err := io.ReadAll(r) - if err != nil { - return errors.Wrap(err, "fillFragmentFromArchive io.ReadAll(r)") - } - if len(data) == 0 { - return nil - } - - // For reference, compare to what fragment.go:313 fragment.importStorage() does. - - clear := false - log := false - rowSize := uint64(0) - itr, err := roaring.NewRoaringIterator(data) - if err != nil { - return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator") - } - changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize) - _, _ = changed, rowSet - if err != nil { - return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits") - } - return nil -} - -func (f *fragment) readCacheFromArchive(r io.Reader) error { - // Slurp data from reader and write to disk. - buf, err := io.ReadAll(r) - if err != nil { - return errors.Wrap(err, "reading") - } else if err := os.WriteFile(f.cachePath(), buf, 0600); err != nil { - return errors.Wrap(err, "writing") - } - - // Re-open cache. - if err := f.openCache(); err != nil { - return errors.Wrap(err, "opening") - } - - return nil -} - -func (f *fragment) minRowID(tx Tx) (uint64, bool, error) { - min, ok, err := tx.Min(f.index(), f.field(), f.view(), f.shard) +func (f *fragment) minRowID(qr qc.QueryRead) (uint64, bool, error) { + min, ok, err := qr.Min() return min / ShardWidth, ok, err } @@ -2483,14 +2298,14 @@ func (f *fragment) minRowID(tx Tx) (uint64, bool, error) { // returning done == true will cause processing to stop after all filters for // this container have been processed. The rows accumulated up to this point // (including this row if all filters passed) will be returned. -func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) { +func (f *fragment) rows(ctx context.Context, qr qc.QueryRead, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) { f.mu.RLock() defer f.mu.RUnlock() - return f.unprotectedRows(ctx, tx, start, filters...) + return f.unprotectedRows(ctx, qr, start, filters...) } // unprotectedRows calls rows without grabbing the mutex. -func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) { +func (f *fragment) unprotectedRows(ctx context.Context, qr qc.QueryRead, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) { var rows []uint64 cb := func(row uint64) error { rows = append(rows, row) @@ -2498,7 +2313,7 @@ func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, fil } startKey := rowToKey(start) filter := roaring.NewBitmapRowFilter(cb, filters...) - err := tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, startKey, filter) + err := qr.ApplyFilter(startKey, filter) if err != nil { return nil, err } else { @@ -2507,16 +2322,16 @@ func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, fil } // unionRows yields the union of the given rows in this fragment -func (f *fragment) unionRows(ctx context.Context, tx Tx, rows []uint64) (*Row, error) { +func (f *fragment) unionRows(ctx context.Context, qr qc.QueryRead, rows []uint64) (*Row, error) { f.mu.RLock() defer f.mu.RUnlock() - return f.unprotectedUnionRows(ctx, tx, rows) + return f.unprotectedUnionRows(ctx, qr, rows) } // unprotectedRows calls rows without grabbing the mutex. -func (f *fragment) unprotectedUnionRows(ctx context.Context, tx Tx, rows []uint64) (*Row, error) { +func (f *fragment) unprotectedUnionRows(ctx context.Context, qr qc.QueryRead, rows []uint64) (*Row, error) { filter := roaring.NewBitmapRowsUnion(rows) - err := tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, 0, filter) + err := qr.ApplyFilter(0, filter) if err != nil { return nil, err } else { @@ -2540,33 +2355,37 @@ type rowIterator interface { Next() (*Row, uint64, *int64, bool, error) } -func (f *fragment) rowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { +func (f *fragment) rowIterator(qr qc.QueryRead, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { if strings.HasPrefix(f.view(), viewBSIGroupPrefix) { - return f.intRowIterator(tx, wrap, filters...) + return f.intRowIterator(qr, wrap, filters...) } // viewStandard // TODO(kuba) - IMHO we should check if f.view() is viewStandard, // but because of testing the function returns set iterator as default one. - return f.setRowIterator(tx, wrap, filters...) + return f.setRowIterator(qr, wrap, filters...) } type timeRowIterator struct { - tx Tx + qcx qc.QueryContext cur int wrap bool allRowIDs []uint64 rowIDToFragments map[uint64][]*fragment } -func timeFragmentsRowIterator(fragments []*fragment, tx Tx, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { +func timeFragmentsRowIterator(fragments []*fragment, qcx qc.QueryContext, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { if len(fragments) == 0 { return nil, fmt.Errorf("there should be at least 1 fragment") } else if len(fragments) == 1 { - return fragments[0].setRowIterator(tx, wrap, filters...) + qr, err := fragments[0].qcxRead(qcx) + if err != nil { + return nil, err + } + return fragments[0].setRowIterator(qr, wrap, filters...) } it := &timeRowIterator{ - tx: tx, + qcx: qcx, cur: 0, wrap: wrap, } @@ -2575,7 +2394,11 @@ func timeFragmentsRowIterator(fragments []*fragment, tx Tx, wrap bool, filters . // rowID back to the fragments that have that rowID rowIDToFragments := make(map[uint64][]*fragment) for _, f := range fragments { - rowIDs, err := f.rows(context.Background(), tx, 0, filters...) + qr, err := f.qcxRead(qcx) + if err != nil { + return nil, err + } + rowIDs, err := f.rows(context.Background(), qr, 0, filters...) if err != nil { return nil, err } @@ -2625,7 +2448,11 @@ func (it *timeRowIterator) Next() (r *Row, rowID uint64, _ *int64, wrapped bool, fragments := it.rowIDToFragments[rowID] rows := make([]*Row, 0, len(fragments)) for _, fragment := range fragments { - row, err := fragment.row(it.tx, rowID) + qr, err := fragment.qcxRead(it.qcx) + if err != nil { + return nil, rowID, nil, wrapped, err + } + row, err := fragment.row(qr, rowID) if err != nil { return row, rowID, nil, wrapped, err } @@ -2647,7 +2474,7 @@ type intRowIterator struct { wrap bool } -func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { +func (f *fragment) intRowIterator(qr qc.QueryRead, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { it := intRowIterator{ f: f, colIDs: make(map[int64][]uint64), @@ -2667,7 +2494,7 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil } val := int64(1 << (rid - bsiOffsetBit)) - r, err := f.unprotectedRow(tx, rid) + r, err := f.unprotectedRow(qr, rid) if err != nil { return err } @@ -2676,18 +2503,18 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil } return nil } - if err := f.foreachRow(tx, filters, callback); err != nil { + if err := f.foreachRow(qr, filters, callback); err != nil { return nil, err } // apply exist and sign bits - r0, err := f.unprotectedRow(tx, 0) + r0, err := f.unprotectedRow(qr, 0) if err != nil { return nil, err } allCols := r0.Columns() - r1, err := f.unprotectedRow(tx, 1) + r1, err := f.unprotectedRow(qr, 1) if err != nil { return nil, err } @@ -2719,10 +2546,10 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil return &it, nil } -func (f *fragment) foreachRow(tx Tx, filters []roaring.BitmapFilter, fn func(rid uint64) error) error { +func (f *fragment) foreachRow(qr qc.QueryRead, filters []roaring.BitmapFilter, fn func(rid uint64) error) error { filter := roaring.NewBitmapRowFilter(fn, filters...) - err := tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, 0, filter) - return errors.Wrap(err, "pilosa.foreachRow: ") + err := qr.ApplyFilter(0, filter) + return errors.Wrap(err, "pilosa.foreachRow") } func (it *intRowIterator) Seek(rowID uint64) { @@ -2750,20 +2577,20 @@ func (it *intRowIterator) Next() (r *Row, rowID uint64, value *int64, wrapped bo } type setRowIterator struct { - tx Tx + qr qc.QueryRead f *fragment rowIDs []uint64 cur int wrap bool } -func (f *fragment) setRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { - rows, err := f.rows(context.Background(), tx, 0, filters...) +func (f *fragment) setRowIterator(qr qc.QueryRead, wrap bool, filters ...roaring.BitmapFilter) (rowIterator, error) { + rows, err := f.rows(context.Background(), qr, 0, filters...) if err != nil { return nil, err } return &setRowIterator{ - tx: tx, + qr: qr, f: f, rowIDs: rows, // TODO: this may be memory intensive in high cardinality cases wrap: wrap, @@ -2786,7 +2613,7 @@ func (it *setRowIterator) Next() (r *Row, rowID uint64, _ *int64, wrapped bool, wrapped = true } id := it.rowIDs[it.cur] - r, err = it.f.row(it.tx, id) + r, err = it.f.row(it.qr, id) if err != nil { return r, rowID, nil, wrapped, err } @@ -2805,7 +2632,7 @@ func pos(rowID, columnID uint64) uint64 { // vector stores the mapping of colID to rowID. // It's used for a mutex field type. type vector interface { - Get(tx Tx, colID uint64) (uint64, bool, error) + Get(qr qc.QueryRead, colID uint64) (uint64, bool, error) } // rowsVector implements the vector interface by looking @@ -2825,8 +2652,8 @@ func newRowsVector(f *fragment) *rowsVector { // Additionally, it returns true if a value was found, // otherwise it returns false. Ensure that you already // have the mutex before calling this. -func (v *rowsVector) Get(tx Tx, colID uint64) (uint64, bool, error) { - rows, err := v.f.unprotectedRows(context.Background(), tx, 0, roaring.NewBitmapColumnFilter(colID)) +func (v *rowsVector) Get(qr qc.QueryRead, colID uint64) (uint64, bool, error) { + rows, err := v.f.unprotectedRows(context.Background(), qr, 0, roaring.NewBitmapColumnFilter(colID)) if err != nil { return 0, false, err } else if len(rows) > 1 { @@ -2861,8 +2688,8 @@ func newBoolVector(f *fragment) *boolVector { // Additionally, it returns true if a value was found, // otherwise it returns false. Ensure that you already // have the fragment mutex before calling this. -func (v *boolVector) Get(tx Tx, colID uint64) (uint64, bool, error) { - rows, err := v.f.unprotectedRows(context.Background(), tx, 0, roaring.NewBitmapColumnFilter(colID)) +func (v *boolVector) Get(qr qc.QueryRead, colID uint64) (uint64, bool, error) { + rows, err := v.f.unprotectedRows(context.Background(), qr, 0, roaring.NewBitmapColumnFilter(colID)) if err != nil { return 0, false, err } else if len(rows) > 1 { @@ -2925,14 +2752,14 @@ func (r *RowKV) Compare(o RowKV, desc bool) (bool, bool) { // sortBSIData, fetches the rows and seperates the positive and negetive values. // these values and sorted seperately and appended -func (f *fragment) sortBsiData(tx Tx, filter *Row, bitDepth uint64, sort_desc bool) (*SortedRow, error) { - consider, err := f.row(tx, bsiExistsBit) +func (f *fragment) sortBsiData(qr qc.QueryRead, filter *Row, bitDepth uint64, sort_desc bool) (*SortedRow, error) { + consider, err := f.row(qr, bsiExistsBit) if err != nil { return nil, err } else if filter != nil { consider = consider.Intersect(filter) } - row, err := f.row(tx, bsiSignBit) + row, err := f.row(qr, bsiSignBit) if err != nil { return nil, err } @@ -2940,9 +2767,9 @@ func (f *fragment) sortBsiData(tx Tx, filter *Row, bitDepth uint64, sort_desc bo neg := consider.Difference(pos) var sortedRowIds []RowKV - f.flattenRowValues(tx, &sortedRowIds, neg, bitDepth, -1) + f.flattenRowValues(qr, &sortedRowIds, neg, bitDepth, -1) ok := true - f.flattenRowValues(tx, &sortedRowIds, pos, bitDepth, 1) + f.flattenRowValues(qr, &sortedRowIds, pos, bitDepth, 1) sort.SliceStable(sortedRowIds, func(i, j int) bool { if c, k := sortedRowIds[i].Compare(sortedRowIds[j], sort_desc); k { return c @@ -2961,10 +2788,10 @@ func (f *fragment) sortBsiData(tx Tx, filter *Row, bitDepth uint64, sort_desc bo }, nil } -func (f *fragment) flattenRowValues(tx Tx, sortedRowIds *[]RowKV, filter *Row, bitDepth uint64, sign int64) error { +func (f *fragment) flattenRowValues(qr qc.QueryRead, sortedRowIds *[]RowKV, filter *Row, bitDepth uint64, sign int64) error { m := make(map[uint64]int64) for i := int(bitDepth - 1); i >= 0; i-- { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) + row, err := f.row(qr, uint64(bsiOffsetBit+i)) if err != nil { return err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9c3c936e8..a6e94711b 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -19,10 +19,12 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/testhook" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck "github.com/pkg/errors" + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -33,115 +35,165 @@ var ( FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") ) +// mustQueryContext requests a brand new query context against a fragment, which +// if it is a write QueryContext, will be scoped to that specific fragment's +// index and shard. This, and mustRead/mustWrite, are workarounds for the fact +// that the whole point of QueryContext is to get us sustained access with shared +// transactions, and fragment_internal_test is all about doing multiple sequential +// operations that aren't sharing a backend transaction. Rather than muddle the +// usual interface around this special case, we write helpers for the test code. +func mustQueryContext(tb testing.TB, f *fragment, write bool) qc.QueryContext { + var qcx qc.QueryContext + var err error + if write { + qcx, err = f.holder.NewIndexQueryContext(context.Background(), f.index(), f.shard) + } else { + qcx, err = f.holder.NewQueryContext(context.Background()) + } + if err != nil { + tb.Fatalf("creating query context (write %t): %v", write, err) + } + tb.Cleanup(qcx.Release) + return qcx +} + +// mustRead yields a new QueryContext and QueryRead, or fails trying. This is +// in test code because it's nonsensical outside of tests; in non-test +// circumstances, you'd have a meaningful higher level operation to own the +// QueryContext. +func mustRead(tb testing.TB, f *fragment) (qc.QueryContext, qc.QueryRead) { + qcx := mustQueryContext(tb, f, false) + qr, err := f.qcxRead(qcx) + if err != nil { + tb.Fatalf("creating query read: %v", err) + } + return qcx, qr +} + +// mustWrite yields a new QueryContext and QueryWrite, or fails trying. This is +// in test code because it's nonsensical outside of tests; in non-test +// circumstances, you'd have a meaningful higher level operation to own the +// QueryContext. +func mustWrite(tb testing.TB, f *fragment) (qc.QueryContext, qc.QueryWrite) { + qcx := mustQueryContext(tb, f, true) + qw, err := f.qcxWrite(qcx) + if err != nil { + tb.Fatalf("creating query write: %v", err) + } + return qcx, qw +} + +// mustRow returns a row by ID. Panic on error. Only used for testing. +func (f *fragment) mustRow(tb testing.TB, qr qc.QueryRead, rowID uint64) *Row { + row, err := f.row(qr, rowID) + if err != nil { + tb.Fatalf("reading row %d: %v", rowID, err) + } + return row +} + // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f, idx, tx := mustOpenFragment(t) + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set bits on the fragment. - if _, err := f.setBit(tx, 120, 1); err != nil { + if _, err := f.setBit(qw, 120, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 120, 6); err != nil { + } else if _, err := f.setBit(qw, 120, 6); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 121, 0); err != nil { + } else if _, err := f.setBit(qw, 121, 0); err != nil { t.Fatal(err) } // should have two containers set in the fragment. // Verify counts on rows. - if n := f.mustRow(tx, 120).Count(); n != 2 { + if n := f.mustRow(t, qw, 120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { + } else if n := f.mustRow(t, qw, 121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } - // commit the change, and verify it is still there - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) // Close and reopen the fragment & verify the data. - err := f.Reopen() + err := f.Reopen(t) if err != nil { t.Fatal(err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) - if n := f.mustRow(tx, 120).Count(); n != 2 { + if n := f.mustRow(t, qr, 120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { + } else if n := f.mustRow(t, qr, 121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set and then clear bits on the fragment. - if _, err := f.setBit(tx, 1000, 1); err != nil { + if _, err := f.setBit(qw, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 1000, 2); err != nil { + } else if _, err := f.setBit(qw, 1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.clearBit(tx, 1000, 1); err != nil { + } else if _, err := f.clearBit(qw, 1000, 1); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.mustRow(tx, 1000).Count(); n != 1 { + if n := f.mustRow(t, qw, 1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // The Reopen below implies this test is looking at storage consistency. // In that spirit, we will check that the Tx Commit is visible afterwards. - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.Reopen(t); err != nil { t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 1 { + } else if n := f.mustRow(t, qr, 1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set and then clear bits on the fragment. - if _, err := f.setBit(tx, 1000, 1); err != nil { + if _, err := f.setBit(qw, 1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 1000, 65536); err != nil { + } else if _, err := f.setBit(qw, 1000, 65536); err != nil { t.Fatal(err) - } else if _, err := f.unprotectedClearRow(tx, 1000); err != nil { + } else if _, err := f.unprotectedClearRow(qw, 1000); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.mustRow(tx, 1000).Count(); n != 0 { + if n := f.mustRow(t, qw, 1000).Count(); n != 0 { t.Fatalf("unexpected count: %d", n) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.Reopen(t); err != nil { t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 0 { + } else if n := f.mustRow(t, qr, 1000).Count(); n != 0 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Obtain transction. @@ -149,86 +201,84 @@ func TestFragment_SetRow(t *testing.T) { rowID := uint64(1000) // Set bits on the fragment. - if _, err := f.setBit(tx, rowID, 1); err != nil { + if _, err := f.setBit(qw, rowID, 1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, rowID, 65536); err != nil { + } else if _, err := f.setBit(qw, rowID, 65536); err != nil { t.Fatal(err) } // Verify data on row. - if cols := f.mustRow(tx, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{1, 65536}) { + if cols := f.mustRow(t, qw, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{1, 65536}) { t.Fatalf("unexpected columns: %+v", cols) } // Verify count on row. - if n := f.mustRow(tx, rowID).Count(); n != 2 { + if n := f.mustRow(t, qw, rowID).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Set row (overwrite existing data). row := NewRow(1, 65537, 140000) - if changed, err := f.unprotectedSetRow(tx, row, rowID); err != nil { + if changed, err := f.unprotectedSetRow(qw, row, rowID); err != nil { t.Fatal(err) } else if !changed { t.Fatalf("expected changed value: %v", changed) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + qcx, qw = mustWrite(t, f) // Verify data on row. - if cols := f.mustRow(tx, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{1, 65537, 140000}) { + if cols := f.mustRow(t, qw, rowID).Columns(); !reflect.DeepEqual(cols, []uint64{1, 65537, 140000}) { t.Fatalf("unexpected columns after set row: %+v", cols) } // Verify count on row. - if n := f.mustRow(tx, rowID).Count(); n != 3 { + if n := f.mustRow(t, qw, rowID).Count(); n != 3 { t.Fatalf("unexpected count after set row: %d", n) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + require.Nil(t, qcx.Commit()) + + qcx, qr := mustRead(t, f) // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.Reopen(t); err != nil { t.Fatal(err) - } else if n := f.mustRow(tx, rowID).Count(); n != 3 { + } else if n := f.mustRow(t, qr, rowID).Count(); n != 3 { t.Fatalf("unexpected count (reopen): %d", n) } - tx.Rollback() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx.Release() + qcx, qw = mustWrite(t, f) // verify that setting something from a row which lacks a segment for // this fragment's shard still clears this fragment correctly. notOurs := NewRow(8*ShardWidth + 1024) - if changed, err := f.unprotectedSetRow(tx, notOurs, rowID); err != nil { + if changed, err := f.unprotectedSetRow(qw, notOurs, rowID); err != nil { t.Fatal(err) } else if !changed { t.Fatalf("setRow didn't report a change") } - if cols := f.mustRow(tx, rowID).Columns(); len(cols) != 0 { + if cols := f.mustRow(t, qw, rowID).Columns(); len(cols) != 0 { t.Fatalf("expected setting a row with no entries to clear the cache") } - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) } // Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set value. - if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { + if changed, err := f.setValue(qw, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -237,21 +287,20 @@ func TestFragment_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { + if changed, err := f.setValue(qw, 100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") } // same after Commit - if err := tx.Commit(); err != nil { + if err := qcx.Commit(); err != nil { t.Fatal(err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qw = mustWrite(t, f) // Read value. - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -260,7 +309,7 @@ func TestFragment_SetValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { + if changed, err := f.setValue(qw, 100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -268,26 +317,25 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set value. - if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { + if changed, err := f.setValue(qw, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.setValue(tx, 100, 16, 2028); err != nil { + if changed, err := f.setValue(qw, 100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -296,13 +344,12 @@ func TestFragment_SetValue(t *testing.T) { } // Read value after commit. - if err := tx.Commit(); err != nil { + if err := qcx.Commit(); err != nil { t.Fatal(err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qw = mustWrite(t, f) - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -313,26 +360,25 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("Clear", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set value. - if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { + if changed, err := f.setValue(qw, 100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Clear value should overwrite all bits, and set not-null to 0. - if changed, err := f.clearValue(tx, 100, 16, 2028); err != nil { + if changed, err := f.clearValue(qw, 100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -341,13 +387,12 @@ func TestFragment_SetValue(t *testing.T) { } // Same after Commit - if err := tx.Commit(); err != nil { + if err := qcx.Commit(); err != nil { t.Fatal(err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qw = mustWrite(t, f) - if value, exists, err := f.value(tx, 100, 16); err != nil { + if value, exists, err := f.value(qw, 100, 16); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -357,20 +402,18 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("NotExists", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - defer tx.Rollback() // Set value. - if changed, err := f.setValue(tx, 100, 10, 20); err != nil { + if changed, err := f.setValue(qw, 100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.value(tx, 101, 11); err != nil { + if value, exists, err := f.value(qw, 101, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -381,14 +424,13 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, _ := mustOpenFragment(t) defer f.Clean(t) - tx.Rollback() + qcx.Release() if err := quick.Check(func(bitDepth uint64, bitN uint64, values []uint64) bool { - tx = idx.holder.txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx, qw := mustWrite(t, f) + defer qcx.Release() // Limit bit depth & maximum values. bitDepth = (bitDepth % 8) + 1 bitN = (bitN % 99) + 1 @@ -404,14 +446,14 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.setValue(tx, columnID, bitDepth, int64(value)); err != nil { + if _, err := f.setValue(qw, columnID, bitDepth, int64(value)); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.value(tx, columnID, bitDepth) + v, exists, err := f.value(qw, columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -422,15 +464,15 @@ func TestFragment_SetValue(t *testing.T) { } // Same after Commit - if err := tx.Commit(); err != nil { + if err := qcx.Commit(); err != nil { t.Fatal(err) } - tx = idx.holder.txf.NewTx(Txo{Write: false, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx, qr := mustRead(t, f) + defer qcx.Release() // Ensure values are set. for columnID, value := range m { - v, exists, err := f.value(tx, columnID, bitDepth) + v, exists, err := f.value(qr, columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -451,7 +493,7 @@ func TestFragment_SetValue(t *testing.T) { func TestFragment_Sum(t *testing.T) { const bitDepth = 16 - f, idx, tx := mustOpenFragment(t) + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. @@ -466,17 +508,16 @@ func TestFragment_Sum(t *testing.T) { {4000, 300}, } for _, v := range vals { - if _, err := f.setValue(tx, v.cid, bitDepth, v.val); err != nil { + if _, err := f.setValue(qw, v.cid, bitDepth, v.val); err != nil { t.Fatal(err) } } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + qcx, qr := mustRead(t, f) t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { + if sum, n, err := f.sum(qr, nil, bitDepth); err != nil { t.Fatal(err) } else if n != 5 { t.Fatalf("unexpected count: %d", n) @@ -486,7 +527,7 @@ func TestFragment_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.sum(tx, NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(qr, NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -495,21 +536,19 @@ func TestFragment_Sum(t *testing.T) { } }) - tx.Rollback() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx.Release() + qcx, qw = mustWrite(t, f) // verify that clearValue clears values - if _, err := f.clearValue(tx, 1000, bitDepth, 23); err != nil { + if _, err := f.clearValue(qw, 1000, bitDepth, 23); err != nil { t.Fatal(err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr = mustRead(t, f) t.Run("ClearValue", func(t *testing.T) { - if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { + if sum, n, err := f.sum(qr, nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -523,31 +562,30 @@ func TestFragment_Sum(t *testing.T) { func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 - f, idx, tx := mustOpenFragment(t) + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(qw, 5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(qw, 6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(qw, 7000, bitDepth, 0); err != nil { t.Fatal(err) } - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) // the new tx is shared by Min/Max below. - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) t.Run("Min", func(t *testing.T) { tests := []struct { @@ -563,7 +601,7 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.min(tx, test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(qr, test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -592,7 +630,7 @@ func TestFragment_MinMax(t *testing.T) { columns = test.filter.Columns() } - if max, cnt, err := f.max(tx, test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(qr, test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp || cnt != test.cnt { t.Errorf("%d. max(%v, %v)=(%v, %v), expected (%v, %v)", i, columns, bitDepth, max, cnt, test.exp, test.cnt) @@ -606,23 +644,22 @@ func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.rangeOp(tx, pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -630,24 +667,23 @@ func TestFragment_Range(t *testing.T) { }) t.Run("EQOversizeRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, 1, 0); err != nil { + if _, err := f.setValue(qw, 1000, 1, 0); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, 1, 1); err != nil { + } else if _, err := f.setValue(qw, 2000, 1, 1); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.rangeOp(tx, pql.EQ, 1, 3); err != nil { + if b, err := f.rangeOp(qw, pql.EQ, 1, 3); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - if b, err := f.rangeOp(tx, pql.EQ, 1, 4); err != nil { + if b, err := f.rangeOp(qw, pql.EQ, 1, 4); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -655,23 +691,22 @@ func TestFragment_Range(t *testing.T) { }) t.Run("NEQ", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.rangeOp(tx, pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -679,48 +714,47 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(qw, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(qw, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values less than (ending with set column). - if b, err := f.rangeOp(tx, pql.LT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(qw, pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than (ending with unset column). - if b, err := f.rangeOp(tx, pql.LT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with set column). - if b, err := f.rangeOp(tx, pql.LTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(qw, pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values less than or equal to (ending with unset column). - if b, err := f.rangeOp(tx, pql.LTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -728,15 +762,14 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LTRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - if _, err := f.setValue(tx, 1, 1, 1); err != nil { + if _, err := f.setValue(qw, 1, 1, 1); err != nil { t.Fatal(err) } - if b, err := f.rangeOp(tx, pql.LT, 1, 2); err != nil { + if b, err := f.rangeOp(qw, pql.LT, 1, 2); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -744,17 +777,16 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LTMaxRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - if _, err := f.setValue(tx, 1, 2, 3); err != nil { + if _, err := f.setValue(qw, 1, 2, 3); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2, 2, 0); err != nil { + } else if _, err := f.setValue(qw, 2, 2, 0); err != nil { t.Fatal(err) } - if b, err := f.rangeLTUnsigned(tx, NewRow(1, 2), 2, 3, false); err != nil { + if b, err := f.rangeLTUnsigned(qw, NewRow(1, 2), 2, 3, false); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -762,48 +794,47 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(qw, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(qw, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset bit). - if b, err := f.rangeOp(tx, pql.GT, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set bit). - if b, err := f.rangeOp(tx, pql.GT, bitDepth, 301); err != nil { + if b, err := f.rangeOp(qw, pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset bit). - if b, err := f.rangeOp(tx, pql.GTE, bitDepth, 300); err != nil { + if b, err := f.rangeOp(qw, pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set bit). - if b, err := f.rangeOp(tx, pql.GTE, bitDepth, 301); err != nil { + if b, err := f.rangeOp(qw, pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -811,17 +842,16 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GTMinRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - if _, err := f.setValue(tx, 1, 2, 0); err != nil { + if _, err := f.setValue(qw, 1, 2, 0); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2, 2, 1); err != nil { + } else if _, err := f.setValue(qw, 2, 2, 1); err != nil { t.Fatal(err) } - if b, err := f.rangeGTUnsigned(tx, NewRow(1, 2), 2, 0, false); err != nil { + if b, err := f.rangeGTUnsigned(qw, NewRow(1, 2), 2, 0, false); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -829,17 +859,16 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GTOversizeRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - if _, err := f.setValue(tx, 1, 2, 0); err != nil { + if _, err := f.setValue(qw, 1, 2, 0); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2, 2, 1); err != nil { + } else if _, err := f.setValue(qw, 2, 2, 1); err != nil { t.Fatal(err) } - if b, err := f.rangeGTUnsigned(tx, NewRow(1, 2), 2, 4, false); err != nil { + if b, err := f.rangeGTUnsigned(qw, NewRow(1, 2), 2, 4, false); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -847,48 +876,47 @@ func TestFragment_Range(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set values. - if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { + if _, err := f.setValue(qw, 1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(qw, 2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(qw, 3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(qw, 4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(qw, 5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(qw, 6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for values greater than (ending with unset column). - if b, err := f.rangeBetween(tx, bitDepth, 300, 2817); err != nil { + if b, err := f.rangeBetween(qw, bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than (ending with set column). - if b, err := f.rangeBetween(tx, bitDepth, 301, 2817); err != nil { + if b, err := f.rangeBetween(qw, bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with unset column). - if b, err := f.rangeBetween(tx, bitDepth, 301, 2816); err != nil { + if b, err := f.rangeBetween(qw, bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for values greater than or equal to (ending with set column). - if b, err := f.rangeBetween(tx, bitDepth, 300, 2816); err != nil { + if b, err := f.rangeBetween(qw, bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -896,17 +924,16 @@ func TestFragment_Range(t *testing.T) { }) t.Run("BetweenCommonBitsRegression", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - if _, err := f.setValue(tx, 1, 64, 0xf0); err != nil { + if _, err := f.setValue(qw, 1, 64, 0xf0); err != nil { t.Fatal(err) - } else if _, err := f.setValue(tx, 2, 64, 0xf1); err != nil { + } else if _, err := f.setValue(qw, 2, 64, 0xf1); err != nil { t.Fatal(err) } - if b, err := f.rangeBetweenUnsigned(tx, NewRow(1, 2), 64, 0xf0, 0xf1); err != nil { + if b, err := f.rangeBetweenUnsigned(qw, NewRow(1, 2), 64, 0xf0, 0xf1); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1, 2}) { t.Fatalf("unepxected coulmns: %+v", b.Columns()) @@ -916,12 +943,12 @@ func TestFragment_Range(t *testing.T) { // benchmarkSetValues is a helper function to explore, very roughly, the cost // of setting values. -func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) { +func benchmarkSetValues(b *testing.B, qw qc.QueryWrite, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) { column := uint64(0) for i := 0; i < b.N; i++ { // We're not checking the error because this is a benchmark. // That does mean the result could be completely wrong... - _, _ = f.setValue(tx, column, bitDepth, int64(i)) + _, _ = f.setValue(qw, column, bitDepth, int64(i)) column = cfunc(column) } } @@ -931,17 +958,15 @@ func BenchmarkFragment_SetValue(b *testing.B) { depths := []uint64{4, 8, 16} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f, idx, tx := mustOpenFragment(b, OptFieldTypeSet("none", 0)) - _ = idx + f, _, qw := mustOpenFragment(b, OptFieldTypeSet("none", 0)) b.Run(name+"_Sparse", func(b *testing.B) { - benchmarkSetValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + benchmarkSetValues(b, qw, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) f.Clean(b) - f, idx, tx = mustOpenFragment(b, OptFieldTypeSet("none", 0)) - _ = idx + f, _, qw = mustOpenFragment(b, OptFieldTypeSet("none", 0)) b.Run(name+"_Dense", func(b *testing.B) { - benchmarkSetValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + benchmarkSetValues(b, qw, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) f.Clean(b) } @@ -984,10 +1009,10 @@ func makeBenchmarkImportValueData(b *testing.B, bitDepth uint64, cfunc func(uint // benchmarkImportValues is a helper function to explore, very roughly, the cost // of setting values using the special setter used for imports. -func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) { +func benchmarkImportValues(b *testing.B, qw qc.QueryWrite, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) { batches := makeBenchmarkImportValueData(b, bitDepth, cfunc) for _, req := range batches { - err := f.importValue(tx, req.ColumnIDs, req.Values, bitDepth, false) + err := f.importValue(qw, req.ColumnIDs, req.Values, bitDepth, false) if err != nil { b.Fatalf("error importing values: %s", err) } @@ -999,16 +1024,14 @@ func BenchmarkFragment_ImportValue(b *testing.B) { depths := []uint64{4, 8, 16, 32} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f, idx, tx := mustOpenFragment(b) - _ = idx + f, _, qw := mustOpenFragment(b) b.Run(name+"_Sparse", func(b *testing.B) { - benchmarkImportValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 19) & (ShardWidth - 1) }) + benchmarkImportValues(b, qw, bitDepth, f, func(u uint64) uint64 { return (u + 19) & (ShardWidth - 1) }) }) f.Clean(b) - f, idx, tx = mustOpenFragment(b) - _ = idx + f, _, qw = mustOpenFragment(b) b.Run(name+"_Dense", func(b *testing.B) { - benchmarkImportValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + benchmarkImportValues(b, qw, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) f.Clean(b) } @@ -1036,26 +1059,27 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { updateRows[i] = uint64(rand.Int63n(int64(numRows))) // row id updateCols[i] = uint64(rand.Int63n(ShardWidth)) // column id } - f, idx, tx := mustOpenFragment(b) - _ = idx - defer f.Clean(b) + func() { + f, qcx, qw := mustOpenFragment(b) + defer f.Clean(b) + defer qcx.Release() - err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) - if err != nil { - b.Fatalf("importing base data for benchmark: %v", err) - } - b.StartTimer() - for i := 0; i < numUpdates; i++ { - err := f.bulkImportStandard(tx, - updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], - updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], - &ImportOptions{}, - ) + err := f.importRoaring(context.Background(), qw, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { - b.Fatalf("doing small bulk import: %v", err) + b.Fatalf("importing base data for benchmark: %v", err) } - } - tx.Rollback() // don't exhaust the Tx space under b.N iterations. + b.StartTimer() + for i := 0; i < numUpdates; i++ { + err := f.bulkImportStandard(qw, + updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], + updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], + &ImportOptions{}, + ) + if err != nil { + b.Fatalf("doing small bulk import: %v", err) + } + } + }() } }) } @@ -1072,23 +1096,25 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { b.StopTimer() // build the update data set all at once - this will get applied // to a fragment in numUpdates batches - f, idx, tx := mustOpenFragment(b) - _ = idx - defer f.Clean(b) + func() { + f, qcx, qw := mustOpenFragment(b) + defer f.Clean(b) + defer qcx.Release() - err := f.importRoaringT(tx, getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) - if err != nil { - b.Fatalf("importing base data for benchmark: %v", err) - } - for i := 0; i < numUpdates; i++ { - data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) - b.StartTimer() - err := f.importRoaringT(tx, data, false) - b.StopTimer() + err := f.importRoaring(context.Background(), qw, getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) if err != nil { - b.Fatalf("doing small roaring import: %v", err) + b.Fatalf("importing base data for benchmark: %v", err) } - } + for i := 0; i < numUpdates; i++ { + data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) + b.StartTimer() + err := f.importRoaring(context.Background(), qw, data, false) + b.StopTimer() + if err != nil { + b.Fatalf("doing small roaring import: %v", err) + } + } + }() } }) } @@ -1120,25 +1146,28 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { b.Run(fmt.Sprintf("Updates%dVals%d", numUpdates, valsPerUpdate), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - f, _, tx := mustOpenFragment(b) + func() { + f, qcx, qw := mustOpenFragment(b) + defer qcx.Release() - err := f.importValue(tx, initialCols, initialVals, 21, false) - if err != nil { - b.Fatalf("initial value import: %v", err) - } - b.StartTimer() - for j := 0; j < numUpdates; j++ { - err := f.importValue(tx, - updateCols[valsPerUpdate*j:valsPerUpdate*(j+1)], - updateVals[valsPerUpdate*j:valsPerUpdate*(j+1)], - 21, - false, - ) + err := f.importValue(qw, initialCols, initialVals, 21, false) if err != nil { - b.Fatalf("importing values: %v", err) + b.Fatalf("initial value import: %v", err) } - } - tx.Rollback() // don't exhaust the Tx over the b.N iterations. + b.StartTimer() + for j := 0; j < numUpdates; j++ { + err := f.importValue(qw, + updateCols[valsPerUpdate*j:valsPerUpdate*(j+1)], + updateVals[valsPerUpdate*j:valsPerUpdate*(j+1)], + 21, + false, + ) + if err != nil { + b.Fatalf("importing values: %v", err) + } + } + }() + } }) } @@ -1147,18 +1176,17 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) // Set bits on the rows 100, 101, & 102. - f.mustSetBits(tx, 100, 1, 3, 200) - f.mustSetBits(tx, 101, 1) - f.mustSetBits(tx, 102, 1, 2) + f.mustSetBits(t, qw, 100, 1, 3, 200) + f.mustSetBits(t, qw, 101, 1) + f.mustSetBits(t, qw, 102, 1, 2) f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{N: 2}); err != nil { + if pairs, err := f.top(qw, topOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1171,22 +1199,21 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) // Create an intersecting input row. src := NewRow(1, 2, 3) // Set bits on various rows. - f.mustSetBits(tx, 100, 1, 10, 11, 12) // one intersection - f.mustSetBits(tx, 101, 1, 2, 3, 4) // three intersections - f.mustSetBits(tx, 102, 1, 2, 4, 5, 6) // two intersections - f.mustSetBits(tx, 103, 1000, 1001, 1002) // no intersection + f.mustSetBits(t, qw, 100, 1, 10, 11, 12) // one intersection + f.mustSetBits(t, qw, 101, 1, 2, 3, 4) // three intersections + f.mustSetBits(t, qw, 102, 1, 2, 4, 5, 6) // two intersections + f.mustSetBits(t, qw, 103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(qw, topOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, @@ -1203,8 +1230,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) // Create an intersecting input row. @@ -1225,14 +1251,14 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { if err != nil { t.Fatalf("writing to bytes: %v", err) } - err = f.importRoaringT(tx, b.Bytes(), false) + err = f.importRoaring(context.Background(), qw, b.Bytes(), false) if err != nil { t.Fatalf("importing data: %v", err) } f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(qw, topOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, @@ -1252,17 +1278,16 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) // Set bits on various rows. - f.mustSetBits(tx, 100, 1, 2, 3) - f.mustSetBits(tx, 101, 4, 5, 6, 7) - f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) + f.mustSetBits(t, qw, 100, 1, 2, 3) + f.mustSetBits(t, qw, 101, 4, 5, 6, 7) + f.mustSetBits(t, qw, 102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(qw, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, @@ -1274,17 +1299,16 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) // Set bits on various rows. - f.mustSetBits(tx, 100, 1, 2, 3) - f.mustSetBits(tx, 101, 4, 5, 6, 7) - f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) + f.mustSetBits(t, qw, 100, 1, 2, 3) + f.mustSetBits(t, qw, 101, 4, 5, 6, 7) + f.mustSetBits(t, qw, 102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(qw, topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) @@ -1293,49 +1317,18 @@ func TestFragment_TopN_NopCache(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { - shard := uint64(0) cacheSize := uint32(3) - // Create Index. - index := mustOpenIndex(t, IndexOptions{}) - - // Create field. - field, err := index.CreateFieldIfNotExists("f", "", OptFieldTypeSet(CacheTypeRanked, cacheSize)) - if err != nil { - t.Fatal(err) - } - - // Create view. - view, err := field.createViewIfNotExists(viewStandard) - if err != nil { - t.Fatal(err) - } - - // Create fragment. - frag, err := view.CreateFragmentIfNotExists(shard) - if err != nil { - t.Fatal(err) - } - // Close the storage so we can re-open it without encountering a flock. - frag.Close() - - f := frag - if err := f.Open(); err != nil { - PanicOn(err) - } + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, cacheSize)) defer f.Clean(t) - // Obtain transaction. - tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - // Set bits on various rows. - f.mustSetBits(tx, 100, 1, 2, 3) - f.mustSetBits(tx, 101, 4, 5, 6, 7) - f.mustSetBits(tx, 102, 8, 9, 10, 11, 12) - f.mustSetBits(tx, 103, 8, 9, 10, 11, 12, 13) - f.mustSetBits(tx, 104, 8, 9, 10, 11, 12, 13, 14) - f.mustSetBits(tx, 105, 10, 11) + f.mustSetBits(t, qw, 100, 1, 2, 3) + f.mustSetBits(t, qw, 101, 4, 5, 6, 7) + f.mustSetBits(t, qw, 102, 8, 9, 10, 11, 12) + f.mustSetBits(t, qw, 103, 8, 9, 10, 11, 12, 13) + f.mustSetBits(t, qw, 104, 8, 9, 10, 11, 12, 13, 14) + f.mustSetBits(t, qw, 105, 10, 11) f.RecalculateCache() @@ -1346,7 +1339,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Retrieve top rows. - if pairs, err := f.top(tx, topOptions{N: 5}); err != nil { + if pairs, err := f.top(qw, topOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) @@ -1359,13 +1352,12 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeLRU, 0)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeLRU, 0)) defer f.Clean(t) // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.setBit(tx, i, 0); err != nil { + if _, err := f.setBit(qw, i, 0); err != nil { t.Fatal(err) } } @@ -1377,10 +1369,10 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { t.Fatalf("unexpected cache len: %d", cache.Len()) } - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) // Reopen the fragment. - if err := f.Reopen(); err != nil { + if err := f.Reopen(t); err != nil { t.Fatal(err) } @@ -1392,114 +1384,47 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } } -// Ensure a fragment can be copied to another fragment. -func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0, _, tx := mustOpenFragment(t) - defer f0.Clean(t) - - // Set and then clear bits on the fragment. - if _, err := f0.setBit(tx, 1000, 1); err != nil { - t.Fatal(err) - } else if _, err := f0.setBit(tx, 1000, 2); err != nil { - t.Fatal(err) - } else if _, err := f0.clearBit(tx, 1000, 1); err != nil { - t.Fatal(err) - } - err := tx.Commit() - if err != nil { - t.Fatalf("committing write: %v", err) - } - - // Verify cache is populated. - if n := f0.cache.Len(); n != 1 { - t.Fatalf("unexpected cache size: %d", n) - } - - // Write fragment to a buffer. - var buf bytes.Buffer - wn, err := f0.WriteTo(&buf) - if err != nil { - t.Fatal(err) - } - - // Read into another fragment. - f1, idx, tx := mustOpenFragment(t) - tx.Rollback() - - defer f1.Clean(t) - - if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive - t.Fatal(err) - } else if wn != rn { - t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn) - } - // make a read-only Tx after ReadFrom has committed. - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard}) - defer tx.Rollback() - - // Verify cache is in other fragment. - if n := f1.cache.Len(); n != 1 { - t.Fatalf("unexpected cache size: %d", n) - } - - // Verify data in other fragment. - if a := f1.mustRow(tx, 1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected columns: %+v", a) - } - - // Close and reopen the fragment & verify the data. - if err := f1.Reopen(); err != nil { - t.Fatal(err) - } else if n := f1.cache.Len(); n != 1 { - t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.mustRow(tx, 1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected columns (reopen): %+v", a) - } -} - func BenchmarkFragment_IntersectionCount(b *testing.B) { - f, idx, tx := mustOpenFragment(b) + f, qcx, qw := mustOpenFragment(b) defer f.Clean(b) // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.setBit(tx, 1, uint64(i)); err != nil { + if _, err := f.setBit(qw, 1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.setBit(tx, 2, uint64(i)); err != nil { + if _, err := f.setBit(qw, 2, uint64(i)); err != nil { b.Fatal(err) } } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(b, qcx.Commit()) + _, qr := mustRead(b, f) // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.mustRow(tx, 1).intersectionCount(f.mustRow(tx, 2)); n == 0 { + if n := f.mustRow(b, qw, 1).intersectionCount(f.mustRow(b, qr, 2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } } func TestFragment_Tanimoto(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.mustSetBits(tx, 100, 1, 3, 2, 200) - f.mustSetBits(tx, 101, 1, 3) - f.mustSetBits(tx, 102, 1, 2, 10, 12) + f.mustSetBits(t, qw, 100, 1, 3, 2, 200) + f.mustSetBits(t, qw, 101, 1, 3) + f.mustSetBits(t, qw, 102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.top(tx, topOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(qw, topOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1511,19 +1436,18 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.mustSetBits(tx, 100, 1, 3, 2, 200) - f.mustSetBits(tx, 101, 1, 3) - f.mustSetBits(tx, 102, 1, 2, 10, 12) + f.mustSetBits(t, qw, 100, 1, 3, 2, 200) + f.mustSetBits(t, qw, 101, 1, 3) + f.mustSetBits(t, qw, 102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.top(tx, topOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(qw, topOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -1538,31 +1462,31 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { - f, _, tx := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) + f, _, qw := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) defer f.Clean(t) var cols []uint64 // Set a value on column 100. - if _, err := f.setBit(tx, 1, 100); err != nil { + if _, err := f.setBit(qw, 1, 100); err != nil { t.Fatal(err) } // Verify the value was set. - cols = f.mustRow(tx, 1).Columns() + cols = f.mustRow(t, qw, 1).Columns() if !reflect.DeepEqual(cols, []uint64{100}) { t.Fatalf("mutex unexpected columns: %v", cols) } // Set a different value on column 100. - if _, err := f.setBit(tx, 2, 100); err != nil { + if _, err := f.setBit(qw, 2, 100); err != nil { t.Fatal(err) } // Verify that value (row 1) was replaced (by row 2). - cols = f.mustRow(tx, 1).Columns() + cols = f.mustRow(t, qw, 1).Columns() if !reflect.DeepEqual(cols, []uint64{}) { t.Fatalf("mutex unexpected columns: %v", cols) } - cols = f.mustRow(tx, 2).Columns() + cols = f.mustRow(t, qw, 2).Columns() if !reflect.DeepEqual(cols, []uint64{100}) { t.Fatalf("mutex unexpected columns: %v", cols) } @@ -1652,33 +1576,32 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -1770,45 +1693,40 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } - tx.Rollback() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx, qw = mustWrite(t, f) // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr = mustRead(t, f) // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -1819,26 +1737,21 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { - shard := uint64(0) - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, _ := mustOpenFragment(t) + // we want to make new ones, so there can't be an existing write + qcx.Release() defer f.Clean(t) - // note: write Tx must be used on the same goroutine that created them. - // So we close out the "default" Tx created by mustOpenFragment, and - // have the goroutines below each make their own. One should get the - // write lock first, and thus they should get serialized. - tx.Rollback() eg := errgroup.Group{} eg.Go(func() error { - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) - defer func() { PanicOn(tx.Commit()) }() - return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) + qcx, qw := mustWrite(t, f) + defer func() { require.Nil(t, qcx.Commit()) }() + return f.bulkImportStandard(qw, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) eg.Go(func() error { - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) - defer func() { PanicOn(tx.Commit()) }() - return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) + qcx, qw := mustWrite(t, f) + defer func() { require.Nil(t, qcx.Commit()) }() + return f.bulkImportStandard(qw, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) err := eg.Wait() if err != nil { @@ -1931,32 +1844,32 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, _, tx := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) + f, _, qw := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d, expected: %v, but got: %v", k, v, cols) } } // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d expected: %v, but got: %v", k, v, cols) } @@ -2050,44 +1963,42 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) + f, qcx, qw := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + qcx, qr := mustRead(t, f) // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d, expected: %v, but got: %v", k, v, cols) } } + qcx.Release() - tx.Rollback() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx, qw = mustWrite(t, f) // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk clearing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + + _, qr = mustRead(t, f) // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("row: %d expected: %v, but got: %v", k, v, cols) } @@ -2181,32 +2092,32 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, _, tx := mustOpenFragment(t, OptFieldTypeBool()) + f, _, qw := mustOpenFragment(t, OptFieldTypeBool()) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qw, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -2300,44 +2211,41 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeBool()) + f, qcx, qw := mustOpenFragment(t, OptFieldTypeBool()) defer f.Clean(t) // Set import. - err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + err := f.bulkImport(qw, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + qcx, qr := mustRead(t, f) // Check for expected results. for k, v := range test.setExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } } - tx.Rollback() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx.Release() + qcx, qw = mustWrite(t, f) // Clear import. - err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + err = f.bulkImport(qw, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) if err != nil { t.Fatalf("bulk importing ids: %v", err) } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr = mustRead(t, f) // Check for expected results. for k, v := range test.clearExp { - cols := f.mustRow(tx, k).Columns() + cols := f.mustRow(t, qr, k).Columns() if !reflect.DeepEqual(cols, v) { t.Fatalf("expected: %v, but got: %v", v, cols) } @@ -2372,15 +2280,16 @@ func BenchmarkFragment_Import(b *testing.B) { // since bulkImport modifies the input slices, we make new copies for each round copy(rowsUse, rows) copy(colsUse, cols) - f, idx, tx := mustOpenFragment(b) - _ = idx - b.StartTimer() - if err := f.bulkImport(tx, rowsUse, colsUse, options); err != nil { - b.Errorf("Error Building Sample: %s", err) - } - b.StopTimer() - tx.Rollback() - f.Clean(b) + func() { + f, qcx, qw := mustOpenFragment(b) + defer f.Clean(b) + defer qcx.Release() + b.StartTimer() + if err := f.bulkImport(qw, rowsUse, colsUse, options); err != nil { + b.Errorf("Error Building Sample: %s", err) + } + b.StopTimer() + }() } } @@ -2399,10 +2308,10 @@ func BenchmarkImportRoaring(b *testing.B) { b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, _, tx := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) + f, _, qw := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) b.StartTimer() - err := f.importRoaringT(tx, data, false) + err := f.importRoaring(context.Background(), qw, data, false) if err != nil { f.Clean(b) b.Fatalf("import error: %v", err) @@ -2432,19 +2341,20 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { b.Run(fmt.Sprintf("Rows%dConcurrency%dCache_%s", numRows, concurrency, cacheType), func(b *testing.B) { b.StopTimer() frags := make([]*fragment, concurrency) - txs := make([]Tx, concurrency) + qcxs := make([]qc.QueryContext, concurrency) + qws := make([]qc.QueryWrite, concurrency) for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { - frags[j], _, txs[j] = mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) + frags[j], qcxs[j], qws[j] = mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) } eg := errgroup.Group{} b.StartTimer() for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - defer txs[j].Rollback() + defer qcxs[j].Release() - err := frags[j].importRoaringT(txs[j], data[j], false) + err := frags[j].importRoaring(context.Background(), qws[j], data[j], false) return err }) } @@ -2473,16 +2383,17 @@ func BenchmarkImportStandard(b *testing.B) { for i := 0; i < b.N; i++ { copy(rowIDs, rowIDsOrig) copy(columnIDs, columnIDsOrig) - f, idx, tx := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) - _ = idx - b.StartTimer() - err := f.bulkImport(tx, rowIDs, columnIDs, &ImportOptions{}) - if err != nil { - b.Errorf("import error: %v", err) - } - b.StopTimer() - tx.Rollback() - f.Clean(b) + func() { + f, qcx, qw := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) + defer f.Clean(b) + defer qcx.Release() + b.StartTimer() + err := f.bulkImport(qw, rowIDs, columnIDs, &ImportOptions{}) + if err != nil { + b.Errorf("import error: %v", err) + } + b.StopTimer() + }() } }) } @@ -2502,17 +2413,16 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.Run(name, func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, idx, tx := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) - _ = idx + f, _, qw := mustOpenFragment(b, OptFieldTypeSet(cacheType, 0)) itr, err := roaring.NewRoaringIterator(data) PanicOn(err) - _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0) + _, _, err = qw.ImportRoaringBits(itr, false, 0) if err != nil { b.Errorf("import error: %v", err) } b.StartTimer() - err = f.importRoaringT(tx, updata, false) + err = f.importRoaring(context.Background(), qw, updata, false) if err != nil { f.Clean(b) b.Errorf("import error: %v", err) @@ -2555,19 +2465,20 @@ func BenchmarkUpdatePathological(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() - f, idx, tx := mustOpenFragment(b, OptFieldTypeSet(DefaultCacheType, 0)) - _ = idx - - err := f.importRoaringT(tx, exists, false) - if err != nil { - b.Fatalf("importing roaring: %v", err) - } - b.StartTimer() - err = f.importRoaringT(tx, inc, false) - if err != nil { - b.Fatalf("importing second: %v", err) - } + func() { + f, qcx, qw := mustOpenFragment(b, OptFieldTypeSet(DefaultCacheType, 0)) + defer qcx.Release() + err := f.importRoaring(context.Background(), qw, exists, false) + if err != nil { + b.Fatalf("importing roaring: %v", err) + } + b.StartTimer() + err = f.importRoaring(context.Background(), qw, inc, false) + if err != nil { + b.Fatalf("importing second: %v", err) + } + }() } } @@ -2576,11 +2487,11 @@ var bigFrag string func initBigFrag(tb testing.TB) { if bigFrag == "" { - f, _, tx := mustOpenFragment(tb, OptFieldTypeSet(DefaultCacheType, 0)) + f, qcx, qw := mustOpenFragment(tb, OptFieldTypeSet(DefaultCacheType, 0)) for i := int64(0); i < 10; i++ { // 10 million rows, 1 bit per column, random seeded by i data := getZipfRowsSliceRoaring(10000000, i, 0, ShardWidth) - err := f.importRoaringT(tx, data, false) + err := f.importRoaring(context.Background(), qw, data, false) if err != nil { PanicOn(fmt.Sprintf("setting up fragment data: %v", err)) } @@ -2590,7 +2501,7 @@ func initBigFrag(tb testing.TB) { PanicOn(fmt.Sprintf("closing fragment: %v", err)) } bigFrag = f.path() - PanicOn(tx.Commit()) + require.Nil(tb, qcx.Commit()) } } @@ -2616,27 +2527,20 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { origF.Close() fi.Close() - h, idx, _, _, f := newTestFragment(b) - - err = f.Open() - if err != nil { - b.Fatalf("opening fragment: %v", err) - } - - // Obtain transaction. - tx := h.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - copy(rows, rowsOrig) - copy(cols, colsOrig) - b.StartTimer() - err = f.bulkImport(tx, rows, cols, opts) - b.StopTimer() - if err != nil { - b.Fatalf("bulkImport: %v", err) - } - PanicOn(tx.Commit()) - f.Clean(b) + func() { + f, qcx, qw := mustOpenFragment(b) + defer qcx.Release() + copy(rows, rowsOrig) + copy(cols, colsOrig) + b.StartTimer() + err = f.bulkImport(qw, rows, cols, opts) + b.StopTimer() + if err != nil { + b.Fatalf("bulkImport: %v", err) + } + require.Nil(b, qcx.Commit()) + f.Clean(b) + }() } } @@ -2659,73 +2563,64 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - - h, idx, _, _, f := newTestFragment(b) - - defer f.Clean(b) - - tx := h.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - err = f.Open() - if err != nil { - b.Fatalf("opening fragment: %v", err) - } - b.StartTimer() - err = f.importRoaringT(tx, updata, false) - b.StopTimer() - if err != nil { - b.Fatalf("bulkImport: %v", err) - } + func() { + f, qcx, qw := mustOpenFragment(b) + defer f.Clean(b) + defer qcx.Release() + b.StartTimer() + err = f.importRoaring(context.Background(), qw, updata, false) + b.StopTimer() + if err != nil { + b.Fatalf("bulkImport: %v", err) + } + }() } } func TestGetZipfRowsSliceRoaring(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(DefaultCacheType, 0)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(DefaultCacheType, 0)) data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) - err := f.importRoaringT(tx, data, false) + err := f.importRoaring(context.Background(), qw, data, false) if err != nil { t.Fatalf("importing roaring: %v", err) } - rows, err := f.rows(context.Background(), tx, 0) + rows, err := f.rows(context.Background(), qw, 0) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { t.Fatalf("unexpected rows: %v", rows) } for i := uint64(1); i < 10; i++ { - if f.mustRow(tx, i).Count() >= f.mustRow(tx, i-1).Count() { + if f.mustRow(t, qw, i).Count() >= f.mustRow(t, qw, i-1).Count() { t.Fatalf("suspect distribution from getZipfRowsSliceRoaring") } } f.Clean(t) } -func prepareSampleRowData(b *testing.B, bits int, rows uint64, width uint64) (*fragment, *Index, Tx) { - f, idx, tx := mustOpenFragment(b, OptFieldTypeSet("none", 0)) +func prepareSampleRowData(b *testing.B, bits int, rows uint64, width uint64) (*fragment, qc.QueryContext, qc.QueryWrite) { + f, qcx, qw := mustOpenFragment(b, OptFieldTypeSet("none", 0)) for i := 0; i < bits; i++ { data := getUniformRowsSliceRoaring(rows, int64(rows)+int64(i), 0, width) - err := f.importRoaringT(tx, data, false) + err := f.importRoaring(context.Background(), qw, data, false) if err != nil { b.Fatalf("creating sample data: %v", err) } } - return f, idx, tx + return f, qcx, qw } type txFrag struct { rows, width uint64 - tx Tx - idx *Index + qr qc.QueryRead // may actually secretly be a querywrite frag *fragment } func benchmarkRowsOnTestcase(b *testing.B, ctx context.Context, txf txFrag) { col := uint64(0) for i := 0; i < b.N; i++ { - _, err := txf.frag.rows(ctx, txf.tx, 0, roaring.NewBitmapColumnFilter(col)) + _, err := txf.frag.rows(ctx, txf.qr, 0, roaring.NewBitmapColumnFilter(col)) if err != nil { b.Fatalf("retrieving rows for col %d: %v", col, err) } @@ -2745,21 +2640,21 @@ func benchmarkRowsMaybeWritable(b *testing.B, writable bool) { }() bg := context.Background() for _, rows := range depths { - frag, idx, tx := prepareSampleRowData(b, 3, rows, ShardWidth) + var qr qc.QueryRead + f, qcx, qw := prepareSampleRowData(b, 3, rows, ShardWidth) + qr = qw if !writable { - err := tx.Commit() + err := qcx.Commit() if err != nil { b.Fatalf("error committing sample data: %v", err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: frag, Shard: 0}) - defer tx.Rollback() + _, qr = mustRead(b, f) } testCases = append(testCases, txFrag{ rows: rows, width: ShardWidth, - frag: frag, - idx: idx, - tx: tx, + frag: f, + qr: qr, }) } for _, testCase := range testCases { @@ -2902,16 +2797,13 @@ func (f *fragment) Clean(t testing.TB) { } } -// importRoaringT calls importRoaring with context.Background() for convenience -func (f *fragment) importRoaringT(tx Tx, data []byte, clear bool) error { - - return f.importRoaring(context.Background(), tx, data, clear) -} - func newTestHolder(tb testing.TB) *Holder { path := tb.TempDir() - h := NewHolder(path, TestHolderConfig()) - err := h.Open() + h, err := NewHolder(path, TestHolderConfig()) + if err != nil { + tb.Fatalf("creating test holder: %v", err) + } + err = h.Open() if err != nil { tb.Fatalf("opening test holder: %v", err) } @@ -2958,48 +2850,52 @@ func newTestFragment(tb testing.TB, fieldOpts ...FieldOption) (*Holder, *Index, } // mustOpenFragment returns a new instance of Fragment with a temporary path. -func mustOpenFragment(tb testing.TB, fieldOpts ...FieldOption) (*fragment, *Index, Tx) { - th, idx, fld, v, f := newTestFragment(tb, fieldOpts...) +// It returns an initial QueryContext and QueryWrite attached to that fragment, +// because it turns out we nearly always want to write to a fragment we're +// creating for testing. +func mustOpenFragment(tb testing.TB, fieldOpts ...FieldOption) (*fragment, qc.QueryContext, qc.QueryWrite) { + _, idx, fld, v, f := newTestFragment(tb, fieldOpts...) fragDir := filepath.Join(idx.path, fld.name, "views", v.name, "fragments") err := os.MkdirAll(fragDir, 0700) if err != nil { tb.Fatalf("creating fragment directory: %v", err) } - tx := th.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: 0}) - testhook.Cleanup(tb, func() { - tx.Rollback() - - if err := th.txf.CloseIndex(idx); err != nil { - tb.Fatalf("closing index after test: %v", err) - } - }) - f.CacheType = fld.options.CacheType + // Note this horrible crime: We create our QueryContext for this fragment + // *before* we open the fragment, because we need a QueryContext for the + // open. Even though in fact nothing ever happens -- we just made a new fragment + // so there can't be an existing cache, we hope. + qcx, qw := mustWrite(tb, f) if err := f.Open(); err != nil { tb.Fatalf("opening fragment: %v", err) } - return f, idx, tx + return f, qcx, qw } // Reopen closes the fragment and reopens it as a new instance. -func (f *fragment) Reopen() error { +// +// This... almost certainly doesn't really mean anything anymore. +// In the days of Roaring, this meant flushing all our data structures and +// re-reading from disk. Now, with RBF, we're not closing the underlying +// database, or anything... +func (f *fragment) Reopen(tb testing.TB) error { if err := f.Close(); err != nil { return err } if err := f.Open(); err != nil { - return err + tb.Fatalf("opening fragment: %v", err) } return nil } // mustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *fragment) mustSetBits(tx Tx, rowID uint64, columnIDs ...uint64) { +func (f *fragment) mustSetBits(tb testing.TB, qw qc.QueryWrite, rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { - if _, err := f.setBit(tx, rowID, columnID); err != nil { - PanicOn(err) + if _, err := f.setBit(qw, rowID, columnID); err != nil { + require.Nil(tb, err) } } } @@ -3015,14 +2911,13 @@ func addToBitmap(bm *roaring.Bitmap, rowID uint64, columnIDs ...uint64) { // Test Various methods of retrieving RowIDs func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) for i := uint64(100); i < uint64(200); i++ { - if _, err := f.setBit(tx, i, i%2); err != nil { + if _, err := f.setBit(qw, i, i%2); err != nil { t.Fatal(err) } expectedAll = append(expectedAll, i) @@ -3031,14 +2926,14 @@ func TestFragment_RowsIteration(t *testing.T) { } } - ids, err := f.rows(context.Background(), tx, 0) + ids, err := f.rows(context.Background(), qw, 0) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expectedAll, ids) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids, err = f.rows(context.Background(), tx, 0, roaring.NewBitmapColumnFilter(1)) + ids, err = f.rows(context.Background(), qw, 0, roaring.NewBitmapColumnFilter(1)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expectedOdd, ids) { @@ -3047,32 +2942,30 @@ func TestFragment_RowsIteration(t *testing.T) { }) t.Run("secondRow", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, qcx, qw := mustOpenFragment(t) defer f.Clean(t) expected := []uint64{1, 2} - if _, err := f.setBit(tx, 1, 66000); err != nil { + if _, err := f.setBit(qw, 1, 66000); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 2, 66000); err != nil { + } else if _, err := f.setBit(qw, 2, 66000); err != nil { t.Fatal(err) - } else if _, err := f.setBit(tx, 2, 166000); err != nil { + } else if _, err := f.setBit(qw, 2, 166000); err != nil { t.Fatal(err) } - PanicOn(tx.Commit()) + require.Nil(t, qcx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) - ids, err := f.rows(context.Background(), tx, 0) + ids, err := f.rows(context.Background(), qr, 0) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } - ids, err = f.rows(context.Background(), tx, 0, roaring.NewBitmapColumnFilter(66000)) + ids, err = f.rows(context.Background(), qr, 0, roaring.NewBitmapColumnFilter(66000)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expected, ids) { @@ -3081,8 +2974,7 @@ func TestFragment_RowsIteration(t *testing.T) { }) t.Run("combinations", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) @@ -3090,17 +2982,17 @@ func TestFragment_RowsIteration(t *testing.T) { for r := uint64(1); r < uint64(10000); r += 250 { expectedRows = append(expectedRows, r) for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) { - if _, err := f.setBit(tx, r, c); err != nil { + if _, err := f.setBit(qw, r, c); err != nil { t.Fatal(err) } - ids, err := f.rows(context.Background(), tx, 0) + ids, err := f.rows(context.Background(), qw, 0) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids, err = f.rows(context.Background(), tx, 0, roaring.NewBitmapColumnFilter(c)) + ids, err = f.rows(context.Background(), qw, 0, roaring.NewBitmapColumnFilter(c)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expectedRows, ids) { @@ -3134,8 +3026,7 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) defer f.Clean(t) for num, input := range test { @@ -3145,14 +3036,14 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - err = f.importRoaringT(tx, buf.Bytes(), false) + err = f.importRoaring(context.Background(), qw, buf.Bytes(), false) if err != nil { t.Fatalf("importing roaring: %v", err) } exp := calcExpected(test[:num+1]...) for row, expCols := range exp { - cols := f.mustRow(tx, uint64(row)).Columns() + cols := f.mustRow(t, qw, uint64(row)).Columns() t.Logf("\nrow: %d\n exp:%v\n got:%v", row, expCols, cols) if !reflect.DeepEqual(cols, expCols) { t.Fatalf("input%d, row %d\n exp:%v\n got:%v", num, row, expCols, cols) @@ -3183,18 +3074,17 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) options := &ImportOptions{} - err := f.bulkImport(tx, test.rowIDs, test.colIDs, options) + err := f.bulkImport(qw, test.rowIDs, test.colIDs, options) if err != nil { t.Fatalf("bulk importing ids: %v", err) } expPairs := calcTop(test.rowIDs, test.colIDs) - pairs, err := f.top(tx, topOptions{}) + pairs, err := f.top(qw, topOptions{}) if err != nil { t.Fatalf("executing top after bulk import: %v", err) } @@ -3202,14 +3092,14 @@ func TestFragment_RoaringImportTopN(t *testing.T) { t.Fatalf("post bulk import:\n exp: %v\n got: %v\n", expPairs, pairs) } - err = f.bulkImport(tx, test.rowIDs2, test.colIDs2, options) + err = f.bulkImport(qw, test.rowIDs2, test.colIDs2, options) if err != nil { t.Fatalf("bulk importing ids: %v", err) } test.rowIDs = append(test.rowIDs, test.rowIDs2...) test.colIDs = append(test.colIDs, test.colIDs2...) expPairs = calcTop(test.rowIDs, test.colIDs) - pairs, err = f.top(tx, topOptions{}) + pairs, err = f.top(qw, topOptions{}) if err != nil { t.Fatalf("executing top after bulk import: %v", err) } @@ -3223,13 +3113,13 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - err = f.importRoaringT(tx, buf.Bytes(), false) + err = f.importRoaring(context.Background(), qw, buf.Bytes(), false) if err != nil { t.Fatalf("importing roaring: %v", err) } rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) - pairs, err = f.top(tx, topOptions{}) + pairs, err = f.top(qw, topOptions{}) if err != nil { t.Fatalf("executing top after roaring import: %v", err) } @@ -3322,17 +3212,16 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 0, 0) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 2, 0) - f.mustSetBits(tx, 3, 0) + f.mustSetBits(t, qw, 0, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 2, 0) + f.mustSetBits(t, qw, 3, 0) - iter, err := f.rowIterator(tx, false) + iter, err := f.rowIterator(qw, false) if err != nil { t.Fatal(err) } @@ -3367,16 +3256,15 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("skipped rows", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 3, 0) - f.mustSetBits(tx, 5, 0) - f.mustSetBits(tx, 7, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 3, 0) + f.mustSetBits(t, qw, 5, 0) + f.mustSetBits(t, qw, 7, 0) - iter, err := f.rowIterator(tx, false) + iter, err := f.rowIterator(qw, false) if err != nil { t.Fatal(err) } @@ -3411,16 +3299,15 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("basic wrapped", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 0, 0) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 2, 0) - f.mustSetBits(tx, 3, 0) + f.mustSetBits(t, qw, 0, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 2, 0) + f.mustSetBits(t, qw, 3, 0) - iter, err := f.rowIterator(tx, true) + iter, err := f.rowIterator(qw, true) if err != nil { t.Fatal(err) } @@ -3444,15 +3331,15 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("skipped rows wrapped", func(t *testing.T) { - f, _, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 3, 0) - f.mustSetBits(tx, 5, 0) - f.mustSetBits(tx, 7, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 3, 0) + f.mustSetBits(t, qw, 5, 0) + f.mustSetBits(t, qw, 7, 0) - iter, err := f.rowIterator(tx, true) + iter, err := f.rowIterator(qw, true) if err != nil { t.Fatal(err) } @@ -3479,21 +3366,19 @@ func TestFragmentRowIterator(t *testing.T) { // same, with commits func TestFragmentRowIterator_WithTxCommit(t *testing.T) { t.Run("basic", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 0, 0) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 2, 0) - f.mustSetBits(tx, 3, 0) + f.mustSetBits(t, qw, 0, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 2, 0) + f.mustSetBits(t, qw, 3, 0) - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) - iter, err := f.rowIterator(tx, false) + iter, err := f.rowIterator(qr, false) if err != nil { t.Fatal(err) } @@ -3528,20 +3413,18 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("skipped rows", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 3, 0) - f.mustSetBits(tx, 5, 0) - f.mustSetBits(tx, 7, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 3, 0) + f.mustSetBits(t, qw, 5, 0) + f.mustSetBits(t, qw, 7, 0) - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) - iter, err := f.rowIterator(tx, false) + iter, err := f.rowIterator(qr, false) if err != nil { t.Fatal(err) } @@ -3576,20 +3459,18 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("basic wrapped", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 0, 0) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 2, 0) - f.mustSetBits(tx, 3, 0) + f.mustSetBits(t, qw, 0, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 2, 0) + f.mustSetBits(t, qw, 3, 0) - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) - iter, err := f.rowIterator(tx, true) + iter, err := f.rowIterator(qr, true) if err != nil { t.Fatal(err) } @@ -3613,20 +3494,18 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("skipped rows wrapped", func(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - f.mustSetBits(tx, 1, 0) - f.mustSetBits(tx, 3, 0) - f.mustSetBits(tx, 5, 0) - f.mustSetBits(tx, 7, 0) + f.mustSetBits(t, qw, 1, 0) + f.mustSetBits(t, qw, 3, 0) + f.mustSetBits(t, qw, 5, 0) + f.mustSetBits(t, qw, 7, 0) - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + require.Nil(t, qcx.Commit()) + _, qr := mustRead(t, f) - iter, err := f.rowIterator(tx, true) + iter, err := f.rowIterator(qr, true) if err != nil { t.Fatal(err) } @@ -3653,6 +3532,8 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { } func TestFragmentPositionsForValue(t *testing.T) { + // We don't need the querycontext or querywrite because positionsForValue + // doesn't actually talk to the fragment in any way, it's just math. f, _, _ := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) @@ -3735,16 +3616,15 @@ func TestFragmentPositionsForValue(t *testing.T) { } func TestIntLTRegression(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) - _ = idx + f, _, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) - _, err := f.setValue(tx, 1, 6, 33) + _, err := f.setValue(qw, 1, 6, 33) if err != nil { t.Fatalf("setting value: %v", err) } - row, err := f.rangeOp(tx, pql.LT, 6, 33) + row, err := f.rangeOp(qw, pql.LT, 6, 33) if err != nil { t.Fatalf("doing range of: %v", err) } @@ -3765,8 +3645,7 @@ func sliceEq(x, y []uint64) bool { } func TestFragmentBSIUnsigned(t *testing.T) { - shard := uint64(0) - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) // Number of bits to test. @@ -3774,14 +3653,14 @@ func TestFragmentBSIUnsigned(t *testing.T) { // Load all numbers into an effectively diagonal matrix. for i := 0; i < 1<", func(t *testing.T) { - - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), false) + row, err := f.rangeGT(qr, k, int64(i), false) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -3866,11 +3739,10 @@ func TestFragmentBSIUnsigned(t *testing.T) { } }) t.Run(">=", func(t *testing.T) { - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), true) + row, err := f.rangeGT(qr, k, int64(i), true) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -3889,13 +3761,11 @@ func TestFragmentBSIUnsigned(t *testing.T) { } }) t.Run("Range", func(t *testing.T) { - - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) for i := minCheck; i < maxCheck; i++ { for j := i; j < maxCheck; j++ { - row, err := f.rangeBetween(tx, k, int64(i), int64(j)) + row, err := f.rangeBetween(qr, k, int64(i), int64(j)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -3925,12 +3795,10 @@ func TestFragmentBSIUnsigned(t *testing.T) { } }) t.Run("==", func(t *testing.T) { - - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) - defer tx.Rollback() + _, qr := mustRead(t, f) for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeEQ(tx, k, int64(i)) + row, err := f.rangeEQ(qr, k, int64(i)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -3948,8 +3816,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { // same, WithTxCommit version func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) // Number of bits to test. @@ -3957,7 +3824,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { // Load all numbers into an effectively diagonal matrix. for i := 0; i < 1<", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), false) + row, err := f.rangeGT(qr, k, int64(i), false) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4041,7 +3907,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { }) t.Run(">=", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), true) + row, err := f.rangeGT(qr, k, int64(i), true) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4062,7 +3928,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { t.Run("Range", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { for j := i; j < maxCheck; j++ { - row, err := f.rangeBetween(tx, k, int64(i), int64(j)) + row, err := f.rangeBetween(qr, k, int64(i), int64(j)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4093,7 +3959,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { }) t.Run("==", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeEQ(tx, k, int64(i)) + row, err := f.rangeEQ(qr, k, int64(i)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4110,8 +3976,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { } func TestFragmentBSISigned(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) - _ = idx + f, qcx, qw := mustOpenFragment(t, OptFieldTypeSet(CacheTypeNone, 0)) defer f.Clean(t) // Number of bits to test. @@ -4120,7 +3985,7 @@ func TestFragmentBSISigned(t *testing.T) { // Load all numbers into an effectively diagonal matrix. minVal, maxVal := 1-(1<", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), false) + row, err := f.rangeGT(qr, k, int64(i), false) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4204,7 +4068,7 @@ func TestFragmentBSISigned(t *testing.T) { }) t.Run(">=", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeGT(tx, k, int64(i), true) + row, err := f.rangeGT(qr, k, int64(i), true) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4225,7 +4089,7 @@ func TestFragmentBSISigned(t *testing.T) { t.Run("Range", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { for j := i; j < maxCheck; j++ { - row, err := f.rangeBetween(tx, k, int64(i), int64(j)) + row, err := f.rangeBetween(qr, k, int64(i), int64(j)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4256,7 +4120,7 @@ func TestFragmentBSISigned(t *testing.T) { }) t.Run("==", func(t *testing.T) { for i := minCheck; i < maxCheck; i++ { - row, err := f.rangeEQ(tx, k, int64(i)) + row, err := f.rangeEQ(qr, k, int64(i)) if err != nil { t.Fatalf("failed to query fragment: %v", err) } @@ -4273,19 +4137,19 @@ func TestFragmentBSISigned(t *testing.T) { } func TestImportValueConcurrent(t *testing.T) { - f, idx, tx := mustOpenFragment(t) + f, qcx, _ := mustOpenFragment(t) defer f.Clean(t) - // we will be making a new Tx each time, so we can rollback the default provided one. - tx.Rollback() + // we will be making a new QueryContext each time, so we can rollback the default provided one. + qcx.Release() eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i eg.Go(func() error { - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + qcx, qw := mustWrite(t, f) + defer qcx.Release() for j := uint64(0); j < 10; j++ { - err := f.importValue(tx, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) + err := f.importValue(qw, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) if err != nil { return err } @@ -4318,17 +4182,17 @@ func TestImportMultipleValues(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - f, _, tx := mustOpenFragment(t) + f, _, qw := mustOpenFragment(t) defer f.Clean(t) - err := f.importValue(tx, test.cols, test.vals, test.depth, false) + err := f.importValue(qw, test.cols, test.vals, test.depth, false) if err != nil { t.Fatalf("importing values: %v", err) } for i := range test.checkCols { cc, cv := test.checkCols[i], test.checkVals[i] - n, exists, err := f.value(tx, cc, test.depth) + n, exists, err := f.value(qw, cc, test.depth) if err != nil { t.Fatalf("getting value: %v", err) } @@ -4372,26 +4236,26 @@ func TestImportValueRowCache(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - f, _, tx := mustOpenFragment(t) + f, _, qw := mustOpenFragment(t) defer f.Clean(t) // First import (tc1) - if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { + if err := f.importValue(qw, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { t.Fatalf("importing values: %v", err) } - if r, err := f.rangeOp(tx, pql.GT, test.tc1.depth, 0); err != nil { + if r, err := f.rangeOp(qw, pql.GT, test.tc1.depth, 0); err != nil { t.Error("getting range of values") } else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) { t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns()) } // Second import (tc2) - if err := f.importValue(tx, test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { + if err := f.importValue(qw, test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { t.Fatalf("importing values: %v", err) } - if r, err := f.rangeOp(tx, pql.GT, test.tc2.depth, 0); err != nil { + if r, err := f.rangeOp(qw, pql.GT, test.tc2.depth, 0); err != nil { t.Error("getting range of values") } else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) { t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns()) @@ -4404,44 +4268,38 @@ func TestImportValueRowCache(t *testing.T) { // do we see races/corruption around concurrent read/write. // especially on writes to the row cache. func TestFragmentConcurrentReadWrite(t *testing.T) { - f, idx, tx := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) + f, qcx, _ := mustOpenFragment(t, OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) defer f.Clean(t) - tx.Rollback() + qcx.Release() eg := &errgroup.Group{} eg.Go(func() error { - - ltx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + qcx, qw := mustWrite(t, f) for i := uint64(0); i < 1000; i++ { - _, err := f.setBit(ltx, i%4, i) + _, err := f.setBit(qw, i%4, i) if err != nil { return errors.Wrap(err, "setting bit") } } - PanicOn(ltx.Commit()) + require.Nil(t, qcx.Commit()) return nil }) - // need read-only Tx so as not to block on the writer finishing above. - tx1 := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx1.Rollback() + _, qr := mustRead(t, f) acc := uint64(0) for i := uint64(0); i < 100; i++ { - r := f.mustRow(tx1, i%4) + r := f.mustRow(t, qr, i%4) acc += r.Count() } if err := eg.Wait(); err != nil { t.Errorf("error from setting a bit: %v", err) } - - t.Logf("%d", acc) } func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { - f, idx, tx := mustOpenFragment(t) - _ = idx + f, _, qw := mustOpenFragment(t) // byShardWidth is a map of the same roaring (fragment) data generated // with different shard widths. // TODO: a better approach may be to generate this in the test based @@ -4456,50 +4314,49 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { b = data } - _ = idx defer f.Clean(t) - err := f.importRoaringT(tx, b, false) + err := f.importRoaring(context.Background(), qw, b, false) if err != nil { t.Fatalf("importing roaring: %v", err) } //check the bit - res := f.mustRow(tx, 1).Columns() - if len(res) < 1 || f.mustRow(tx, 1).Columns()[0] != 1 { + res := f.mustRow(t, qw, 1).Columns() + if len(res) < 1 || f.mustRow(t, qw, 1).Columns()[0] != 1 { t.Fatalf("expecting 1 got: %v", res) } //clear the bit - changed, _ := f.clearBit(tx, 1, 1) + changed, _ := f.clearBit(qw, 1, 1) if !changed { t.Fatalf("expected change got %v", changed) } //check missing - res = f.mustRow(tx, 1).Columns() + res = f.mustRow(t, qw, 1).Columns() if len(res) != 0 { t.Fatalf("expected nothing got %v", res) } // import again - err = f.importRoaringT(tx, b, false) + err = f.importRoaring(context.Background(), qw, b, false) if err != nil { t.Fatalf("importing roaring: %v", err) } //check - res = f.mustRow(tx, 1).Columns() - if len(res) < 1 || f.mustRow(tx, 1).Columns()[0] != 1 { + res = f.mustRow(t, qw, 1).Columns() + if len(res) < 1 || f.mustRow(t, qw, 1).Columns()[0] != 1 { t.Fatalf("again expecting 1 got: %v", res) } - changed, _ = f.clearBit(tx, 1, 1) + changed, _ = f.clearBit(qw, 1, 1) if !changed { t.Fatalf("again expected change got %v", changed) // again expected change got false } //check missing - res = f.mustRow(tx, 1).Columns() + res = f.mustRow(t, qw, 1).Columns() if len(res) != 0 { t.Fatalf("expected nothing got %v", res) } @@ -4692,7 +4549,7 @@ func TestImportMutexSampleData(t *testing.T) { seen := make(map[uint64]struct{}) t.Run(data.name, func(t *testing.T) { batchSize := 16384 - f, _, tx := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) + f, _, qw := mustOpenFragment(t, OptFieldTypeMutex(DefaultCacheType, DefaultCacheSize)) defer f.Clean(t) // Set import. var err error @@ -4709,14 +4566,14 @@ func TestImportMutexSampleData(t *testing.T) { if len(cols) < max { max = len(cols) } - err = f.bulkImport(tx, rows[j:max:max], cols[j:max:max], &ImportOptions{}) + err = f.bulkImport(qw, rows[j:max:max], cols[j:max:max], &ImportOptions{}) if err != nil { t.Fatalf("bulk importing ids [%d/3] [%d:%d]: %v", i+1, j, max, err) } } count := uint64(0) for k := uint32(0); k < data.rng.rows(); k++ { - c := f.mustRow(tx, uint64(k)).Count() + c := f.mustRow(t, qw, uint64(k)).Count() count += c } if int(count) != len(seen) { @@ -4742,9 +4599,9 @@ func BenchmarkImportMutexSampleData(b *testing.B) { var data *mutexSampleData var cache string var batchSize int - var frag *fragment - var tx Tx - var idx *Index + var f *fragment + var qcx qc.QueryContext + var qw qc.QueryWrite benchmarkOneFragmentImports := func(b *testing.B, idx int) { cols, rows = data.scratchSpace(idx, cols, rows) toDo := b.N << 16 @@ -4754,7 +4611,7 @@ func BenchmarkImportMutexSampleData(b *testing.B) { if len(cols) < max { max = len(cols) } - err := frag.bulkImport(tx, rows[start:max:max], cols[start:max:max], &ImportOptions{}) + err := f.bulkImport(qw, rows[start:max:max], cols[start:max:max], &ImportOptions{}) if err != nil { b.Fatalf("bulk importing ids [%d:%d]: %v", start, max, err) } @@ -4770,19 +4627,19 @@ func BenchmarkImportMutexSampleData(b *testing.B) { } } benchmarkFragmentImports := func(b *testing.B) { - frag, idx, tx = mustOpenFragment(b, OptFieldTypeMutex(cache, DefaultCacheSize)) - defer frag.Clean(b) + f, qcx, qw = mustOpenFragment(b, OptFieldTypeMutex(cache, DefaultCacheSize)) + defer f.Clean(b) + defer qcx.Release() for i := range data.colIDs { b.Run(fmt.Sprintf("write-%d", i), func(b *testing.B) { benchmarkOneFragmentImports(b, i) }) // Then commit that write and do another one as a new Tx. - err := tx.Commit() + err := qcx.Commit() if err != nil { b.Fatalf("error commiting write: %v", err) } - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: frag, Shard: 0}) - defer tx.Rollback() + qcx, qw = mustWrite(b, f) } } for _, data = range sampleMutexData { @@ -5058,7 +4915,7 @@ func TestSliceDifference(t *testing.T) { } func TestImportRoaringSingleValued(t *testing.T) { - f, _, tx := mustOpenFragment(t) + f, _, qw := mustOpenFragment(t) defer f.Clean(t) clear := roaring.NewBitmap(0, 1, ShardWidth-1) @@ -5068,7 +4925,7 @@ func TestImportRoaringSingleValued(t *testing.T) { {0, 1, ShardWidth - 1}, }...) - err := f.ImportRoaringSingleValued(context.Background(), tx, clear.Roaring(), set.Roaring()) + err := f.ImportRoaringSingleValued(context.Background(), qw, clear.Roaring(), set.Roaring()) if err != nil { t.Fatalf("importing: %v", err) } @@ -5078,7 +4935,7 @@ func TestImportRoaringSingleValued(t *testing.T) { {}, {0, 1, ShardWidth - 1}, }...) - if err := f.ImportRoaringSingleValued(context.Background(), tx, clear.Roaring(), set.Roaring()); err != nil { + if err := f.ImportRoaringSingleValued(context.Background(), qw, clear.Roaring(), set.Roaring()); err != nil { t.Fatalf("importing: %v", err) } @@ -5090,11 +4947,11 @@ func TestImportRoaringSingleValued(t *testing.T) { {0, 1, ShardWidth - 1}, }...) - if err := f.ImportRoaringSingleValued(context.Background(), tx, clear.Roaring(), set.Roaring()); err != nil { + if err := f.ImportRoaringSingleValued(context.Background(), qw, clear.Roaring(), set.Roaring()); err != nil { t.Fatalf("importing: %v", err) } - result, err := tx.RoaringBitmap("i", "f", "v", 0) + result, err := qw.RoaringBitmap() if err != nil { t.Fatalf("getting bitmap: %v", err) } diff --git a/handler.go b/handler.go index e08197982..5ed2e5c5f 100644 --- a/handler.go +++ b/handler.go @@ -448,7 +448,6 @@ type ImportRoaringRequest struct { FieldCreatedAt int64 Clear bool Action string // [set, clear, overwrite] - Block int Views map[string][]byte UpdateExistence bool SuppressLog bool diff --git a/holder.go b/holder.go index feeef5dcf..158517556 100644 --- a/holder.go +++ b/holder.go @@ -14,13 +14,15 @@ import ( "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/logger" + qc "github.com/molecula/featurebase/v3/querycontext" rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/storage" + "github.com/molecula/featurebase/v3/task" "github.com/molecula/featurebase/v3/testhook" - "github.com/molecula/featurebase/v3/vprint" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -127,7 +129,7 @@ type Holder struct { Auditor testhook.Auditor - txf *TxFactory + txStore qc.TxStore lookupDB *sql.DB @@ -146,6 +148,35 @@ type Holder struct { // snapshotter/writelogger; then MDS should only start directing queries to // that computer once it has completed applying the snapshot. directiveApplied bool + + dbContents keys.DBContents // used during startup to determine which views to open, etc +} + +func (h *Holder) NewQueryContext(ctx context.Context) (qc.QueryContext, error) { + return h.TxStore().NewQueryContext(ctx) +} + +func (h *Holder) NewWriteQueryContext(ctx context.Context, scope qc.QueryScope) (qc.QueryContext, error) { + return h.TxStore().NewWriteQueryContext(ctx, scope) +} + +// NewIndexQueryContext is a helper to create a scope for a given index, and +// optional list of shards. If no shards are provided, the context is +// index-wide. +func (h *Holder) NewIndexQueryContext(ctx context.Context, index string, shards ...uint64) (qc.QueryContext, error) { + txs := h.TxStore() + var typeShifted []keys.Shard + // helpfully treat a shard of -1 as no shard + if len(shards) > 0 && shards[0] == ^uint64(0) { + shards = shards[1:] + } + if len(shards) > 0 { + typeShifted = make([]keys.Shard, len(shards)) + for i, v := range shards { + typeShifted[i] = keys.Shard(v) + } + } + return txs.NewWriteQueryContext(ctx, txs.Scope().AddIndexShards(keys.Index(index), typeShifted...)) } // HolderOpts holds information about the holder which other things might want @@ -166,6 +197,16 @@ func (h *Holder) Directive() dax.Directive { return *h.directive } +// TxStore yields the backing TxStore used by this holder. If no TxStore is +// set, it yields a NopTxStore which errors out on usage, rather than panicing. +func (h *Holder) TxStore() qc.TxStore { + if h.txStore != nil { + return h.txStore + } + // if you somehow didn't pick a TxStore, we want to error peacefully rather than panicing + return qc.NopTxStore +} + func (h *Holder) SetDirective(d *dax.Directive) { if d == nil { return @@ -303,7 +344,7 @@ func TestHolderConfig() *HolderConfig { } // NewHolder returns a new instance of Holder for the given path. -func NewHolder(path string, cfg *HolderConfig) *Holder { +func NewHolder(path string, cfg *HolderConfig) (*Holder, error) { if cfg == nil { cfg = DefaultHolderConfig() } @@ -335,6 +376,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { Schemator: cfg.Schemator, Logger: cfg.Logger, Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend}, + txStore: qc.NopTxStore, Auditor: NewAuditor(), @@ -343,12 +385,8 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { indexes: make(map[string]*Index), } - txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h) - vprint.PanicOn(err) - h.txf = txf - _ = testhook.Created(h.Auditor, h, nil) - return h + return h, nil } // Path returns the path directory the holder was created with. @@ -361,6 +399,228 @@ func (h *Holder) IndexesPath() string { return filepath.Join(h.path, IndexesDir) } +// transactExistRow atomically grabs a currently-unused row of the existence +// field to store some bits in. These bits are used to denote records which +// we are in the process of deleting. By "atomically" we mean that this +// operation gets its own QueryContext, which it commits before returning. +// You cannot use this while you already have a live QueryContext referring +// to this shard. +func (h *Holder) transactExistRow(ctx context.Context, qcx qc.QueryContext, idx *Index, shard uint64, frag *fragment, src *Row) (uint64, error) { + qw, err := frag.qcxWrite(qcx) + if err != nil { + return 0, err + } + rows, err := frag.rows(ctx, qw, 1) + if err != nil { + return 0, err + } + // obtain a rowID which is higher than any currently present row ID. + rowID := uint64(1) + if len(rows) > 0 { + rowID = rows[len(rows)-1] + 1 + } + _, err = frag.setRow(qw, src, rowID) + if err != nil { + return 0, err + } + return rowID, qcx.Flush(keys.Index(idx.name), keys.Shard(shard)) +} + +// deleteRows deletes the everything from the given index/shard matching a provided +// Row. +func (h *Holder) deleteRows(ctx context.Context, qcx qc.QueryContext, src *Row, idx *Index, shard uint64) (bool, error) { + return h.deleteRowsWithFlow(ctx, qcx, src, idx, shard, false) +} + +// deleteRowsWithFlowWithKeys deletes the given columns from every field, for the given +// index/shard. The "normalFlow" parameter tells whether we're trying to do a recovery +// of an interrupted delete. +func (h *Holder) deleteRowsWithFlowWithKeys(ctx context.Context, qcx qc.QueryContext, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (bool, error) { + var existenceFragment *fragment + var deletedRowID uint64 + var commitor Commitor = &NopCommitor{} + var err error // store columns in exits field ToBeDelete row commited + if normalFlow { // normalFlow is the standard path, "not normal" is recoverory + existenceFragment = h.fragment(idx.Name(), existenceFieldName, viewStandard, shard) + if existenceFragment == nil { + // no exists field + return false, errors.New("can't bulk delete without existence field") + } + src := NewRowFromBitmap(columns) + deletedRowID, err = h.transactExistRow(ctx, qcx, idx, shard, existenceFragment, src) + if err != nil { + return false, err + } + } + commitor, err = deleteKeyTranslation(ctx, idx, shard, columns) + if err != nil { + return false, err + } + changed := false + defer func() { + // if there is an error on the bit clearing rollback the keys + if err != nil { + changed = false + commitor.Rollback() + return + } + // if there is an error in the key commit, then rollback the delete + // write records before keys to remove possiblity of unmatch keys=records + err = qcx.Flush(keys.Index(idx.name), keys.Shard(shard)) + if err != nil { + changed = false + commitor.Rollback() + return + } + if er := commitor.Commit(); er != nil { + err = er + } + if err != nil { + h.Logger.Errorf("problems committing delete in rbf %v shard %v", err, shard) + } + }() + + for _, field := range idx.Fields() { + for _, view := range field.views() { + frag := view.Fragment(shard) + if frag == nil { + continue + } + qw, err := frag.qcxWrite(qcx) + if err != nil { + return false, err + } + c, err := frag.clearRecordsByBitmap(qw, columns) + if err != nil { + return false, err + } + changed = changed || c + } + } + if existenceFragment == nil { + return changed, nil + } + // a string keys have been deleted and the deleteRow was created + qw, err := existenceFragment.qcxWrite(qcx) + if err != nil { + return false, err + } + if normalFlow { + existenceFragment.clearRow(qw, deletedRowID) + } else { + // this is if we are recovering from failure and cleaning up + rows, err := existenceFragment.rows(ctx, qw, 1) + if err != nil { + return false, err + } + for _, rowId := range rows { + existenceFragment.clearRow(qw, rowId) + } + } + // Unlike other operations, Delete wants to ensure that its operations are flushed. + return changed, qcx.Flush(keys.Index(idx.name), keys.Shard(shard)) +} + +// deleteRowsWithOutKeys deletes the data for a given index/shard, matching the columns +// in the given bitmap, but does not attempt to delete corresponding keys. +func (h *Holder) deleteRowsWithOutKeysFlow(ctx context.Context, qcx qc.QueryContext, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (changed bool, err error) { + var existenceFragment *fragment + var deletedRowID uint64 + var commitor Commitor = &NopCommitor{} + defer func() { + // if there is an error in the key commit, then rollback the delete + // write records before keys to remove possiblity of unmatch keys=records + err := qcx.Flush(keys.Index(idx.name), keys.Shard(shard)) + if err != nil { + changed = false + commitor.Rollback() + return + } + }() + for _, field := range idx.Fields() { + for _, view := range field.views() { + frag := view.Fragment(shard) + if frag == nil { + continue + } + qw, err := frag.qcxWrite(qcx) + if err != nil { + return false, err + } + c, err := frag.clearRecordsByBitmap(qw, columns) + if err != nil { + return false, err + } + if c { + changed = true + } + + } + } + if existenceFragment == nil { // a string keys have been deleted and the deleteRow was created + return changed, nil + } + qw, err := existenceFragment.qcxWrite(qcx) + if err != nil { + return false, err + } + + if normalFlow { + existenceFragment.clearRow(qw, deletedRowID) + return changed, nil + } + + // this is if we are recovering from failure and cleaning up + rows, err := existenceFragment.rows(ctx, qw, 1) + if err != nil { + return false, err + } + for _, rowId := range rows { + existenceFragment.clearRow(qw, rowId) + } + return changed, nil +} + +// deleteRowsWithFlow deletes all the entries from the index for a given +// index/shard. Note that we expect the source row to have only one +// segment, which is the right one. +func (h *Holder) deleteRowsWithFlow(ctx context.Context, qcx qc.QueryContext, src *Row, idx *Index, shard uint64, normalFlow bool) (change bool, err error) { + if len(src.Segments) == 0 { // nothing to remove + return false, nil + } + if src.Segments[0].shard != shard { + return false, fmt.Errorf("data mismatch: expected data to delete for shard %d, got shard %d", shard, src.Segments[0].shard) + } + columns := src.Segments[0].data // should only be one segment + if columns.Count() == 0 { + return false, nil + } + bits := src.Segments[0].data.Slice() + min := func(a, b int) int { + if a <= b { + return a + } + return b + } + // We may not be able to delete all of the keys at once, so we have to batch + // them. + limit := h.cfg.RBFConfig.MaxDelete + var anyChanges bool + for i := 0; i < len(bits); i += limit { + batch := roaring.NewBitmap(bits[i:min(i+limit, len(bits))]...) + if idx.Keys() { + change, err = h.deleteRowsWithFlowWithKeys(ctx, qcx, batch, idx, shard, normalFlow) + } else { + change, err = h.deleteRowsWithOutKeysFlow(ctx, qcx, batch, idx, shard, normalFlow) + } + anyChanges = anyChanges || change + if err != nil { + return anyChanges, err + } + } + return anyChanges, err +} + func (h *Holder) deletePerShard(index *Index, shard uint64) error { inprocessRecords := NewRow() @@ -369,11 +629,18 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error { return nil } - tx := h.Txf().NewTx(Txo{Write: !writable, Index: index, Shard: shard}) - defer tx.Rollback() + qcx, err := h.NewIndexQueryContext(context.TODO(), index.name, shard) + if err != nil { + return err + } + defer qcx.Release() + qr, err := frag.qcxRead(qcx) + if err != nil { + return err + } // filter rows based on having _exists>=1, which is used to flag delete in-flight - rows, err := frag.rows(context.Background(), tx, 1) + rows, err := frag.rows(context.Background(), qr, 1) if err != nil { return err } @@ -384,7 +651,7 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error { } for _, record := range rows { - row, err2 := frag.row(tx, record) + row, err2 := frag.row(qr, record) if err2 != nil { return fmt.Errorf("getting row IDs: %v", err2) } @@ -392,9 +659,7 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error { } h.Logger.Printf("retrying delete: index=%v shard=%v record count=%v", index.name, shard, inprocessRecords.Count()) - tx.Rollback() // release the read tx in case a checksum is needed in DeleteRows - - _, err = DeleteRows(context.Background(), inprocessRecords, index, shard) + _, err = h.deleteRows(context.Background(), qcx, inprocessRecords, index, shard) if err != nil { return fmt.Errorf("deleting rows: %v", err) } @@ -439,12 +704,27 @@ func (h *Holder) Open() error { h.opening = true defer func() { h.opening = false }() - if h.txf == nil { - txf, err := NewTxFactory(h.cfg.StorageConfig.Backend, h.IndexesPath(), h) - if err != nil { - return errors.Wrap(err, "Holder.Open NewTxFactory()") - } - h.txf = txf + // allow overwriting a NopTxStore, but not a real one + if h.txStore != nil && h.txStore != qc.NopTxStore { + return errors.New("holder already had previous TxStore on open") + } + var workerPool *task.Pool + // in production this can almost certainly never be nil. with test + // holders, it is often nil and there's no worker pool to worry about. + // since the worker pool is used only to notify the worker pool that + // we're blocked, that's probably harmless. + if h.executor != nil { + workerPool = h.executor.workers + } + txs, err := qc.NewRBFTxStore(h.IndexesPath(), h.cfg.RBFConfig, h.Logger, workerPool, nil) + if err != nil { + return err + } + h.txStore = txs + + h.dbContents, err = h.txStore.Contents() + if err != nil { + return errors.Wrap(err, "obtaining existing fields/views from store") } // Reset closing in case Holder is being reopened. @@ -506,7 +786,7 @@ func (h *Holder) Open() error { err = index.OpenWithSchema(idx) if err != nil { - _ = h.txf.Close() + _ = h.txStore.Close() if err == ErrName { h.Logger.Errorf("opening index: %s, err=%s", index.Name(), err) continue @@ -532,10 +812,6 @@ func (h *Holder) Open() error { _ = testhook.Opened(h.Auditor, h, nil) - if err := h.txf.Open(); err != nil { - return errors.Wrap(err, "Holder.Open h.txf.Open()") - } - if h.cfg.LookupDBDSN != "" { h.Logger.Printf("connecting to lookup database") @@ -624,10 +900,6 @@ func (h *Holder) Close() error { return nil } - if globalUseStatTx { - fmt.Printf("%v\n", globalCallStats.report()) - } - h.Stats.Close() // Notify goroutines of closing and wait for completion. @@ -638,15 +910,16 @@ func (h *Holder) Close() error { return errors.Wrap(err, "closing index") } } - if err := h.txf.Close(); err != nil { - return errors.Wrap(err, "holder.Txf.Close()") + if err := h.txStore.Close(); err != nil { + return errors.Wrap(err, "closing database backend") } + // set txStore to something that errors harmlessly, since it's closed now. + h.txStore = qc.NopTxStore if err := h.ida.Close(); err != nil { return errors.Wrap(err, "closing ID allocator") } // Reset opened in case Holder needs to be reopened. - h.txf = nil h.opened.mu.Lock() h.opened.ch = make(chan struct{}) h.opened.mu.Unlock() @@ -1047,7 +1320,6 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar index.trackExistence = cim.Meta.TrackExistence index.createdAt = cim.CreatedAt index.translatePartitions = translatePartitions - if err = index.Open(); err != nil { return nil, errors.Wrap(err, "opening") } @@ -1190,8 +1462,8 @@ func (h *Holder) deleteIndex(name string) error { } // remove any backing store. - if err := h.txf.DeleteIndex(name); err != nil { - return errors.Wrap(err, "h.Txf.DeleteIndex") + if err := h.txStore.DeleteIndex(keys.Index(name)); err != nil { + return errors.Wrap(err, "deleting index") } // Delete index directory. @@ -1745,18 +2017,6 @@ func (h *Holder) addIndex(idx *Index) { h.imu.Unlock() } -func (h *Holder) Txf() *TxFactory { - h.mu.Lock() - defer h.mu.Unlock() - return h.txf -} - -// BeginTx starts a transaction on the holder. The index and shard -// must be specified. -func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) { - return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil -} - func decodeCreateIndexMessage(ser Serializer, b []byte) (*CreateIndexMessage, error) { var cim CreateIndexMessage if err := ser.Unmarshal(b, &cim); err != nil { diff --git a/holder_internal_test.go b/holder_internal_test.go index d22ae820b..ad5917889 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -2,9 +2,41 @@ package pilosa import ( + "context" "testing" + + qc "github.com/molecula/featurebase/v3/querycontext" + "github.com/stretchr/testify/require" ) +// MustIndexQueryContext gets a query context which can write to +// the specified index (and possibly shards), or fails the test. +// The qcx will be automatically cleaned up when the test completes. +func (h *Holder) MustIndexQueryContext(tb testing.TB, index string, shards ...uint64) qc.QueryContext { + tb.Helper() + // disregard a leading ^0, because that's idiomatic for "all shards" + if len(shards) > 0 && shards[0] == ^uint64(0) { + shards = shards[1:] + } + qcx, err := h.NewIndexQueryContext(context.Background(), index, shards...) + if err != nil { + tb.Fatalf("creating query context: %v", err) + } + tb.Cleanup(qcx.Release) + return qcx +} + +// MustQueryContext gets a read-only query context or fails the test. +func (h *Holder) MustQueryContext(tb testing.TB) qc.QueryContext { + tb.Helper() + qcx, err := h.NewQueryContext(context.Background()) + if err != nil { + tb.Fatalf("creating query context: %v", err) + } + tb.Cleanup(qcx.Release) + return qcx +} + func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*Index, *Field) { idx, err := h.CreateIndexIfNotExists(indexName, "", IndexOptions{TrackExistence: true}) if err != nil { @@ -16,8 +48,7 @@ func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*In } existencefield := idx.existenceFld - qcx := h.Txf().NewWritableQcx() - defer qcx.Abort() + qcx := h.MustIndexQueryContext(t, indexName) for _, r := range rowCol { _, err = f.SetBit(qcx, r.row, r.col, nil) @@ -31,9 +62,7 @@ func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*In } } - if err = qcx.Finish(); err != nil { - t.Fatalf("failed to commit tx for index %v: %v", indexName, err) - } + require.Nil(t, qcx.Commit()) shardsFound := idx.AvailableShards(includeRemote).Slice() if len(shardsFound) != 3 { @@ -76,8 +105,7 @@ func TestHolder_ProcessDeleteInflight(t *testing.T) { for _, test := range tests { func() { idx, f := test.idx, test.f - qcx := h.Txf().NewQcx() - defer qcx.Abort() + qcx := h.MustQueryContext(t) for _, r := range rowCol { row, err := f.Row(qcx, r.row) if err != nil { diff --git a/holder_test.go b/holder_test.go index f909331c8..2ab2e540f 100644 --- a/holder_test.go +++ b/holder_test.go @@ -162,7 +162,10 @@ func TestHolder_HasData(t *testing.T) { // Note that we are intentionally not using test.NewHolder, // because we want to create a Holder object with an invalid path, // rather than creating a valid holder with a temporary path. - h := pilosa.NewHolder("bad-path", pilosa.TestHolderConfig()) + h, err := pilosa.NewHolder("bad-path", pilosa.TestHolderConfig()) + if err != nil { + t.Fatalf("surprisingly, got an error from NewHolder with an invalid path which we didn't expect: %v", err) + } if ok, err := h.HasData(); ok || err != nil { t.Fatal("expected HasData to return false, no err, but", ok, err) diff --git a/http_handler.go b/http_handler.go index c17c2775d..5b7965284 100644 --- a/http_handler.go +++ b/http_handler.go @@ -323,9 +323,6 @@ func (h *Handler) populateValidators() { h.validators["GetStatus"] = queryValidationSpecRequired() h.validators["GetVersion"] = queryValidationSpecRequired() h.validators["PostClusterMessage"] = queryValidationSpecRequired() - h.validators["GetFragmentBlockData"] = queryValidationSpecRequired() - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard") h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") h.validators["GetPartitionNodes"] = queryValidationSpecRequired("partition") h.validators["GetNodes"] = queryValidationSpecRequired() @@ -572,9 +569,6 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/mem-usage", handler.chkAuthZ(handler.handleGetMemUsage, authz.Read)).Methods("GET").Name("GetUsage") router.HandleFunc("/internal/disk-usage", handler.chkAuthZ(handler.handleGetDiskUsage, authz.Read)).Methods("GET").Name("GetUsage") router.HandleFunc("/internal/disk-usage/{index}", handler.chkAuthZ(handler.handleGetDiskUsage, authz.Read)).Methods("GET").Name("GetUsage") - router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData") - router.HandleFunc("/internal/fragment/blocks", handler.chkAuthN(handler.handleGetFragmentBlocks)).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/internal/fragment/data", handler.chkAuthN(handler.handleGetFragmentData)).Methods("GET").Name("GetFragmentData") router.HandleFunc("/internal/fragment/nodes", handler.chkAuthN(handler.handleGetFragmentNodes)).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/internal/partition/nodes", handler.chkAuthN(handler.handleGetPartitionNodes)).Methods("GET").Name("GetPartitionNodes") router.HandleFunc("/internal/translate/keys", handler.chkAuthN(handler.handlePostTranslateKeys)).Methods("POST").Name("PostTranslateKeys") @@ -2583,14 +2577,14 @@ func validateProtobufHeader(r *http.Request) (error string, code int) { // handleGetInternalDebugRBFJSON handles /internal/debug/rbf requests. func (h *Handler) handleGetInternalDebugRBFJSON(w http.ResponseWriter, r *http.Request) { - buf, err := json.MarshalIndent(h.api.RBFDebugInfo(), "", " ") + w.Header().Set("Content-Type", "application/octet-stream") + var buf bytes.Buffer + err := h.api.holder.TxStore().DumpDot(&buf) if err != nil { - http.Error(w, "marshal json: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "rendering DOT: "+err.Error(), http.StatusInternalServerError) return } - - w.Header().Set("Content-Type", "application/json") - w.Write(buf) + w.Write(buf.Bytes()) } // handleGetMetricsJSON handles /metrics.json requests, translating text metrics results to more consumable JSON. @@ -2746,37 +2740,6 @@ func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) { } } -// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests. -func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - http.Error(w, "fragment blocks feature removed", http.StatusNotFound) -} - -// handleGetFragmentBlocks handles GET /internal/fragment/blocks requests. -func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { - http.Error(w, "fragment blocks feature removed", http.StatusNotFound) -} - -// handleGetFragmentData handles GET /internal/fragment/data requests. -func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { - // Read shard parameter. - q := r.URL.Query() - shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) - if err != nil { - http.Error(w, "shard required", http.StatusBadRequest) - return - } - // Retrieve fragment data from holder. - f, err := h.api.FragmentData(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - // Stream fragment to response body. - if _, err := f.WriteTo(w); err != nil { - h.logger.Errorf("error streaming fragment data: %s", err) - } -} - // handleGetTranslateData handles GET /internal/translate/data requests. func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -3075,12 +3038,15 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re return } - qcx := h.api.Txf().NewQcx() + qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer qcx.Release() err = h.api.ImportAtomicRecord(r.Context(), qcx, req, opt) if err == nil { - err = qcx.Finish() - } else { - qcx.Abort() + err = qcx.Commit() } if err != nil { switch errors.Cause(err) { @@ -3153,9 +3119,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - - qcx := h.api.Txf().NewQcx() - defer qcx.Abort() + // mark that we don't really have a specific shard + if len(req.ColumnKeys) != 0 { + req.Shard = ^uint64(0) + } + // ^0 is special and doesn't count as a shard + qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer qcx.Release() if err := h.api.ImportValue(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { @@ -3169,9 +3143,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - err := qcx.Finish() + err = qcx.Commit() if err != nil { - http.Error(w, fmt.Sprintf("error in qcx.Finish(): '%v'", err.Error()), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("error committing import: '%v'", err.Error()), http.StatusInternalServerError) return } } else { @@ -3183,8 +3157,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - qcx := h.api.Txf().NewQcx() - defer qcx.Abort() + // mark that we don't really have a specific shard + if len(req.ColumnKeys) != 0 { + req.Shard = ^uint64(0) + } + // ^0 is special and doesn't count as a shard + qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer qcx.Release() if err := h.api.Import(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { @@ -3197,9 +3180,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } return } - err := qcx.Finish() + err = qcx.Commit() if err != nil { - http.Error(w, fmt.Sprintf("error in qcx.Finish() on set,time,mutex: '%v'", err.Error()), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("error committing import: '%v'", err.Error()), http.StatusInternalServerError) return } } @@ -3231,8 +3214,12 @@ func (h *Handler) handleGetMutexCheck(w http.ResponseWriter, r *http.Request) { http.Error(w, "limit must be numeric", http.StatusBadRequest) } } - qcx := h.api.Txf().NewQcx() - defer qcx.Abort() + qcx, err := h.api.NewQueryContext(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer qcx.Release() out, err := h.api.MutexCheck(r.Context(), qcx, indexName, fieldName, details, limit) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3268,8 +3255,12 @@ func (h *Handler) handleInternalGetMutexCheck(w http.ResponseWriter, r *http.Req http.Error(w, "limit must be numeric", http.StatusBadRequest) } } - qcx := h.api.Txf().NewQcx() - defer qcx.Abort() + qcx, err := h.api.NewQueryContext(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer qcx.Release() out, err := h.api.MutexCheckNode(r.Context(), qcx, indexName, fieldName, details, limit) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3805,6 +3796,7 @@ func (h *Handler) handleRestoreIDAlloc(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() indexName, ok := mux.Vars(r)["index"] if !ok { http.Error(w, "index name is required", http.StatusBadRequest) diff --git a/index.go b/index.go index d3d8ee962..67ba4bccf 100644 --- a/index.go +++ b/index.go @@ -14,6 +14,7 @@ import ( "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/stats" @@ -57,9 +58,6 @@ type Index struct { // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc - // track the subset of shards available to our views - fieldView2shard *FieldView2Shards - // indicate that we're closing and should wrap up and not allow new actions closing chan struct{} } @@ -97,10 +95,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { return idx, nil } -func (i *Index) NewTx(txo Txo) Tx { - return i.holder.txf.NewTx(txo) -} - // CreatedAt is an timestamp for a specific version of an index. func (i *Index) CreatedAt() int64 { i.mu.RLock() @@ -168,7 +162,8 @@ func (i *Index) Open() error { } // OpenWithSchema opens the index and uses the provided schema to verify that -// the index's fields are expected. +// the index's fields are expected. The provided QueryContext is to be used +// if any of this requires actual database reads. func (i *Index) OpenWithSchema(idx *disco.Index) error { if idx == nil { return ErrInvalidSchema @@ -208,17 +203,6 @@ func (i *Index) open(idx *disco.Index) (err error) { } i.closing = make(chan struct{}) - // fmt.Printf("new channel %p for index %p\n", i.closing, i) - - // we don't want to open *all* the views for each shard, since - // most are empty when we are doing time quantums. It slows - // down startup dramatically. So we ask for the meta data - // of what fields/views/shards are present with data up front. - fieldView2shard, err := i.holder.txf.GetFieldView2ShardsMapForIndex(i) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("i.holder.txf.GetFieldView2ShardsMapForIndex('%v')", i.name)) - } - i.fieldView2shard = fieldView2shard // Add index to a map in holder. Used by openFields. i.holder.addIndex(i) @@ -228,15 +212,11 @@ func (i *Index) open(idx *disco.Index) (err error) { return errors.Wrap(err, "opening fields") } - // Set bit depths. - // This is called in Index.open() (as opposed to Field.Open()) because the - // Field.bitDepth() method uses a transaction which relies on the index and - // its entry for the field in the Index.field map. If we try to set a - // field's BitDepth in Field.Open(), which itself might be inside the - // Index.openField() loop, then the field has not yet been added to the - // Index.field map. I think it would be better if Field.bitDepth didn't rely - // on its index at all, but perhaps with transactions that not possible. I - // don't know. + // Set bit depths based on current contents of fields. + // We could in theory do this as part of opening each field, + // but what query context would they use for the actual + // database transactions? So we have our own top-level + // thing to do that. if err := i.setFieldBitDepths(); err != nil { return errors.Wrap(err, "setting field bitDepths") } @@ -414,7 +394,14 @@ func (i *Index) openExistenceField() error { } // setFieldBitDepths sets the BitDepth for all int and decimal fields in the index. +// We do it here, rather than when each field is opened, so we can open a +// single QueryContext to handle them all. func (i *Index) setFieldBitDepths() error { + qcx, err := i.holder.NewIndexQueryContext(context.TODO(), i.name) + if err != nil { + return err + } + defer qcx.Release() for name, f := range i.fields { switch f.Type() { case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: @@ -422,7 +409,7 @@ func (i *Index) setFieldBitDepths() error { default: continue } - bd, err := f.bitDepth() + bd, err := f.bitDepth(qcx) if err != nil { return errors.Wrapf(err, "getting bit depth for field: %s", name) } @@ -451,11 +438,6 @@ func (i *Index) Close() error { _ = testhook.Closed(i.holder.Auditor, i, nil) }() - err := i.holder.txf.CloseIndex(i) - if err != nil { - return errors.Wrap(err, "closing index") - } - // Close partitioned translation stores. for _, store := range i.translateStores { if err := store.Close(); err != nil { @@ -514,11 +496,6 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { return b } -// Begin starts a transaction on a shard of the index. -func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) { - return i.holder.txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil -} - // fieldPath returns the path to a field in the index. func (i *Index) fieldPath(name string) string { return filepath.Join(i.FieldsPath(), name) } @@ -956,15 +933,18 @@ func (i *Index) DeleteField(name string) error { return errors.Wrap(err, "closing") } - if err := i.holder.txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil { - return errors.Wrap(err, "Txf.DeleteFieldFromStore") + fieldPath := i.fieldPath(name) + err := os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "deleting field directory") + } + + if err := i.holder.txStore.DeleteField(keys.Index(i.name), keys.Field(name)); err != nil { + return errors.Wrap(err, "deleting field from store") } // Remove reference. delete(i.fields, name) - - // remove shard metadata for field - i.fieldView2shard.removeField(name) return i.translationSyncer.Reset() } diff --git a/index_internal_test.go b/index_internal_test.go deleted file mode 100644 index 6915bb385..000000000 --- a/index_internal_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "testing" -) - -// mustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index { - h := newTestHolder(tb) - index, err := h.CreateIndex("i", "", opt) - - if err != nil { - panic(err) - } - - index.keys = opt.Keys - index.trackExistence = opt.TrackExistence - - return index -} diff --git a/internal_client.go b/internal_client.go index 8892f72dd..65da56be8 100644 --- a/internal_client.go +++ b/internal_client.go @@ -18,6 +18,7 @@ import ( "time" fbcontext "github.com/molecula/featurebase/v3/context" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/hashicorp/go-retryablehttp" "github.com/molecula/featurebase/v3/authn" @@ -856,7 +857,7 @@ func (c *InternalClient) importHelper(ctx context.Context, req Message, process // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { +func (c *InternalClient) Import(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() @@ -884,7 +885,7 @@ func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportReques // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error { +func (c *InternalClient) ImportValue(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() if req.ColumnKeys != nil { @@ -2175,6 +2176,32 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui return resp.Body, nil } +func (c *InternalClient) RestoreShard(ctx context.Context, index string, shard uint64, body io.Reader) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RestoreShard") + defer span.Finish() + + url := fmt.Sprintf("%s%s/internal/restore/%s/%d", c.defaultURI, c.prefix(), index, shard) + req, err := http.NewRequest("POST", url, body) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + + token, ok := authn.GetAccessToken(ctx) + if ok && token != "" { + req.Header.Set("Authorization", token) + } + req.Header.Set("User-Agent", "pilosa/"+Version) + + // Execute request. + resp, err := c.executeRequest(req) + if resp != nil { + resp.Body.Close() + } + return err +} + // IDAllocDataReader returns a reader that provides a snapshot of ID allocation data. func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.IDAllocDataReader") diff --git a/internal_client_test.go b/internal_client_test.go index 9aef3841e..3b2f065a8 100644 --- a/internal_client_test.go +++ b/internal_client_test.go @@ -24,6 +24,7 @@ import ( "github.com/molecula/featurebase/v3/vprint" "github.com/pkg/errors" "github.com/ricochet2200/go-disk-usage/du" + "github.com/stretchr/testify/require" ) // Test distributed TopN Row count across 3 nodes. @@ -484,15 +485,12 @@ func TestClient_Import(t *testing.T) { // do a clear. also, do the clear with a Qcx. func() { // inner function so the deferred abort isn't delayed a lot - qcx := api.Txf().NewQcx() - defer qcx.Abort() + qcx := mustIndexQueryContext(t, api, req.Index, req.Shard) if err := c.Import(context.Background(), qcx, req.Clone(), &pilosa.ImportOptions{Clear: true}); err != nil { t.Fatalf("%s/%s: %v", indexName, fieldName, err) } - if err := qcx.Finish(); err != nil { - t.Fatalf("committing write: %v", err) - } + require.Nil(t, qcx.Commit()) }() // Now do a query to see whether it worked... if fieldName == "keyedf" { @@ -1532,18 +1530,6 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring } } -// verify that serverInfo has Backend -func TestClient_ServerInfoHasBackend(t *testing.T) { - //srcs := []string{"roaring", "rbf", "lmdb"} - cluster := test.MustRunCluster(t, 1) - defer cluster.Close() - cmd := cluster.GetNode(0) - si := cmd.API.Info() - if si.StorageBackend == "" { - panic("should have gotten a StorageBackend back") - } - pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid -} func TestClient_ImportRoaringExists(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() diff --git a/pql/ast.go b/pql/ast.go index d60c2d7ec..35399babe 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -317,7 +317,7 @@ func (q *Query) WriteCallN() int { var n int for _, call := range q.Calls { switch call.Name { - case "Set", "Clear", "ClearRow", "Store", "SetBit": + case "Set", "Clear", "ClearRow", "Store", "SetBit", "Delete": n++ } } diff --git a/rbf.go b/rbf.go deleted file mode 100644 index 19cc647ac..000000000 --- a/rbf.go +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "fmt" - "io" - "math" - "os" - "strings" - "sync" - - "github.com/molecula/featurebase/v3/rbf" - rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" - "github.com/molecula/featurebase/v3/roaring" - txkey "github.com/molecula/featurebase/v3/short_txkey" - "github.com/molecula/featurebase/v3/storage" - - "github.com/pkg/errors" -) - -// RbfDBWrapper wraps an *rbf.DB -type RbfDBWrapper struct { - path string - db *rbf.DB - cfg *rbfcfg.Config - reg *rbfDBRegistrar - muDb sync.Mutex - - openTx map[*RBFTx]bool - - // make Close() idempotent, avoiding panic on double Close() - closed bool - - //DeleteEmptyContainer bool // needed for roaring compat? - - doAllocZero bool -} - -func (w *RbfDBWrapper) Path() string { - return w.path -} - -func (w *RbfDBWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -func (w *RbfDBWrapper) CleanupTx(tx Tx) { - r := tx.(*RBFTx) - r.mu.Lock() - if r.done { - r.mu.Unlock() - return - } - r.done = true - r.mu.Unlock() - - // try not to hold r.mu while locking w.muDb - w.muDb.Lock() - - delete(w.openTx, r) - - w.muDb.Unlock() -} - -// rbfDBRegistrar also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -type rbfDBRegistrar struct { - mu sync.Mutex - mp map[*RbfDBWrapper]bool - - path2db map[string]*RbfDBWrapper - - rbfConfig *rbfcfg.Config -} - -func (r *rbfDBRegistrar) SetRBFConfig(cfg *rbfcfg.Config) { - r.mu.Lock() - defer r.mu.Unlock() - r.rbfConfig = cfg -} - -func (r *rbfDBRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalRbfDBReg *rbfDBRegistrar = newRbfDBRegistrar() - -func newRbfDBRegistrar() *rbfDBRegistrar { - return &rbfDBRegistrar{ - mp: make(map[*RbfDBWrapper]bool), - path2db: make(map[string]*RbfDBWrapper), - } -} - -// register each rbf.DB created, so we dedup and can -// can clean them up. This is called by OpenDBWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *rbfDBRegistrar) unprotectedRegister(w *RbfDBWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -// OpenDBWrapper opens the database in the path directory -// without deleting any prior content. Any -// database directory will have the "-rbf" suffix. -// -// OpenDBWrapper will check the registry and make a new instance only -// if one does not exist for its path. Otherwise it returns -// the existing instance. This insures only one RbfDBWrapper -// per bpath in this pilosa node. -func (r *rbfDBRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) { - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - // creates the effect of having only one DB open per pilosa node. - return w, nil - } - if r.rbfConfig == nil { - r.rbfConfig = rbfcfg.NewDefaultConfig() - r.rbfConfig.DoAllocZero = doAllocZero - r.rbfConfig.FsyncEnabled = cfg.FsyncEnabled - } - db := rbf.NewDB(path, r.rbfConfig) - - w = &RbfDBWrapper{ - reg: r, - path: path, - db: db, - doAllocZero: doAllocZero, - openTx: make(map[*RBFTx]bool), - cfg: r.rbfConfig, - } - r.unprotectedRegister(w) - - err := db.Open() - if err != nil { - panic(fmt.Sprintf("cannot open rbfDB at path '%v': '%v'", path, err)) - } - return w, nil -} - -type RBFTx struct { - // initialIndex is only a debugging aid. Transactions - // can cross indexes. It can be left empty without consequence. - initialIndex string - tx *rbf.Tx - o Txo - Db *RbfDBWrapper - - done bool - mu sync.Mutex // protect done as it changes state -} - -func (tx *RBFTx) DBPath() string { - return tx.tx.DBPath() -} - -func (tx *RBFTx) Type() string { - return RBFTxn -} - -func (tx *RBFTx) Rollback() { - tx.tx.Rollback() - tx.Db.CleanupTx(tx) -} - -func (tx *RBFTx) Commit() (err error) { - err = tx.tx.Commit() - tx.Db.CleanupTx(tx) - return err -} - -func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - return tx.tx.RoaringBitmap(rbfName(index, field, view, shard)) -} - -func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) { - return tx.tx.Container(rbfName(index, field, view, shard), key) -} - -func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error { - return tx.tx.PutContainer(rbfName(index, field, view, shard), key, c) -} - -func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - return tx.tx.RemoveContainer(rbfName(index, field, view, shard), key) -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *RBFTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, true, a...) -} - -// sortedParanoia is a flag to enable a check for unsorted inputs to addOrRemove, -// which is expensive in practice and only really useful occasionally. -const sortedParanoia = false - -func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { - if len(a) == 0 { - return 0, nil - } - name := rbfName(index, field, view, shard) - // this special case can/should possibly go away, except that it - // turns out to be by far the most common case, and we need to know - // there's at least two items to simplify the check-sorted thing. - if len(a) == 1 { - hi, lo := highbits(a[0]), lowbits(a[0]) - rc, err := tx.tx.Container(name, hi) - if err != nil { - return 0, errors.Wrap(err, "failed to retrieve container") - } - if remove { - if rc.N() == 0 { - return 0, nil - } - rc1, chng := rc.Remove(lo) - if !chng { - return 0, nil - } - if rc1.N() == 0 { - err = tx.tx.RemoveContainer(name, hi) - } else { - err = tx.tx.PutContainer(name, hi, rc1) - } - if err != nil { - return 0, err - } - return 1, nil - } else { - rc2, chng := rc.Add(lo) - if !chng { - return 0, nil - } - err = tx.tx.PutContainer(name, hi, rc2) - if err != nil { - return 0, err - } - return 1, nil - } - } - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - // we can accept sorted either ascending or descending. - sign := a[1] - a[0] - prev := a[0] - sign - sign >>= 63 - for i, v := range a { - // This check is noticably expensive (a few percent in some - // use cases) and as long as it passes occasionally it's probably - // not important to run it all the time, and anyway panic is - // not a good choice outside of testing. - if sortedParanoia { - if (v-prev)>>63 != sign { - explain := fmt.Sprintf("addOrRemove: %d < %d != %d < %d", v, prev, a[1], a[0]) - panic(explain) - } - if v == prev { - explain := fmt.Sprintf("addOrRemove: %d twice", v) - panic(explain) - } - } - prev = v - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.tx.RemoveContainer(name, lastHi) - if err != nil { - return 0, errors.Wrap(err, "failed to remove container") - } - } else { - err = tx.tx.PutContainer(name, lastHi, rc) - if err != nil { - return 0, errors.Wrap(err, "failed to put container") - } - } - } - // get the next container - rc, err = tx.tx.Container(name, hi) - if err != nil { - return 0, errors.Wrap(err, "failed to retrieve container") - } - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.tx.RemoveContainer(name, hi) - if err != nil { - return 0, errors.Wrap(err, "failed to remove container") - } - } else { - err = tx.tx.PutContainer(name, hi, rc) - if err != nil { - return 0, errors.Wrap(err, "failed to put container") - } - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.tx.PutContainer(name, hi, rc) - if err != nil { - return 0, errors.Wrap(err, "failed to put container") - } - } - return -} - -func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { - return tx.tx.Contains(rbfName(index, field, view, shard), v) -} - -func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { - return tx.tx.ContainerIterator(rbfName(index, field, view, shard), key) -} - -func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) { - return tx.tx.Count(rbfName(index, field, view, shard)) -} - -func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) { - return tx.tx.Max(rbfName(index, field, view, shard)) -} - -func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - return tx.tx.Min(rbfName(index, field, view, shard)) -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - return tx.tx.CountRange(rbfName(index, field, view, shard), start, end) -} - -func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) { - return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end) -} - -func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize) -} - -func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - err = tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter) - return errors.Wrap(err, fmt.Sprintf("applying filter for index %s, field %s, view %s, shard %d", index, field, view, shard)) -} - -func (tx *RBFTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) { - err = tx.tx.ApplyRewriter(rbfName(index, field, view, shard), ckey, filter) - return errors.Wrap(err, fmt.Sprintf("applying rewriter for index %s, field %s, view %s, shard %d", index, field, view, shard)) -} - -func (tx *RBFTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { - return tx.tx.GetSortedFieldViewList() -} - -func (tx *RBFTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return tx.tx.GetSizeBytesWithPrefix(string(txkey.FieldPrefix(index, field))) -} - -// SnapshotReader returns a reader that provides a snapshot of the current database. -func (tx *RBFTx) SnapshotReader() (io.Reader, error) { - return tx.tx.SnapshotReader() -} - -// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. -func rbfName(index, field, view string, shard uint64) string { - return string(txkey.Prefix(index, field, view, shard)) -} - -// rbfFieldPrefix returns a prefix for field keys in RBF. -func rbfFieldPrefix(index, field string) string { - //return fmt.Sprintf("%s\x00%s\x00", index, field) - return string(txkey.FieldPrefix(index, field)) -} - -func (w *RbfDBWrapper) HasData() (has bool, err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - return w.db.HasData(false) // false => any prior attempt at write means we "have data" -} - -func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error { - w.muDb.Lock() - defer w.muDb.Unlock() - - if err := os.RemoveAll(fieldPath); err != nil { - return errors.Wrap(err, "removing directory") - } - - tx, err := w.db.Begin(true) - if err != nil { - return err - } - defer tx.Rollback() - - if err := tx.DeleteBitmapsWithPrefix(rbfFieldPrefix(index, field)); err != nil { - return err - } - return tx.Commit() -} - -func (w *RbfDBWrapper) DeleteIndex(indexName string) error { - - if strings.Contains(indexName, "'") { - return fmt.Errorf("error: bad indexName `%v` in RbfDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - - w.muDb.Lock() - defer w.muDb.Unlock() - - tx, err := w.db.Begin(true) - if err != nil { - return err - } - defer tx.Rollback() - - if err := tx.DeleteBitmapsWithPrefix(string(prefix)); err != nil { - return err - } - return tx.Commit() -} - -func (w *RbfDBWrapper) Close() error { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - w.reg.unregister(w) - w.closed = true - } - return w.db.Close() -} - -// needed to handle the special case on reload, the close method unregisters the wrapper and all that is -// required is the backing file get reloaded - -func (w *RbfDBWrapper) CloseDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - w.closed = true - return w.db.Close() -} -func (w *RbfDBWrapper) OpenDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - err := w.db.Open() - if err != nil { - return err - } - w.closed = false - return nil -} - -func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) { - tx, err := w.db.Begin(write) - if err != nil { - return nil, err - } - - rtx := &RBFTx{ - tx: tx, - initialIndex: initialIndex, - o: o, - Db: w, - } - - w.muDb.Lock() - w.openTx[rtx] = true - w.muDb.Unlock() - - return rtx, nil -} - -func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - tx, err := w.db.Begin(true) - if err != nil { - return err - } - defer tx.Rollback() - - err = tx.DeleteBitmapsWithPrefix(rbfName(index, field, view, shard)) - if err != nil { - return err - } - return tx.Commit() -} - -func (w *RbfDBWrapper) OpenListString() (r string) { - return "rbf OpenListString not implemented yet" -} diff --git a/rbf/ingest_test.go b/rbf/ingest_test.go index d9b7be2e6..4ea25cf48 100644 --- a/rbf/ingest_test.go +++ b/rbf/ingest_test.go @@ -16,12 +16,15 @@ import ( "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/testhook" - txkey "github.com/molecula/featurebase/v3/short_txkey" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck ) +// This is intended to be similar to the actual field/view keys we +// generate over in querycontext for fragment keys, but it actually +// doesn't matter, as long as it's some kind of a string that reflects +// both field and view. func rbfName(index, field, view string, shard uint64) string { - return string(txkey.Prefix(index, field, view, shard)) + return fmt.Sprintf("~%s;%s<", field, view) } var _ = rbfName // keep linter happy diff --git a/rbf/tx.go b/rbf/tx.go index 2795d74a0..5094712e4 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -11,13 +11,10 @@ import ( "github.com/benbjohnson/immutable" "github.com/molecula/featurebase/v3/roaring" - txkey "github.com/molecula/featurebase/v3/short_txkey" "github.com/molecula/featurebase/v3/vprint" "github.com/pkg/errors" ) -var _ = txkey.ToString - // Tx represents an RBF transaction. Transactions provide guarantees such as // atomicity for all writes that occur as well as serializable isolation. // Transactions can be obtained by calling DB.Begin() and provide a snapshot @@ -2247,21 +2244,6 @@ func (tx *Tx) PageData(pgno uint32) ([]byte, error) { return buf, err } -func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) { - records, err := tx.RootRecords() - if err != nil { - return nil, err - } - it := records.Iterator() - for !it.Done() { - k, _, _ := it.Next() - root := k - fv := txkey.FieldViewFromPrefix([]byte(root)) - fvs = append(fvs, fv) - } - return -} - func (tx *Tx) DebugInfo() *TxDebugInfo { return &TxDebugInfo{ Ptr: fmt.Sprintf("%p", tx), diff --git a/rbf/util.go b/rbf/util.go index 4b29b2a2e..dc6df1405 100644 --- a/rbf/util.go +++ b/rbf/util.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - txkey "github.com/molecula/featurebase/v3/short_txkey" "github.com/molecula/featurebase/v3/vprint" ) @@ -52,7 +51,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error { rootRecords, err := readRootRecords(page) vprint.PanicOn(err) for k, rr := range rootRecords { - fmt.Printf(" [%02v] Name:'%v' pgno:%v\n", k, prefixToString(rr.Name), rr.Pgno) + fmt.Printf(" [%02v] Name:'%v' pgno:%v\n", k, PrefixToString(rr.Name), rr.Pgno) } case *LeafPageInfo: @@ -61,7 +60,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error { } fmt.Printf("Pgno:%-8d ", pgno) fmt.Printf("%-10s ", "leaf") - fmt.Printf("%-54q ", prefixToString(info.Tree)) + fmt.Printf("%-54q ", PrefixToString(info.Tree)) fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) page, _, err := tx.readPage(uint32(pgno)) @@ -76,7 +75,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error { case *BranchPageInfo: fmt.Printf("Pgno:%-8d ", pgno) fmt.Printf("%-10s ", "branch") - fmt.Printf("%-54q ", prefixToString(info.Tree)) + fmt.Printf("%-54q ", PrefixToString(info.Tree)) fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) page, _, err := tx.readPage(uint32(pgno)) @@ -90,7 +89,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error { case *BitmapPageInfo: fmt.Printf("Pgno:%-8d ", pgno) fmt.Printf("%-10s ", "bitmap") - fmt.Printf("%-54q ", prefixToString(info.Tree)) + fmt.Printf("%-54q ", PrefixToString(info.Tree)) fmt.Printf("-\n") case *FreePageInfo: @@ -195,13 +194,19 @@ func printFreePage(page *FreePage) { fmt.Printf("Type: free\n") } -func prefixToString(s string) (ret string) { - defer func() { - if err := recover(); err != nil { - ret = s - } - }() - return txkey.PrefixToString([]byte(s)) +// PrefixToString converts a fragment key (used to denote +// a root bitmap in an RBF file) into a description of it +// suitable for printing. This behavior reflects the +// historical practice of the short_txkey package, which +// we no longer have. It's exported because the rbf_pages +// command wants to use it to display things. +func PrefixToString(s string) (ret string) { + var field, view string + n, err := fmt.Sscanf(s, "~%s;%s<", &field, &view) + if err != nil || n != 2 { + return s + } + return fmt.Sprintf("field:%s;view:%s", field, view) } ///////////////// happy linter @@ -212,4 +217,3 @@ var _ = printLeafPage var _ = printBranchPage var _ = printBitmapPage var _ = printFreePage -var _ = prefixToString diff --git a/roaring/filter.go b/roaring/filter.go index 11f16670d..672cbb80b 100644 --- a/roaring/filter.go +++ b/roaring/filter.go @@ -1297,6 +1297,27 @@ func getFirstRowAsContainers(citer ContainerIterator) []containerWithKey { return citerContainers } +type infiniteOnesIterator struct { + count uint64 + c *Container +} + +func NewInfiniteOnesIterator() *infiniteOnesIterator { + return &infiniteOnesIterator{c: NewContainerRun([]Interval16{{Start: 0, Last: 65535}})} +} + +func (i *infiniteOnesIterator) Next() bool { + i.count++ + return true +} + +func (i *infiniteOnesIterator) Value() (uint64, *Container) { + return i.count, i.c +} + +func (i *infiniteOnesIterator) Close() { +} + // NewClearAndSetRewriter instantiates a ClearAndSetRewriter func NewClearAndSetRewriter(clear, set ContainerIterator) (*ClearAndSetRewriter, error) { curSetKey, curSet := getNextFromIterator(set) diff --git a/roaring/roaring.go b/roaring/roaring.go index 431752f3b..c96b90e57 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2265,33 +2265,6 @@ func (r *baseRoaringIterator) Current() (key uint64, cType byte, n int, length i return r.currentKey, r.currentType, r.currentN, r.currentLen, r.currentPointer, r.lastErr } -// SanityCheckMapping is a debugging function which checks whether containers -// are *correctly* recorded as mapped or unmapped. -func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut int64, unmappedIn int64, errs int, err error) { - b.Containers.UpdateEvery(func(key uint64, c *Container, existed bool) (*Container, bool) { - dptr := uintptr(unsafe.Pointer(c.pointer)) - if dptr >= from && dptr < to { - if c.Mapped() { - mappedIn++ - } else { - err = fmt.Errorf("container key %d, addr %x, inside %x+%d", - key, dptr, from, to-from) - errs++ - unmappedIn++ - } - } else { - if c.Mapped() { - err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped", - key, dptr, from, to-from) - errs++ - mappedOut++ - } - } - return c, false - }) - return mappedIn, mappedOut, unmappedIn, errs, err -} - // RemapRoaringStorage tries to update all containers to refer to // the roaring bitmap in the provided []byte. If any containers are // marked as mapped, but do not match the provided storage, they will diff --git a/server.go b/server.go index 1e7f8cbee..862d3c530 100644 --- a/server.go +++ b/server.go @@ -18,6 +18,7 @@ import ( daxstorage "github.com/molecula/featurebase/v3/dax/storage" "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" @@ -540,7 +541,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { if err != nil { return nil, err } - s.holder = NewHolder(path, s.holderConfig) + s.holder, err = NewHolder(path, s.holderConfig) + if err != nil { + return nil, err + } s.holder.Stats.SetLogger(s.logger) cwd, err := os.Getwd() if err != nil { @@ -914,14 +918,17 @@ func (s *Server) ViewsRemoval(ctx context.Context) { timeSince := time.Since(viewTime) if timeSince >= field.Options().TTL { - for _, shard := range field.AvailableShards(true).Slice() { - err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil) - if err != nil { - s.logger.Errorf("view: %s, shard: %d, ttl delete fragment: %s", shard, viewName, err) - } + shardUints := field.AvailableShards(true).Slice() + shards := make([]keys.Shard, len(shardUints)) + for i, v := range shardUints { + shards[i] = keys.Shard(v) + } + err := s.holder.txStore.DeleteFragments(keys.Index(index.Name()), keys.Field(field.Name()), []keys.View{keys.View(view.name)}, shards) + if err != nil { + s.logger.Errorf("view: %s ttl delete fragment: %s", viewName, err) } - err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) + err = s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) if err != nil { s.logger.Errorf("view: %s, ttl delete view: %s", viewName, err) } @@ -931,15 +938,17 @@ func (s *Server) ViewsRemoval(ctx context.Context) { } } if field.Options().NoStandardView && field.view(viewStandard) != nil { - // delete view "standard" if NoStandardView is true and view "standard" exists - for _, shard := range field.AvailableShards(true).Slice() { - err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil) - if err != nil { - s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err) - } + shardUints := field.AvailableShards(true).Slice() + shards := make([]keys.Shard, len(shardUints)) + for i, v := range shardUints { + shards[i] = keys.Shard(v) + } + err := s.holder.txStore.DeleteFragments(keys.Index(index.Name()), keys.Field(field.Name()), []keys.View{keys.View(viewStandard)}, shards) + if err != nil { + s.logger.Errorf("view: %s ttl delete fragment: %s", viewStandard, err) } - err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) + err = s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard) if err != nil { s.logger.Errorf("view: %s, delete view: %s", viewStandard, err) } diff --git a/server/grpc.go b/server/grpc.go index d0c0532aa..0ece4ace9 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -736,8 +736,11 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe return errToStatusError(err) } - qcx := h.api.Holder().Txf().NewQcx() - defer qcx.Abort() + qcx, err := h.api.NewQueryContext(context.TODO()) + if err != nil { + return err + } + defer qcx.Release() var fields []*pilosa.Field for _, field := range index.Fields() { diff --git a/server/handler_test.go b/server/handler_test.go index 399750165..f9333083d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -23,6 +23,7 @@ import ( pb "github.com/molecula/featurebase/v3/proto" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" + "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -175,31 +176,28 @@ func TestHandler_Endpoints(t *testing.T) { }) i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - tx0 := holder.Txf().NewWritableQcx() - defer tx0.Abort() + qcx, err := hldr.NewIndexQueryContext(context.Background(), "i0") + require.Nil(t, err) + defer qcx.Release() if f, err := i0.CreateFieldIfNotExists("f1", "", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } if _, err := i0.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if err := tx0.Finish(); err != nil { - t.Fatal(err) - } + require.Nil(t, qcx.Commit()) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - tx1 := holder.Txf().NewWritableQcx() - defer tx1.Abort() + qcx, err = hldr.NewIndexQueryContext(context.Background(), "i1") + require.Nil(t, err) if f, err := i1.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx1, 0, 0, nil); err != nil { - t.Fatal(err) - } - if err := tx1.Finish(); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } + require.Nil(t, qcx.Commit()) t.Run("Schema", func(t *testing.T) { w := httptest.NewRecorder() @@ -255,11 +253,11 @@ func TestHandler_Endpoints(t *testing.T) { // i2 is for SchemaDetails i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{}) - tx2 := holder.Txf().NewWritableQcx() - defer tx2.Abort() + qcx, err = holder.NewIndexQueryContext(context.Background(), "i2") + require.Nil(t, err) if f, err := i2.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } @@ -269,7 +267,7 @@ func TestHandler_Endpoints(t *testing.T) { } for n := 0; n < 4; n++ { - if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil { + if _, err := f.SetValue(qcx, uint64(n), int64(n)); err != nil { t.Fatal(err) } } @@ -280,30 +278,27 @@ func TestHandler_Endpoints(t *testing.T) { } for n := 0; n < 5; n++ { - if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil { + if _, err := f.SetValue(qcx, uint64(n), int64(n)); err != nil { t.Fatal(err) } } if f, err := i2.CreateFieldIfNotExists("f3", "", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } if f, err := i2.CreateFieldIfNotExists("f4", "", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } if f, err := i2.CreateFieldIfNotExists("f5", "", pilosa.OptFieldTypeBool()); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { - t.Fatal(err) - } - - if err := tx2.Finish(); err != nil { + } else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil { t.Fatal(err) } + require.Nil(t, qcx.Commit()) t.Run("SchemaDetails", func(t *testing.T) { w := httptest.NewRecorder() @@ -493,7 +488,6 @@ func TestHandler_Endpoints(t *testing.T) { msg := pilosa.ImportRoaringRequest{ Action: pilosa.RequestActionOverwrite, - Block: 0, Views: map[string][]byte{ "bsig_int-field": roaringData, }, diff --git a/server/server_test.go b/server/server_test.go index 749130600..068dc0832 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -43,6 +43,7 @@ func TestMain_Set_Quick(t *testing.T) { } for i := 0; i < 10; i++ { + i := i // make local copy of i, loop variable capture t.Run(fmt.Sprint(i), func(t *testing.T) { t.Parallel() @@ -677,13 +678,13 @@ func TestMain_ImportTimestamp(t *testing.T) { } // Import data. - qcx := m.API.Txf().NewQcx() + ctx := context.Background() + qcx, err := m.API.NewIndexQueryContext(ctx, data.Index) + require.Nil(t, err) if err := m.API.Import(context.Background(), qcx, &data); err != nil { /// first write i/0 here. 2nd write here. t.Fatal(err) } - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } + require.Nil(t, qcx.Commit()) // Ensure the correct views were created. dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName) files, err := os.ReadDir(dir) @@ -732,13 +733,13 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } // Import data. - qcx := m.API.Txf().NewQcx() + ctx := context.Background() + qcx, err := m.API.NewIndexQueryContext(ctx, data.Index) + require.Nil(t, err) if err := m.API.Import(context.Background(), qcx, &data); err != nil { t.Fatal(err) } - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } + require.Nil(t, qcx.Commit()) // Ensure the correct views were created. dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName) diff --git a/short_txkey/txkey.go b/short_txkey/txkey.go deleted file mode 100644 index 66c4c072d..000000000 --- a/short_txkey/txkey.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -// Package txkey consolidates in one place the use of keys to index into our -// various storage/txn back-ends. The short_txkey version omits the -// index and shard, since these are implicitly part of our database-per-shard -// in an index scheme. In other words, every database is only in exactly -// one shard of one index already. There is no need to repeat the index -// and shard in these keys. -package short_txkey - -import ( - "encoding/binary" - "fmt" -) - -// FieldView is here to avoid circular import. -type FieldView struct { - Field string - View string -} - -func FieldViewFromPrefix(prefix []byte) FieldView { - field, view := SplitPrefix(prefix) - return FieldView{Field: field, View: view} -} - -func FieldViewFromFullKey(fullKey []byte) FieldView { - field, view, _ := Split(fullKey) - return FieldView{Field: field, View: view} -} - -// Key produces the bytes that we use as a key to query the storage/tx engine. -// The roaringContainerKey argument to Key() is a container key into a roaring Container. -// The return value from Key() is constructed as follows: -// -// ~field;view'. Keys always end with '#'. -// Keys always contain exactly one each of ';' and '<', in that order. -// The field is between the '~' and the ';'. It must be at least 1 byte long. -// The view is between the ';' and the '<'. It must be at least 1 byte long. -// The ckey is the 8 bytes between the '<' and the '#'. -// The Prefix of a key ends at, and includes, the '<'. It is at least 13 bytes long. -// The index, field, and view are not allowed to contain these reserved bytes: -// -// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} -// -// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable -// base-64 encoding. -// -// The shortest possible key is 14 bytes. It would be laid out like this: -// -// ~f;v<12345678# -// 12345678901234 -// -// keys starting with '~' are regular value keys. -// keys starting with '>' are symlink keys. -// -// NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) { - - prefix := Prefix(index, field, view, shard) - - var ckey [9]byte - binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey) - ckey[8] = byte('#') - return append(prefix, ckey[:]...) -} - -// KeyAndPrefix returns the equivalent of Key() and Prefix() calls. -func KeyAndPrefix(index, field, view string, shard, roaringContainerKey uint64) (key, prefix []byte) { - prefix = Prefix(index, field, view, shard) - - var ckey [9]byte - binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey) - ckey[8] = byte('#') - key = append(prefix, ckey[:]...) - return -} - -var _ = KeyAndPrefix // keep linter happy - -func MustValidateKey(bkey []byte) { - n := len(bkey) - if n < 14 { - panic(fmt.Sprintf("bkey too short, must have at least 14 bytes: '%v'", string(bkey))) - } - typ := bkey[0] - if typ != '~' && typ != '>' { - panic(fmt.Sprintf("bkey did not start with '~' for value nor '>' for symlink: '%v'", string(bkey))) - } - if bkey[n-10] != '<' { - panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey))) - } - if bkey[n-1] != '#' { - panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey))) - } -} - -// KeyExtractContainerKey extracts the containerKey from bkey. -// key example: field;view [1, 2, 3, 65536, 65539] - // (0, 65536, 131072) => [0, 3] - // (65536, 65536, 131072) => [65536, 65539] - // (262144, 65536, 131072) => [262144, 262147] - // - OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) - - // ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator. - // - // See the roaring package for details of the RoaringIterator. - // - // If clear is true, the bits from rit are cleared, otherwise they are set in the - // specifed fragment. - // - // ImportRoaringBits return values changed and rowSet may be inaccurate if - // the data []byte is supplied (the RoaringTx implementation neglects this for speed). - ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) - - // GetSortedFieldViewList gets the set of FieldView(s) - GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) - - GetFieldSizeBytes(index, field string) (uint64, error) -} - -// GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, -// as a convenience if a Tx backend hasn't implemented this new function yet. -func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - iter, _, err := tx.ContainerIterator(index, field, view, shard, ckey) - if err != nil { - return err - } - // ApplyFilterToIterator closes the iterator for us. - return roaring.ApplyFilterToIterator(filter, iter) -} diff --git a/tx_internal_test.go b/tx_internal_test.go deleted file mode 100644 index 72c29ba42..000000000 --- a/tx_internal_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "bytes" - "sync" - "testing" - - "github.com/molecula/featurebase/v3/roaring" -) - -const countRangeMaxN = 8192 - -var countRangeSampleData []byte -var prepareCountRangeSampleData sync.Once - -// The sample data for the counter is just a series of containers, -// each with cardinality equal to its container key. -func requireCountRangeSampleData(tb testing.TB) (*fragment, Tx) { - prepareCountRangeSampleData.Do(func() { - var arraySample [4096]uint16 - // This horrible hack relies on a quirk of roaring's internals: It'll - // copy the bitmap if its length isn't exactly 1024. This lets us - // request that each container get its own copy of the bitmap. - var bitmapSample [1025]uint64 - for i := range arraySample { - arraySample[i] = uint16(i * 2) - } - // Put corresponding bits in the bitmap... - for i := 0; i < 4096/32; i++ { - // bit 0 is 0x1, bit 2 is 0x4, so even-numbered bits - // are 0x5555.... - bitmapSample[i] = 0x5555555555555555 - } - bm := roaring.NewSliceBitmap() - for n := 0; n < 4096 && n < countRangeMaxN; n++ { - c := roaring.NewContainerArray(arraySample[:n]) - bm.Put(uint64(n), c) - } - // Start filling in the missing bits. This starts us out with - // bitmap containers, but then eventually converts to things - // that are more likely to be run containers. At the end of this, - // we should have exactly the first 8,192 bits set, for a single - // run of 8k. - for n := 4096; n < 8192; n++ { - c := roaring.NewContainerBitmapN(bitmapSample[:], int32(n)) - bm.Put(uint64(n), c) - w := n - 4096 - bitmapSample[w/32] |= 1 << (((n % 32) * 2) + 1) - } - var asBytes bytes.Buffer - n, err := bm.WriteTo(&asBytes) - if err != nil { - tb.Fatalf("writing bitmap: %v", err) - } - countRangeSampleData = asBytes.Bytes() - tb.Logf("creating bitmap: %d containers, %d bytes of data", countRangeMaxN, n) - }) - f, idx, tx := mustOpenFragment(tb) - // Properly close this transaction, but not the next one we create that the - // caller will be responsible for. The deferred callback will - // be a nop if the Commit happened. - defer tx.Rollback() - err := f.importRoaringT(tx, countRangeSampleData, false) - if err != nil { - tb.Fatalf("importing sample data: %v", err) - } - err = tx.Commit() - if err != nil { - tb.Fatalf("committing sample data: %v", err) - } - tx = idx.holder.txf.NewTx(Txo{Write: false, Index: idx, Fragment: f, Shard: 0}) - return f, tx -} - -func TestTx_CountRange(t *testing.T) { - f, tx := requireCountRangeSampleData(t) - defer f.Clean(t) - defer tx.Rollback() - // CountRange accesses the fragment without locking. Normally we only - // call it from inside a fragment routine with locking. Otherwise, you - // can have a race condition with snapshots, for instance. - f.mu.Lock() - defer f.mu.Unlock() - - expected := uint64(0) - j := uint64(0) - for i := uint64(0); i < countRangeMaxN; i += 7 { - expected += i - if i%4 == 3 { - expected -= (j * 7) + 21 - j += 7 - } - // Every other bit gets set, for a total of i bits in container - // i, so they're all in the first (i*2) bits of the container. - got, err := tx.CountRange("i", "f", "v", 0, uint64(j)<<16, (uint64(i)<<16)+(i*2)) - if err != nil { - t.Fatalf("counting range: %v", err) - } - if got != expected { - t.Fatalf("counting from container %d to %d, expected %d, got %d", - j, i, expected, got) - } - // The -i here undoes the +i at the top of this loop. - expected += (i * 7) + 21 - i - } -} - -func BenchmarkTx_CountRange(b *testing.B) { - f, tx := requireCountRangeSampleData(b) - defer f.Clean(b) - defer tx.Rollback() - - for k := 0; k < b.N; k++ { - expected := uint64(0) - j := uint64(0) - for i := uint64(0); i < countRangeMaxN; i += 7 { - if i%4 == 3 { - expected -= (j * 7) + 21 - j += 7 - } - got, err := tx.CountRange("i", "f", "v", 0, uint64(j)<<16, uint64(i)<<16) - if err != nil { - b.Fatalf("counting range: %v", err) - } - if got != expected { - b.Fatalf("counting from container %d to %d, expected %d, got %d", - j, i, expected, got) - } - expected += (i * 7) + 21 - } - } -} diff --git a/tx_test.go b/tx_test.go deleted file mode 100644 index fe79abe5f..000000000 --- a/tx_test.go +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa_test - -import ( - "context" - "fmt" - "testing" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/test" -) - -func queryIRABit(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowID uint64, index string) (bit bool) { - query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) // acctOwnerID) - res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - if err != nil { - t.Fatalf("querying IRA bit: %v", err) - } - cols := res.Results[0].(*pilosa.Row).Columns() - for i := range cols { - if cols[i] == acctOwnerID { - return true - } - } - return false -} - -func mustQueryAcct(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) { - query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID) - res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - if err != nil { - t.Fatalf("querying account: %v", err) - } - - if len(res.Results) == 0 { - return 0 - } - valCount := res.Results[0].(pilosa.ValCount) - return valCount.Val -} - -func queryBalances(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, index string) (acct0bal, acct1bal int64) { - - acct0bal = mustQueryAcct(t, m0api, acctOwnerID, fldAcct0, index) - acct1bal = mustQueryAcct(t, m0api, acctOwnerID, fldAcct1, index) - return -} - -func TestAPI_ImportAtomicRecord(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - - m0 := c.GetNode(0) - m0api := m0.API - - ctx := context.Background() - index := c.Idx() - - fieldAcct0 := "acct0" - fieldAcct1 := "acct1" - - transferUSD := int64(100) - _ = transferUSD - opts := pilosa.OptFieldTypeInt(-1000, 1000) - - _, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - _, err = m0api.CreateField(ctx, index, fieldAcct0, opts) - if err != nil { - t.Fatalf("creating fieldAcct0: %v", err) - } - _, err = m0api.CreateField(ctx, index, fieldAcct1, opts) - if err != nil { - t.Fatalf("creating fieldAcct1: %v", err) - } - - iraField := "ira" // set field. - iraRowID := uint64(3) - _, err = m0api.CreateField(ctx, index, iraField) - if err != nil { - t.Fatalf("creating fieldIRA: %v", err) - } - - acctOwnerID := uint64(78) // ColumnID - shard := acctOwnerID / ShardWidth - - // setup 500 USD in acct1 and 700 USD in acct2. - // transfer 100 USD. - // should see 400 USD in acct, and 800 USD in acct2. - // - - // setup initial balances - - createAIRUpdate := func(acct0bal, acct1bal int64) (air *pilosa.AtomicRecord) { - ivr0 := &pilosa.ImportValueRequest{ - Index: index, - Field: fieldAcct0, - Shard: shard, - ColumnIDs: []uint64{acctOwnerID}, - Values: []int64{acct0bal}, - } - ivr1 := &pilosa.ImportValueRequest{ - Index: index, - Field: fieldAcct1, - Shard: shard, - ColumnIDs: []uint64{acctOwnerID}, - Values: []int64{acct1bal}, - } - - ir0 := &pilosa.ImportRequest{ - Index: index, - Field: iraField, - Shard: shard, - ColumnIDs: []uint64{acctOwnerID}, - RowIDs: []uint64{iraRowID}, - } - - air = &pilosa.AtomicRecord{ - Index: index, - Shard: shard, - Ivr: []*pilosa.ImportValueRequest{ - ivr0, ivr1, - }, - Ir: []*pilosa.ImportRequest{ir0}, - } - return - } - - expectedBalStartingAcct0 := int64(500) - expectedBalStartingAcct1 := int64(700) - - air := createAIRUpdate(expectedBalStartingAcct0, expectedBalStartingAcct1) - - //vv("BEFORE the first ImportAtomicRecord!") - - qcx := m0api.Txf().NewQcx() - if err := m0api.ImportAtomicRecord(ctx, qcx, air); err != nil { - qcx.Abort() - t.Fatal(err) - } - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } - - //vv("AFTER the first ImportAtomicRecord!") - - iraBit := queryIRABit(t, m0api, acctOwnerID, iraField, iraRowID, index) - if !iraBit { - t.Fatal("IRA bit should have been set") - } - - startingBalanceAcct0, startingBalanceAcct1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) - //vv("starting balance: acct0=%v, acct1=%v", startingBalanceAcct0, startingBalanceAcct1) - - if startingBalanceAcct0 != expectedBalStartingAcct0 { - t.Fatalf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, startingBalanceAcct0) - } - if startingBalanceAcct1 != expectedBalStartingAcct1 { - t.Fatalf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, startingBalanceAcct1) - } - - //vv("sad path: transferUSD %v from %v -> %v, with power loss half-way through", transferUSD, fieldAcct0, fieldAcct1) - - opt := func(o *pilosa.ImportOptions) error { - o.SimPowerLossAfter = 1 - return nil - } - expectedBalEndingAcct0 := expectedBalStartingAcct0 - 100 - expectedBalEndingAcct1 := expectedBalStartingAcct1 + 100 - - air = createAIRUpdate(expectedBalEndingAcct0, expectedBalEndingAcct1) - - qcx = m0api.Txf().NewQcx() - //vv("just before the SECOND ImportAtomicRecord, qcx is %p, should NOT BE NIL", qcx) - err = m0api.ImportAtomicRecord(ctx, qcx, air.Clone(), opt) - //err = m0api.ImportAtomicRecord(ctx, nil, air, opt) - if err != pilosa.ErrAborted { - t.Fatalf("expected ErrTxnAborted but got err='%#v'", err) - } - // sad path, cleanup - qcx.Abort() - qcx = nil - - b0, b1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) - //vv("after power failure tx, balance: acct0=%v, acct1=%v", b0, b1) - - if b0 != expectedBalStartingAcct0 { - t.Fatalf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, b0) - } - if b1 != expectedBalStartingAcct1 { - t.Fatalf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, b1) - } - //vv("good: with power loss half-way, no change in account balances; acct0=%v; acct1=%v", b0, b1) - - // next part of the test, just make sure we do the update. - //vv("happy path: transferUSD %v from %v -> %v, with no interruption.", transferUSD, fieldAcct0, fieldAcct1) - - // happy path with no power failure half-way through. - - qcx = m0api.Txf().NewQcx() - err = m0api.ImportAtomicRecord(ctx, qcx, air.Clone()) - if err != nil { - t.Fatalf("importing record: %v", err) - } - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } - eb0, eb1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) - - // should have been applied this time. - if eb0 != expectedBalEndingAcct0 || - eb1 != expectedBalEndingAcct1 { - t.Fatalf("problem: transaction did not get committed/applied. transferUSD=%v, but we see: startingBalanceAcct0=%v -> endingBalanceAcct0=%v; startingBalanceAcct1=%v -> endingBalanceAcct1=%v", transferUSD, startingBalanceAcct0, eb0, startingBalanceAcct1, eb1) - } - //vv("ending balance: acct0=%v, acct1=%v", eb0, eb1) - - // clear all the bits - air.Ivr[0].Clear = true - air.Ivr[1].Clear = true - air.Ir[0].Clear = true - - qcx = m0api.Txf().NewQcx() - err = m0api.ImportAtomicRecord(ctx, qcx, air) - if err != nil { - t.Fatalf("importing record: %v", err) - } - if err := qcx.Finish(); err != nil { - t.Fatal(err) - } - - eb0, eb1 = queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index) - if eb0 != 0 || - eb1 != 0 { - t.Fatal("problem: bits did not clear") - } - //vv("cleared balances: acct0=%v, acct1=%v", eb0, eb1) - - iraBit = queryIRABit(t, m0api, acctOwnerID, iraField, iraRowID, index) - if iraBit { - t.Fatal("IRA bit should have been cleared") - } -} diff --git a/txfactory.go b/txfactory.go deleted file mode 100644 index 39b4029a9..000000000 --- a/txfactory.go +++ /dev/null @@ -1,702 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "fmt" - "os" - "strings" - "sync" - - "github.com/molecula/featurebase/v3/task" - "github.com/molecula/featurebase/v3/testhook" - "github.com/molecula/featurebase/v3/vprint" - "github.com/pkg/errors" -) - -// public strings that pilosa/server/config.go can reference -const ( - RBFTxn string = "rbf" -) - -// DetectMemAccessPastTx true helps us catch places in api and executor -// where mmapped memory is being accessed after the point in time -// which the transaction has committed or rolled back. Since -// memory segments will be recycled by the underlying databases, -// this can lead to corruption. When DetectMemAccessPastTx is true, -// code in bolt.go will copy the transactionally viewed memory before -// returning it for bitmap reading, and then zero it or overwrite it -// with -2 when the Tx completes. -// -// Should be false for production. -const DetectMemAccessPastTx = false - -var sep = string(os.PathSeparator) - -// Qcx is a (Pilosa) Query Context. -// -// It flexibly expresses the desired grouping of Tx for mass -// rollback at a query's end. It provides one-time commit for -// an atomic import write Tx that involves multiple fragments. -// -// The most common use of Qcx is to call GetTx() to obtain a Tx locally, -// once the index/shard pair is known: -// -// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) { -// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...}) -// defer finisher(&err0) -// ... -// } -// -// Qcx reuses read-only Tx on the same index/shard pair. See -// the Qcx.GetTx() for further discussion. The caveat is of -// course that your "new" read Tx actually has an "old" view -// of the database. -// -// At the moment, most -// writes to individual shards are commited eagerly and locally -// when the `defer finisher(&err0)` is run. -// This is done by returning a finisher that actually Commits, -// thus freeing the one write slot for re-use. A single -// writer is also required by RBF, so this design accomodates -// both. -// -// In contrast, the default read Tx generated (or re-used) will -// return a no-op finisher and the group of reads as a whole -// will be rolled back (mmap memory released) en-mass when -// Qcx.Abort() is called at the top-most level. -// -// Local use of a (Tx, finisher) pair obtained from Qcx.GetTx() -// doesn't need to care about these details. Local use should -// always invoke finisher(&err0) or finisher(nil) to complete -// the Tx within the local function scope. -// -// In summary write Tx are typically "local" -// and are never saved into the TxGroup. The parallelism -// supplied by TxGroup typically applies only to read Tx. -// -// The one exception is this rule is for the one write Tx -// used during the api.ImportAtomicRecord routine. There -// we make a special write Tx and use it for all matching writes. -// This is then committed at the final, top-level, Qcx.Finish() call. -// -// See also the Qcx.GetTx() example and the TxGroup description below. -type Qcx struct { - Grp *TxGroup - Txf *TxFactory - workers *task.Pool - - // if we go back to using Qcx values, this must become a pointer, - // or otherwise be dealt with because copies of Mutex are a no-no. - mu sync.Mutex - - // RequiredForAtomicWriteTx is used by api.ImportAtomicRecord - // to ensure that all writes happen on this one Tx. - RequiredForAtomicWriteTx *Tx - - // efficient access to the options for RequiredForAtomicWriteTx - RequiredTxo *Txo - - isRoaring bool - - // top-level context is for a write, so re-use a - // writable tx for all reads and writes on each given - // shard - write bool - - // don't allow automatic reuse now. Must manually call Reset, or NewQcx(). - done bool -} - -// Finish commits/rollsback all stored Tx. It no longer resets the -// Qcx for further operations automatically. User must call Reset() -// or NewQxc() again. -func (q *Qcx) Finish() (err error) { - q.mu.Lock() - defer q.mu.Unlock() - if q.RequiredForAtomicWriteTx != nil { - if q.RequiredTxo.Write { - err = (*q.RequiredForAtomicWriteTx).Commit() // PanicOn here on 2nd. is this a double commit? - } else { - (*q.RequiredForAtomicWriteTx).Rollback() - } - } - err2 := q.Grp.FinishGroup() - // drop the old group so we aren't holding references to all those Tx - q.Grp = q.Txf.NewTxGroup() - if !q.done { - _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) - } - q.done = true - - if err != nil { - return err - } - return err2 -} - -// Abort rolls back all Tx generated and stored within the Qcx. -// The Qcx is then reset and can be used again immediately. -func (q *Qcx) Abort() { - q.mu.Lock() - defer q.mu.Unlock() - if q.RequiredForAtomicWriteTx != nil { - (*q.RequiredForAtomicWriteTx).Rollback() - } - q.Grp.AbortGroup() - // drop the old group so we aren't holding references to all those Tx - q.Grp = q.Txf.NewTxGroup() - if !q.done { - _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) - } - q.done = true -} - -// Reset forgets everything are starts fresh with an empty -// group, ready for use again as if NewQcx() had been called. -func (q *Qcx) Reset() { - q.mu.Lock() - defer q.mu.Unlock() - q.unprotected_reset() -} - -func (q *Qcx) unprotected_reset() { - q.RequiredForAtomicWriteTx = nil - q.RequiredTxo = nil - q.Grp = q.Txf.NewTxGroup() - q.done = false -} - -// NewQcx allocates a freshly allocated and empty Grp. -// The top-level Qcx is not marked writable. Non-writable -// Qcx should not be used to request write Tx. -func (f *TxFactory) NewQcx() (qcx *Qcx) { - qcx = &Qcx{ - Grp: f.NewTxGroup(), - Txf: f, - } - if f.typeOfTx == "roaring" { - qcx.isRoaring = true - } - if f.holder != nil { - if f.holder.executor != nil { - qcx.workers = f.holder.executor.workers - } - _ = testhook.Opened(f.holder.Auditor, qcx, nil) - } - return -} - -// NewWritableQcx allocates a freshly allocated and empty Grp. -// The resulting Qcx is marked writable. -func (f *TxFactory) NewWritableQcx() (qcx *Qcx) { - qcx = &Qcx{ - Grp: f.NewTxGroup(), - Txf: f, - } - if f.holder != nil && f.holder.executor != nil { - qcx.workers = f.holder.executor.workers - } - if f.typeOfTx == "roaring" { - qcx.isRoaring = true - } - _ = testhook.Opened(f.holder.Auditor, qcx, nil) - qcx.write = true - return -} - -var NoopFinisher = func(perr *error) {} - -var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset before re-use") - -// GetTx is used like this: -// -// someFunc(ctx context.Context, shard uint64) (_ interface{}, err0 error) { -// -// tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) -// defer finisher(&err0) -// -// return e.executeIncludesColumnCallShard(ctx, tx, index, c, shard, col) -// } -// -// Note we are tracking the returned err0 error value of someFunc(). An option instead is to say -// -// defer finisher(nil) -// -// This means always Commit writes, ignoring if there were errors. This style -// is expected to be rare compared to the typical -// -// defer finisher(&err0) -// -// invocation, where err0 is your return from the enclosing function error. -// If the Tx is local and not a part of a group, then the finisher -// consults that error to decides whether to Commit() or Rollback(). -// -// If instead the Tx becomes part of a group, then the local finisher() is -// always a no-op, in deference to the Qcx.Finish() -// or Qcx.Abort() calls. -// -// Take care the finisher(&err) is capturing the address of the -// enclosing function's err and that it has not been shadowed -// locally by another _, err := f() call. For this reason, it can -// be clearer (and much safer) to rename the enclosing functions 'err' to 'err0', -// to make it clear we are referring to the first and final error. -func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { - if qcx.workers != nil { - qcx.workers.Block() - defer qcx.workers.Unblock() - } - qcx.mu.Lock() - defer qcx.mu.Unlock() - - if qcx.done { - return nil, nil, ErrQcxDone - } - - // roaring uses finer grain, a file per fragment rather than - // db per shard. So we can't re-use the readTx. Moreover, - // roaring Tx are No-ops anyway, so just give it a new Tx - // everytime. - if qcx.isRoaring { - return qcx.Txf.NewTx(o), NoopFinisher, nil - } - - // qcx.write reflects the top executor determination - // if a write will be happen at some point, in which case, to avoid - // locking problems with multi-shard things, we (probably incorrectly) - // treat every Tx as its own individual separate Tx. - // - // But we still want to open non-write transactions individually, we - // just can't recycle them (because write operations will come in and - // we want them to work and commit right away so we're not holding a write - // lock for long). - writeLogic := o.Write || qcx.write - - // In general, we make ALL write transactions local, and never reuse them - // below. Previously this was to help lmdb. - // - // *However* there is one exception: when we have set RequiredForAtomicWriteTx - // for the importing of an AtomicRequest, then we must use that - // our single RequiredForAtomicWriteTx for all writes until it - // is cleared. This one is kept separately from the read TxGroup. - // - if o.Write && qcx.RequiredForAtomicWriteTx != nil { - // verify that shard and index match! - ro := qcx.RequiredTxo - if o.Shard != ro.Shard { - vprint.PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard)) - } - if o.Index == nil { - vprint.PanicOn("o.Index annot be nil") - } - if ro.Index == nil { - vprint.PanicOn("ro.Index annot be nil") - } - if o.Index.name != ro.Index.name { - vprint.PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name)) - } - return *qcx.RequiredForAtomicWriteTx, NoopFinisher, nil - } - - if !writeLogic && qcx.Grp != nil { - // read, with a group in place. - finisher = func(perr *error) {} // finisher is a returned value - - already := false - tx, already = qcx.Grp.AlreadyHaveTx(o) - if already { - return - } - tx = qcx.Txf.NewTx(o) - qcx.Grp.AddTx(tx, o) - return - } - - // non atomic writes or not grouped reads - tx = qcx.Txf.NewTx(o) - if o.Write { - finisherDone := false - finisher = func(perr *error) { - if finisherDone { - return - } - finisherDone = true // only Commit once. - // so defer finisher(nil) means always Commit writes, ignoring - // the enclosing functions return status. - if perr == nil || *perr == nil { - vprint.PanicOn(tx.Commit()) - } else { - tx.Rollback() - } - } - } else { - // read-only txn - finisher = func(perr *error) { - tx.Rollback() - } - } - return -} - -// StartAtomicWriteTx allocates a Tx and stores it -// in qcx.RequiredForAtomicWriteTx. All subsequent writes -// to this shard/index will re-use it. -func (qcx *Qcx) StartAtomicWriteTx(o Txo) { - if !o.Write { - vprint.PanicOn("must have o.Write true") - } - qcx.mu.Lock() - defer qcx.mu.Unlock() - - if qcx.RequiredForAtomicWriteTx == nil { - // new Tx needed - tx := qcx.Txf.NewTx(o) - qcx.RequiredForAtomicWriteTx = &tx - qcx.RequiredTxo = &o - return - } - - // re-using existing - - // verify that shard and index match! - ro := qcx.RequiredTxo - if o.Shard != ro.Shard { - vprint.PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard)) - } - if o.Index == nil { - vprint.PanicOn("o.Index annot be nil") - } - if ro.Index == nil { - vprint.PanicOn("ro.Index annot be nil") - } - if o.Index.name != ro.Index.name { - vprint.PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name)) - } -} - -func (qcx *Qcx) ListOpenTx() string { - return qcx.Grp.String() -} - -// TxFactory abstracts the creation of Tx interface-level -// transactions so that RBF, or Roaring-fragment-files, or several -// of these at once in parallel, is used as the storage and transction layer. -type TxFactory struct { - typeOfTx string - - typ txtype - - dbsClosed bool // idemopotent CloseDB() - - dbPerShard *DBPerShard - - holder *Holder -} - -// integer types for fast switch{} -type txtype int - -const ( - noneTxn txtype = 0 - rbfTxn txtype = 2 -) - -// DirectoryName just returns a string version of the transaction type. We -// really need to consolidate the storage backend and tx stuff because it's -// currently rather confusing. This method should be addressed (i.e. -// replaced/removed) during that refactor. -func (ty txtype) DirectoryName() string { - switch ty { - case rbfTxn: - return "rbf" - } - vprint.PanicOn(fmt.Sprintf("unkown txtype %v", int(ty))) - return "" -} - -func MustBackendToTxtype(backend string) (typ txtype) { - if strings.Contains(backend, "_") { - panic("blue-green comparisons removed") - } - - switch backend { - case RBFTxn: // "rbf" - return rbfTxn - } - panic(fmt.Sprintf("unknown backend '%v'", backend)) -} - -// NewTxFactory always opens an existing database. If you -// want to a fresh database, os.RemoveAll on dir/name ahead of time. -// We always store files in a subdir of holderDir. -func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) { - typ := MustBackendToTxtype(backend) - - f = &TxFactory{ - typ: typ, - typeOfTx: backend, - holder: holder, - } - f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder) - - if f.hasRBF() { - holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig) - } - - return f, err -} - -// Open should be called only once the index metadata is loaded -// from Holder.Open(), so we find all of our indexes. -func (f *TxFactory) Open() error { - return f.dbPerShard.LoadExistingDBs() -} - -// Txo holds the transaction options -type Txo struct { - Write bool - Field *Field - Index *Index - Fragment *fragment - Shard uint64 - - dbs *DBShard -} - -func (f *TxFactory) TxType() string { - return f.typeOfTx -} - -func (f *TxFactory) TxTyp() txtype { - return f.typ -} - -func (f *TxFactory) DeleteIndex(name string) (err error) { - return f.dbPerShard.DeleteIndex(name) -} - -func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - return f.dbPerShard.DeleteFieldFromStore(index, field, fieldPath) -} - -func (f *TxFactory) DeleteFragmentFromStore( - index, field, view string, shard uint64, frag *fragment, -) (err error) { - return f.dbPerShard.DeleteFragment(index, field, view, shard, frag) -} - -// CloseIndex is a no-op. This seems to be in place for debugging purposes. -func (f *TxFactory) CloseIndex(idx *Index) error { - return nil -} - -func (f *TxFactory) Close() (err error) { - if f.dbsClosed { - return nil - } - f.dbsClosed = true - return f.dbPerShard.Close() -} - -var globalUseStatTx = false - -func init() { - v := os.Getenv("PILOSA_CALLSTAT") - if v != "" { - globalUseStatTx = true - } -} - -// TxGroup holds a set of read transactions -// that will en-mass have Rollback() (for the read set) called on -// them when TxGroup.Finish() is invoked. -// Alternatively, TxGroup.Abort() will call Rollback() -// on all Tx group memebers. -// -// It used to have writes but we never actually used that because -// of the Qcx needing to make every commit get its own transaction. -type TxGroup struct { - mu sync.Mutex - fac *TxFactory - reads []Tx - finished bool - - all map[grpkey]Tx -} - -type grpkey struct { - index string - shard uint64 -} - -func mustHaveIndexShard(o *Txo) { - if o.Index == nil || o.Index.name == "" { - vprint.PanicOn("index must be set on Txo") - } -} - -func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { - mustHaveIndexShard(&o) - g.mu.Lock() - defer g.mu.Unlock() - key := grpkey{index: o.Index.name, shard: o.Shard} - tx, already = g.all[key] - return -} - -func (g *TxGroup) String() (r string) { - g.mu.Lock() - defer g.mu.Unlock() - if len(g.reads) == 0 { - return "" - } - r += "\n" - for i, tx := range g.reads { - r += fmt.Sprintf("[%v]read: %#v,\n", i, tx) - } - return r -} - -// NewTxGroup -func (f *TxFactory) NewTxGroup() (g *TxGroup) { - g = &TxGroup{ - fac: f, - all: make(map[grpkey]Tx), - } - return -} - -// AddTx adds tx to the group. -func (g *TxGroup) AddTx(tx Tx, o Txo) { - g.mu.Lock() - defer g.mu.Unlock() - if g.finished { - vprint.PanicOn("in TxGroup.Finish(): TxGroup already finished") - } - - g.reads = append(g.reads, tx) - - key := grpkey{index: o.Index.name, shard: o.Shard} - prior, ok := g.all[key] - if ok { - vprint.PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx)) - } - g.all[key] = tx -} - -// Finish commits the write tx and calls Rollback() on -// the read tx contained in the group. Either Abort() or Finish() must -// be called on the TxGroup exactly once. -func (g *TxGroup) FinishGroup() (err error) { - g.mu.Lock() - defer g.mu.Unlock() - if g.finished { - vprint.PanicOn("in TxGroup.Finish(): TxGroup already finished") - } - g.finished = true - for _, r := range g.reads { - r.Rollback() - } - return -} - -// Abort calls Rollback() on all the group Tx, and marks -// the group as finished. Either Abort() or Finish() must -// be called on the TxGroup. -func (g *TxGroup) AbortGroup() { - g.mu.Lock() - defer g.mu.Unlock() - if g.finished { - // defer Abort() probably gets here often by default, just ignore. - return - } - g.finished = true - - for _, r := range g.reads { - r.Rollback() - } -} - -func (f *TxFactory) NewTx(o Txo) (txn Tx) { - defer func() { - if globalUseStatTx { - txn = newStatTx(txn) - } - }() - - indexName := "" - if o.Index != nil { - indexName = o.Index.name - } - - if o.Fragment != nil { - if o.Fragment.index() != indexName { - vprint.PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.index='%v' but indexName='%v'", o.Fragment.index(), indexName)) - } - if o.Fragment.shard != o.Shard { - vprint.PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.shard='%v' but o.Shard='%v'", o.Fragment.shard, o.Shard)) - } - } - - // look up in the collection of open databases, and get our - // per-shard database. Opens a new one if needed. - dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index) - vprint.PanicOn(err) - - if dbs.Shard != o.Shard { - vprint.PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard))) - } - //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W) - o.dbs = dbs - - tx, err := dbs.NewTx(o.Write, indexName, o) - if err != nil { - vprint.PanicOn(errors.Wrap(err, "dbs.NewTx transaction errored")) - } - return tx -} - -// has to match the const strings at the top of the file. -func (ty txtype) String() string { - switch ty { - case noneTxn: - return "noneTxn" - case rbfTxn: - return "rbf" - } - vprint.PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) - return "" -} - -func dirExists(name string) bool { - fi, err := os.Stat(name) - if err != nil { - return false - } - if fi.IsDir() { - return true - } - return false -} - -var _ = anyGlobalDBWrappersStillOpen // happy linter - -func anyGlobalDBWrappersStillOpen() bool { - return globalRbfDBReg.Size() != 0 -} - -func (f *TxFactory) hasRBF() bool { - return f.typ == rbfTxn -} - -func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) { - dbs, err := f.dbPerShard.GetDBShard(index, shard, idx) - if err != nil { - return "", errors.Wrap(err, fmt.Sprintf("GetDBShardPath(index='%v', shard='%v', ty='%v')", index, shard, ty.String())) - } - shardPath = dbs.pathForType(ty) - return -} - -func (txf *TxFactory) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) { - return txf.dbPerShard.GetFieldView2ShardsMapForIndex(idx) -} diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go deleted file mode 100644 index b32887605..000000000 --- a/txfactory_internal_test.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "testing" -) - -func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { - // txtype.String() method MUST return strings that match - // our const definitions at the top of txfactory.go. - check := []txtype{rbfTxn} - expect := []string{RBFTxn} - for i, chk := range check { - obs := chk.String() - if obs != expect[i] { - t.Fatalf("expected '%v' but got '%v'", expect[i], obs) - } - } -} diff --git a/txkey/txkey.go b/txkey/txkey.go deleted file mode 100644 index 772dcb7f1..000000000 --- a/txkey/txkey.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -// Package txkey consolidates in one place the use of keys to index into our -// various storage/txn back-ends. Databases LMDB and rbfDB both use it, -// so that debug Dumps are comparable. -package txkey - -import ( - "encoding/binary" - "fmt" -) - -// FieldView is here to avoid circular import. -type FieldView struct { - Field string - View string -} - -func FieldViewFromPrefix(prefix []byte) FieldView { - _, field, view, _ := SplitPrefix(prefix) - return FieldView{Field: field, View: view} -} - -func FieldViewFromFullKey(fullKey []byte) FieldView { - _, field, view, _, _ := Split(fullKey) - return FieldView{Field: field, View: view} -} - -// Key produces the bytes that we use as a key to query the storage/tx engine. -// The roaringContainerKey argument to Key() is a container key into a roaring Container. -// The return value from Key() is constructed as follows: -// -// ~index%field;view:shard'. Keys always end with '#'. -// Keys always contain exactly one each of '%', ';', ':' and '<', in that order. -// The index is between the first byte and the '%'. It must be at least 1 byte long. -// The field is between the '%' and the ';'. It must be at least 1 byte long. -// The view is between the ';' and the ':'. It must be at least 1 byte long. -// The shard is the 8 bytes between the ':' and the '<'. -// The ckey is the 8 bytes between the '<' and the '#'. -// The Prefix of a key ends at, and includes, the '<'. It is at least 16 bytes long. -// The index, field, and view are not allowed to contain these reserved bytes: -// -// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} -// -// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable -// base-64 encoding. -// -// The shortest possible key is 25 bytes. It would be laid out like this: -// -// ~i%f;v:12345678<12345678# -// 1234567890123456789012345 -// -// keys starting with '~' are regular value keys. -// keys starting with '>' are symlink keys. -// -// NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) { - - prefix := Prefix(index, field, view, shard) - - var ckey [9]byte - binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey) - ckey[8] = byte('#') - return append(prefix, ckey[:]...) -} - -// ShardFromKey key example: index/field;view:shard' for symlink: '%v'", string(bkey))) - } - if bkey[n-10] != '<' { - panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey))) - } - if bkey[n-19] != ':' { - panic(fmt.Sprintf("bkey did not have '<' at 18 bytes from the end: '%v'", string(bkey))) - } - if bkey[n-1] != '#' { - panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey))) - } -} - -// KeyExtractContainerKey extracts the containerKey from bkey. -// key example: index/field;view:shard> 16 } -func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } - // GetLoopProgress returns the estimated remaining time to iterate through some // items as well as the loop completion percentage with the following // parameters: diff --git a/view.go b/view.go index 9fc727cde..114954287 100644 --- a/view.go +++ b/view.go @@ -13,7 +13,9 @@ import ( "sync/atomic" "time" + "github.com/molecula/featurebase/v3/keys" "github.com/molecula/featurebase/v3/pql" + qc "github.com/molecula/featurebase/v3/querycontext" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" @@ -111,20 +113,11 @@ func (v *view) addKnownShard(shard uint64) { vprint.PanicOn(err) } -// removeKnownShard removes a known shard from v. See the notes on addKnownShard. -func (v *view) removeKnownShard(shard uint64) { - if atomic.LoadUint32(&v.knownShardsCopied) == 1 { - v.knownShards = v.knownShards.Clone() - atomic.StoreUint32(&v.knownShardsCopied, 0) - } - _, _ = v.knownShards.Remove(shard) -} - // openWithShardSet opens the view. Importantly, it // only opens the fragments that have data. This saves // a ton of time. If you have no data and want a new // view, call view.openEmpty(). -func (v *view) openWithShardSet(ss *shardSet) error { +func (v *view) openWithShardSet(ss keys.ViewContents) error { if v.knownShards == nil { v.knownShards = roaring.NewSliceBitmap() } @@ -134,19 +127,18 @@ func (v *view) openWithShardSet(ss *shardSet) error { v.cacheType = CacheTypeNone } - shards := ss.CloneMaybe() - var frags []*fragment - for shard := range shards { - frag := v.newFragment(shard) + for shard := range ss { + frag := v.newFragment(uint64(shard)) frags = append(frags, frag) v.fragments[frag.shard] = frag } - nGoro := runtime.NumCPU() - if v.idx.holder.txf.TxType() != "roaring" { - nGoro = nGoro / 4 - } + // We used to only divide by 4 if we weren't using the + // roaring backend, but we no longer have it, so this is + // unconditional for now. If we add new backends, this may + // want reconsidering. + nGoro := runtime.NumCPU() / 4 if nGoro < 4 { nGoro = 4 } @@ -179,8 +171,8 @@ func (v *view) openWithShardSet(ss *shardSet) error { // serial, not parallel, because no locking inside addKnownShard at the moment. // TODO(jea): is this slow on a cluster? can we optimize it // by running it on a goroutine in the background? - for shard := range shards { - v.addKnownShard(shard) + for shard := range ss { + v.addKnownShard(uint64(shard)) } _ = testhook.Opened(v.holder.Auditor, v, nil) @@ -401,38 +393,15 @@ func (v *view) newFragment(shard uint64) *fragment { return frag } -// deleteFragment removes the fragment from the view. -func (v *view) deleteFragment(shard uint64) error { - v.mu.Lock() - defer v.mu.Unlock() - f := v.fragments[shard] - if f == nil { - return ErrFragmentNotFound - } - - v.holder.Logger.Infof("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) - - idx := f.holder.Index(v.index) - f.Close() - if err := idx.holder.txf.DeleteFragmentFromStore(f.index(), f.field(), f.view(), f.shard, f); err != nil { - return errors.Wrap(err, "DeleteFragment") - } - delete(v.fragments, shard) - v.removeKnownShard(shard) - - return nil -} - // row returns a row for a shard of the view. -func (v *view) row(qcx *Qcx, rowID uint64) (*Row, error) { +func (v *view) row(qcx qc.QueryContext, rowID uint64) (*Row, error) { row := NewRow() for _, frag := range v.allFragments() { - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard}) + qr, err := frag.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err) - fr, err := frag.row(tx, rowID) + fr, err := frag.row(qr, rowID) if err != nil { return nil, err } else if fr == nil { @@ -446,7 +415,7 @@ func (v *view) row(qcx *Qcx, rowID uint64) (*Row, error) { // mutexCheck checks all available fragments for duplicate values. The return // is map[column]map[shard][]values for collisions only. -func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { +func (v *view) mutexCheck(ctx context.Context, qcx qc.QueryContext, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { // We don't need the context, we just want the context-awareness on the error groups. // It would be nice if the inner functions could use this too... eg, _ := errgroup.WithContext(ctx) @@ -462,12 +431,14 @@ func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int defer func() { <-throttle }() - tx, finisher, err := qcx.GetTx(Txo{Index: v.idx, Shard: frag.shard}) + qr, err := frag.qcxRead(qcx) if err != nil { return err } - defer finisher(&err) - results[i], err = frag.mutexCheck(tx, details, limit) + if err != nil { + return err + } + results[i], err = frag.mutexCheck(qr, details, limit) if err != nil { return err } @@ -500,83 +471,86 @@ func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int } // setBit sets a bit within the view. -func (v *view) setBit(qcx *Qcx, rowID, columnID uint64) (changed bool, err error) { +func (v *view) setBit(qcx qc.QueryContext, rowID, columnID uint64) (changed bool, err error) { shard := columnID / ShardWidth - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard}) - defer finisher(&err) var frag *fragment frag, err = v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - - return frag.setBit(tx, rowID, columnID) + qw, err := frag.qcxWrite(qcx) + if err != nil { + return changed, err + } + return frag.setBit(qw, rowID, columnID) } // clearBit clears a bit within the view. -func (v *view) clearBit(qcx *Qcx, rowID, columnID uint64) (changed bool, err error) { +func (v *view) clearBit(qcx qc.QueryContext, rowID, columnID uint64) (changed bool, err error) { shard := columnID / ShardWidth - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard}) - defer finisher(&err) frag := v.Fragment(shard) if frag == nil { return false, nil } - - return frag.clearBit(tx, rowID, columnID) + qw, err := frag.qcxWrite(qcx) + if err != nil { + return changed, err + } + return frag.clearBit(qw, rowID, columnID) } // value uses a column of bits to read a multi-bit value. -func (v *view) value(qcx *Qcx, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) { +func (v *view) value(qcx qc.QueryContext, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) { shard := columnID / ShardWidth - tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: v.idx, Shard: shard}) - defer finisher(&err) frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return value, exists, err } + qr, err := frag.qcxRead(qcx) + if err != nil { + return value, exists, err + } - return frag.value(tx, columnID, bitDepth) + return frag.value(qr, columnID, bitDepth) } // setValue uses a column of bits to set a multi-bit value. -func (v *view) setValue(qcx *Qcx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) { +func (v *view) setValue(qcx qc.QueryContext, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) { shard := columnID / ShardWidth - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard}) - defer finisher(&err) frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - - return frag.setValue(tx, columnID, bitDepth, value) + qw, err := frag.qcxWrite(qcx) + if err != nil { + return changed, err + } + return frag.setValue(qw, columnID, bitDepth, value) } // clearValue removes a specific value assigned to columnID -func (v *view) clearValue(qcx *Qcx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) { +func (v *view) clearValue(qcx qc.QueryContext, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) { shard := columnID / ShardWidth - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard}) - defer finisher(&err) frag := v.Fragment(shard) if frag == nil { return false, nil } - - return frag.clearValue(tx, columnID, bitDepth, value) + qw, err := frag.qcxWrite(qcx) + if err != nil { + return changed, err + } + return frag.clearValue(qw, columnID, bitDepth, value) } // rangeOp returns rows with a field value encoding matching the predicate. -func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint64, predicate int64) (_ *Row, err0 error) { +func (v *view) rangeOp(qcx qc.QueryContext, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) { r := NewRow() for _, frag := range v.allFragments() { - - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: v.idx, Shard: frag.shard}) + qr, err := frag.qcxRead(qcx) if err != nil { return nil, err } - defer finisher(&err0) - - other, err := frag.rangeOp(tx, op, bitDepth, predicate) + other, err := frag.rangeOp(qr, op, bitDepth, predicate) if err != nil { return nil, err } @@ -585,18 +559,22 @@ func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint64, predicate int64) return r, nil } -func (v *view) bitDepth(shards []uint64) (uint64, error) { +func (v *view) bitDepth(qcx qc.QueryContext, shards keys.Shards) (uint64, error) { var maxBitDepth uint64 - for _, shard := range shards { + for shard := range shards { v.mu.RLock() - frag, ok := v.fragments[shard] + frag, ok := v.fragments[uint64(shard)] v.mu.RUnlock() if !ok || frag == nil { continue } + qr, err := frag.qcxRead(qcx) + if err != nil { + return 0, err + } - bd, err := frag.bitDepth() + bd, err := frag.bitDepth(qr) if err != nil { return 0, errors.Wrapf(err, "getting fragment(%d) bit depth", shard) } diff --git a/view_internal_test.go b/view_internal_test.go index 6832b0c49..da31fcb7b 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -17,38 +17,6 @@ func mustOpenView(tb testing.TB) *view { return v } -// Ensure view can open and retrieve a fragment. -func TestView_DeleteFragment(t *testing.T) { - v := mustOpenView(t) - - shard := uint64(9) - - // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(shard) - if err != nil { - t.Fatal(err) - } else if fragment == nil { - t.Fatal("expected fragment") - } - - err = v.deleteFragment(shard) - if err != nil { - t.Fatal(err) - } - - if v.Fragment(shard) != nil { - t.Fatal("fragment still exists in view") - } - - // Recreate fragment with same shard, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(shard) - if err != nil { - t.Fatal(err) - } else if fragment == fragment2 { - t.Fatal("failed to create new fragment") - } -} - // Ensure that simultaneous attempts to grab a new fragment don't clash even // if the broadcast operation takes a bit of time. func TestView_CreateFragmentRace(t *testing.T) {