scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

overhaul: switch over to using QueryContext

Browse source 
We switch everything to use QueryContext/QueryRead/etc instead
of Qcx/Tx.

We drop the short_txkey subpackage (it's now handled by either
keys or querycontext).

We drop all the dbshard stuff, and all the tx/txfactory stuff.

We remove all the things that related to the old "Block" concept,
which was mostly used by the anti-entropy code, but had one
fragmentary usage left in the ImportRoaringOverwrite case of
ImportRoaring. That's replaced by using a rewriter that deletes
all bits (not just bits in specific columns) from an existing
thing, but writes in new bits. Actually we could probably do that
better with a custom "eradicate-rewriter" that doesn't try to
be clever, and just eliminates things.

This includes a number of minor bug fixes that were
exposed by getting the testing to work. For example:
* When checking whether an operation "requires write", we
  now consider a Delete a kind of a Write, because it is.
* Several tests were relying on the fact that writes through
  Qcx were being committed whether or not the Qcx was ever
  told to finish. With QueryContext, you actually have to
  reach a Commit() or the writes don't happen (except for
  special cases in Delete).
* Replaced a lot of panics with t.Fatalf in tests.

There's also some minor staticcheck fixes, like deleting the
unused "db" member of a boltdb transaction wrapper.
This commit is contained in:
Seebs 2023-01-11 12:12:04 -06:00
parent 75a68cf60c
commit cf97a0dcb8
61 changed files with 2577 additions and 7110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

345
api.go
View file

@ -4,7 +4,6 @@
package pilosa
import (
"bufio"
"bytes"
"context"
"encoding/binary"
@ -13,7 +12,6 @@ import (
"io"
"math"
"net/url"
"os"
"runtime"
"sort"
"strconv"
@ -26,8 +24,9 @@ import (
"github.com/molecula/featurebase/v3/dax/computer"
"github.com/molecula/featurebase/v3/dax/storage"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/rbf"
qc "github.com/molecula/featurebase/v3/querycontext"
//"github.com/molecula/featurebase/v3/pg"
"github.com/molecula/featurebase/v3/pql"
@ -73,6 +72,28 @@ func (api *API) logger() logger.Logger {
return api.server.logger
}
// NewQueryContext requests a new read-only query context from the API's holder.
func (api *API) NewQueryContext(ctx context.Context) (qc.QueryContext, error) {
return api.holder.NewQueryContext(ctx)
}
// NewWriteQueryContext requests a new write query context from the API's holder,
// using the provided scope.
func (api *API) NewWriteQueryContext(ctx context.Context, scope qc.QueryScope) (qc.QueryContext, error) {
return api.holder.NewWriteQueryContext(ctx, scope)
}
// NewIndexQueryContext requests a new write query context from the API's holder,
// using the provided index. If shards are provided, it's restricted to those shards,
// otherwise it's the whole index.
func (api *API) NewIndexQueryContext(ctx context.Context, index string, shards ...uint64) (qc.QueryContext, error) {
// helpfully treat a shard of -1 as no shard
if len(shards) > 0 && shards[0] == ^uint64(0) {
shards = shards[1:]
}
return api.holder.NewIndexQueryContext(ctx, index, shards...)
}
// apiOption is a functional option type for pilosa.API
type apiOption func(*API) error
@ -200,10 +221,6 @@ func (api *API) Close() error {
return nil
}
func (api *API) Txf() *TxFactory {
return api.holder.Txf()
}
// Query parses a PQL query out of the request and executes it.
func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) {
start := time.Now()
@ -474,7 +491,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) {
type importJob struct {
ctx context.Context
qcx *Qcx
qcx qc.QueryContext
req *ImportRoaringRequest
shard uint64
field *Field
@ -511,18 +528,11 @@ func importWorker(importWork chan importJob) {
doAction = RequestActionSet
}
}
if err := func() (err1 error) {
tx, finisher, err := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard})
if err != nil {
return err
}
defer finisher(&err1)
var doClear bool
switch doAction {
case RequestActionOverwrite:
err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block)
err := j.field.importRoaringOverwrite(j.ctx, j.qcx, viewData, j.shard, viewName)
if err != nil {
return errors.Wrap(err, "importing roaring as overwrite")
}
@ -546,17 +556,19 @@ func importWorker(importWork chan importJob) {
return errors.Wrap(err, "merging existence on roaring import")
}
err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false)
err = ef.importRoaring(j.ctx, j.qcx, existence, j.shard, "standard", false)
if err != nil {
return errors.Wrap(err, "updating existence on roaring import")
}
}
}
err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
err := j.field.importRoaring(j.ctx, j.qcx, data, j.shard, viewName, doClear)
if err != nil {
return errors.Wrap(err, "importing standard roaring")
}
default:
return fmt.Errorf("unexpected action type %q", doAction)
}
return nil
}(); err != nil {
@ -574,7 +586,7 @@ func importWorker(importWork chan importJob) {
}
// combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment.
func combineForExistence(inputRoaringData []byte) ([]byte, error) {
func combineForExistence(inputRoaringData []byte) (a []byte, b error) {
rowSize := uint64(1 << shardVsContainerExponent)
rit, err := roaring.NewRoaringIterator(inputRoaringData)
if err != nil {
@ -635,8 +647,11 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return newPreconditionFailedError(err)
}
qcx := api.Txf().NewQcx()
defer qcx.Abort()
qcx, err := api.NewIndexQueryContext(ctx, indexName, shard)
if err != nil {
return errors.Wrap(err, "creating query context")
}
defer qcx.Release()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := api.cluster.NewSnapshot()
@ -690,7 +705,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
Shard: shard,
Clear: req.Clear,
Action: req.Action,
Block: req.Block,
UpdateExistence: req.UpdateExistence,
Views: req.Views,
}
@ -711,8 +725,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
}
}
return qcx.Finish()
return qcx.Commit()
}
}
}
@ -823,8 +836,15 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Obtain transaction
tx := index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Shard: shard})
defer tx.Rollback()
qcx, err := api.holder.NewQueryContext(ctx)
if err != nil {
return err
}
defer qcx.Release()
qr, err := f.qcxRead(qcx)
if err != nil {
return err
}
// Wrap writer with a CSV writer.
cw := csv.NewWriter(w)
@ -860,7 +880,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
return cw.Write([]string{rowStr, colStr})
}
citer, _, err := tx.ContainerIterator(indexName, fieldName, viewStandard, shard, 0)
citer, _, err := qr.ContainerIterator(0)
if err != nil {
return err
}
@ -884,7 +904,6 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
// Ensure data is flushed.
cw.Flush()
span.LogKV("n", n)
tx.Rollback()
return nil
}
@ -918,23 +937,6 @@ func (api *API) PartitionNodes(ctx context.Context, partitionID int) ([]*disco.N
return snap.PartitionNodes(partitionID), nil
}
// FragmentData returns all data in the specified fragment.
func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentData")
defer span.Finish()
if err := api.validate(apiFragmentData); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.holder.fragment(indexName, fieldName, viewName, shard)
if f == nil {
return nil, ErrFragmentNotFound
}
return f, nil
}
type RedirectError struct {
HostPort string
error string
@ -1265,35 +1267,24 @@ func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard
return nil, newNotFoundError(ErrIndexNotFound, indexName)
}
// Start transaction.
tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard, Write: writeTx})
// Ensure transaction is an RBF transaction.
rtx, ok := tx.(*RBFTx)
// check whether txStore backend supports backup
br, ok := api.holder.txStore.(qc.TxBackupRestore)
if !ok {
tx.Rollback()
return nil, fmt.Errorf("snapshot not available for %q storage", tx.Type())
return nil, errors.New("backend does not support backup/restore operations")
}
r, err := rtx.SnapshotReader()
// Start transaction.
qcx, err := api.NewIndexQueryContext(ctx, indexName, shard)
if err != nil {
tx.Rollback()
return nil, err
}
return &txReadCloser{tx: tx, Reader: r}, nil
}
var _ io.ReadCloser = (*txReadCloser)(nil)
// txReadCloser wraps a reader to close a tx on close.
type txReadCloser struct {
io.Reader
tx Tx
}
func (r *txReadCloser) Close() error {
r.tx.Rollback()
return nil
// Backup either releases the QueryContext, or returns a ReadCloser which
// releases it on close. Note that we DO NOT release/close our
// QueryContext, because the entire point is to block any other writes
// to that shard from happening until the backup completes, because we
// don't want to allow any new writes to start and thus send data to the
// write log until we've snapshotted.
return br.Backup(qcx, keys.Index(indexName), keys.Shard(shard))
}
// ImportOptions holds the options for the API.Import
@ -1351,7 +1342,7 @@ func OptImportOptionsSuppressLog(b bool) ImportOption {
var ErrAborted = fmt.Errorf("error: update was aborted")
func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRecord, opts ...ImportOption) error {
func (api *API) ImportAtomicRecord(ctx context.Context, qcx qc.QueryContext, req *AtomicRecord, opts ...ImportOption) error {
simPowerLoss := false
lossAfter := -1
var opt ImportOptions
@ -1368,14 +1359,6 @@ func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRec
lossAfter = opt.SimPowerLossAfter
}
idx, err := api.Index(ctx, req.Index)
if err != nil {
return errors.Wrap(err, "getting index")
}
// the whole point is to run this part of the import atomically.
// Begin that Tx now!
qcx.StartAtomicWriteTx(Txo{Write: writable, Index: idx, Shard: req.Shard})
tot := 0
options, err := setUpImportOptions(opts...)
@ -1429,7 +1412,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption {
}
// Import does the top-level importing.
func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) {
func (api *API) Import(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, opts ...ImportOption) (err error) {
if req.Clear {
opts = addClearToImportOptions(opts)
}
@ -1506,7 +1489,7 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts .
}
// ImportWithTx bulk imports data into a particular index,field,shard.
func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error {
func (api *API) ImportWithTx(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, options *ImportOptions) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
defer span.Finish()
@ -1640,61 +1623,57 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
}
// we really only need a Tx, but getting a Qcx so that there's only one path for getting a Tx
qcx := api.Txf().NewQcx()
qcx.write = true
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
qcx, err := api.NewIndexQueryContext(ctx, indexName, shard)
if err != nil {
return errors.Wrap(err, "getting Tx")
return err
}
defer qcx.Finish()
var err1 error
defer finisher(&err1)
defer qcx.Release()
if !req.Remote {
err1 = errors.New("forwarding unimplemented on this endpoint")
return err1
return errors.New("forwarding unimplemented on this endpoint")
}
for _, viewUpdate := range req.Views {
field := index.Field(viewUpdate.Field)
if field == nil {
err1 = errors.Errorf("no field named '%s' found.", viewUpdate.Field)
return err1
return errors.Errorf("no field named '%s' found.", viewUpdate.Field)
}
fieldType := field.Options().Type
if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
return err1
if err := cleanupView(fieldType, &viewUpdate); err != nil {
return err
}
view, err := field.createViewIfNotExists(viewUpdate.View)
if err != nil {
err1 = errors.Wrap(err, "getting view")
return err1
return errors.Wrap(err, "getting view")
}
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
err1 = errors.Wrap(err, "getting fragment")
return err1
return errors.Wrap(err, "getting fragment")
}
qw, err := frag.qcxWrite(qcx)
if err != nil {
return err
}
switch fieldType {
case FieldTypeSet, FieldTypeTime:
if !viewUpdate.ClearRecords {
err1 = frag.ImportRoaringClearAndSet(ctx, tx, viewUpdate.Clear, viewUpdate.Set)
err = frag.ImportRoaringClearAndSet(ctx, qw, viewUpdate.Clear, viewUpdate.Set)
} else {
err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set)
err = frag.ImportRoaringSingleValued(ctx, qw, viewUpdate.Clear, viewUpdate.Set)
}
case FieldTypeInt, FieldTypeTimestamp, FieldTypeDecimal:
err1 = frag.ImportRoaringBSI(ctx, tx, viewUpdate.Clear, viewUpdate.Set)
err = frag.ImportRoaringBSI(ctx, qw, viewUpdate.Clear, viewUpdate.Set)
case FieldTypeMutex, FieldTypeBool:
err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set)
err = frag.ImportRoaringSingleValued(ctx, qw, viewUpdate.Clear, viewUpdate.Set)
default:
err1 = errors.Errorf("field type %s is not supported", fieldType)
err = errors.Errorf("field type %s is not supported", fieldType)
}
if err1 != nil {
return err1
if err != nil {
return err
}
// need to update field/bsiGroup bitDepth value if this is an int-like field.
@ -1703,10 +1682,9 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
// that we have to do this is weird and since this state isn't
// in RBF might have transactional issues.
if len(field.bsiGroups) > 0 {
maxRowID, _, err := frag.maxRow(tx, nil)
maxRowID, _, err := frag.maxRow(qw, nil)
if err != nil {
err1 = errors.Wrapf(err, "getting fragment max row id")
return err1
return errors.Wrapf(err, "getting fragment max row id")
}
var bd uint64
if maxRowID+1 > bsiOffsetBit {
@ -1741,18 +1719,16 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
if err != nil {
err1 = errors.Wrap(err, "marshalling log message")
return err1
return errors.Wrap(err, "marshalling log message")
}
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
err1 = errors.Wrap(resource.Append(b), "appending shard data")
if err1 != nil {
return err1
err = errors.Wrap(resource.Append(b), "appending shard data")
if err != nil {
return err
}
}
return nil
return qcx.Commit()
}
func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
@ -1778,7 +1754,7 @@ func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
// ImportValue is a wrapper around the common code in ImportValueWithTx, which
// currently just translates req.Clear into a clear ImportOption.
func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
func (api *API) ImportValue(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, opts ...ImportOption) error {
if req.Clear {
opts = addClearToImportOptions(opts)
}
@ -1849,7 +1825,7 @@ func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueReque
}
// ImportValueWithTx bulk imports values into a particular field.
func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) (err0 error) {
func (api *API) ImportValueWithTx(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, options *ImportOptions) (err0 error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue")
defer span.Finish()
@ -2029,7 +2005,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
return nil
}
func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
func importExistenceColumns(qcx qc.QueryContext, index *Index, columnIDs []uint64, shard uint64) error {
ef := index.existenceField()
if ef == nil {
return nil
@ -2176,7 +2152,7 @@ func (api *API) Info() serverInfo {
CPUMHz: mhz,
CPUType: si.CPUModel(),
Memory: mem,
StorageBackend: api.holder.txf.TxType(),
StorageBackend: api.holder.txStore.Backend(),
ReplicaN: api.cluster.ReplicaN,
ShardHash: api.cluster.Hasher.Name(),
KeyHash: api.cluster.Hasher.Name(),
@ -2578,91 +2554,61 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
}
idx := api.holder.Index(indexName)
// need to get a dbShard
dbs, err := api.holder.Txf().dbPerShard.GetDBShard(indexName, shard, idx)
if err != nil {
return err
br, ok := api.holder.txStore.(qc.TxBackupRestore)
if !ok {
return errors.New("backend does not support backup/restore operations")
}
db := dbs.W
finalPath := db.Path() + "/data"
tempPath := finalPath + ".tmp"
o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
qcx, err := api.NewIndexQueryContext(ctx, indexName, shard)
if err != nil {
return err
return errors.Wrap(err, "getting query context")
}
defer o.Close()
bw := bufio.NewWriter(o)
if _, err = io.Copy(bw, rd); err != nil {
return err
} else if err := bw.Flush(); err != nil {
return err
} else if err := o.Sync(); err != nil {
return err
} else if err := o.Close(); err != nil {
return err
defer qcx.Release()
err = br.Restore(qcx, keys.Index(indexName), keys.Shard(shard), rd)
if err != nil {
return errors.Wrap(err, "underlying restore")
}
flvs, err := api.holder.txStore.ListFieldViews(keys.Index(indexName), keys.Shard(shard))
if err != nil {
_ = os.Remove(tempPath)
return err
return errors.Wrap(err, "finding field/view list")
}
err = db.CloseDB()
if err != nil {
return err
}
err = os.Rename(tempPath, finalPath)
if err != nil {
_ = os.Remove(tempPath)
return err
}
err = db.OpenDB()
if err != nil {
return err
}
tx, err := db.NewTx(false, idx.name, Txo{})
if err != nil {
return err
}
defer tx.Rollback()
// arguments idx,shard do not matter for rbf they
// are ignored
flvs, err := tx.GetSortedFieldViewList(idx, shard)
if err != nil {
return nil
}
for _, flv := range flvs {
fld := idx.field(flv.Field)
view := fld.view(flv.View)
if view == nil {
view, err = fld.createViewIfNotExists(flv.View)
for field, views := range flvs {
fld := idx.field(string(field))
for _, viewName := range views {
view := fld.view(string(viewName))
if view == nil {
view, err = fld.createViewIfNotExists(string(viewName))
if err != nil {
return err
}
}
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
return err
}
qr, err := frag.qcxRead(qcx)
if err != nil {
return err
}
err = frag.RebuildRankCache(ctx, qr)
if err != nil {
return err
}
bd, err := view.bitDepth(qcx, map[keys.Shard]struct{}{keys.Shard(shard): {}})
if err != nil {
return err
}
err = fld.cacheBitDepth(bd)
if err != nil {
return err
}
}
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
return err
}
err = frag.RebuildRankCache(ctx)
if err != nil {
return err
}
bd, err := view.bitDepth([]uint64{shard})
if err != nil {
return err
}
err = fld.cacheBitDepth(bd)
if err != nil {
return err
}
}
return nil
}
func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
func (api *API) mutexCheckThisNode(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
index := api.holder.Index(indexName)
if index == nil {
return nil, newNotFoundError(ErrIndexNotFound, indexName)
@ -2735,7 +2681,7 @@ func mergeKeyLists(dst []string, src []string) []string {
// MutexCheckNode checks for collisions in a given mutex field. The response is
// a map[shard]map[column]values, not translated.
func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
func (api *API) MutexCheckNode(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
if err := api.validate(apiMutexCheck); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -2754,7 +2700,7 @@ func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string,
// details false:
// []uint64 // unkeyed index
// []string // keyed index
func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) {
func (api *API) MutexCheck(ctx context.Context, qcx qc.QueryContext, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) {
if err = api.validate(apiMutexCheck); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -3045,21 +2991,6 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp
return api.server.CompileExecutionPlan(ctx, q)
}
func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
infos := make(map[string]*rbf.DebugInfo)
for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap {
wrapper, ok := dbShard.W.(*RbfDBWrapper)
if !ok {
continue
}
skey := fmt.Sprintf("%s/%d", key.index, key.shard)
infos[skey] = wrapper.db.DebugInfo()
}
return infos
}
// Directive applies the provided Directive to the local computer.
func (api *API) Directive(ctx context.Context, d *dax.Directive) error {
return api.ApplyDirective(ctx, d)
@ -3203,8 +3134,8 @@ const (
apiDeleteIndex
apiDeleteView
apiExportCSV
apiFragmentBlockData
apiFragmentBlocks
// apiFragmentBlockData
// apiFragmentBlocks
apiFragmentData
apiTranslateData
apiFieldTranslateData
@ -3247,8 +3178,6 @@ var methodsCommon = map[apiMethod]struct{}{
var methodsDegraded = map[apiMethod]struct{}{
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiIndex: {},
apiQuery: {},
@ -3273,8 +3202,6 @@ var methodsNormal = map[apiMethod]struct{}{
apiDeleteIndex: {},
apiDeleteView: {},
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiFieldTranslateData: {},
apiImport: {},

67
api_directive.go
View file

@ -552,7 +552,6 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar
req := &ImportRoaringRequest{
Clear: msg.Clear,
Action: msg.Action,
Block: msg.Block,
Views: msg.Views,
UpdateExistence: msg.UpdateExistence,
SuppressLog: true,
@ -574,17 +573,30 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar
Clear: msg.Clear,
}
qcx := api.Txf().NewQcx()
defer qcx.Abort()
// subfunc so the qcx gets released after each message is handled.
err := func() error {
qcx, err := api.NewIndexQueryContext(ctx, req.Index, req.Shard)
if err != nil {
return errors.Wrapf(err, "creating query context")
}
defer qcx.Release()
opts := []ImportOption{
OptImportOptionsClear(msg.Clear),
OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
OptImportOptionsPresorted(msg.Presorted),
OptImportOptionsSuppressLog(true),
}
if err := api.Import(ctx, qcx, req, opts...); err != nil {
return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
opts := []ImportOption{
OptImportOptionsClear(msg.Clear),
OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
OptImportOptionsPresorted(msg.Presorted),
OptImportOptionsSuppressLog(true),
}
if err := api.Import(ctx, qcx, req, opts...); err != nil {
return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
}
if err := qcx.Commit(); err != nil {
return errors.Wrap(err, "committing write")
}
return nil
}()
if err != nil {
return err
}
case *computer.ImportValueMessage:
@ -601,17 +613,30 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Shar
Clear: msg.Clear,
}
qcx := api.Txf().NewQcx()
defer qcx.Abort()
// subfunc so the qcx gets released after each message is handled.
err = func() error {
qcx, err := api.NewIndexQueryContext(ctx, req.Index, req.Shard)
if err != nil {
return errors.Wrapf(err, "creating query context")
}
defer qcx.Release()
opts := []ImportOption{
OptImportOptionsClear(msg.Clear),
OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
OptImportOptionsPresorted(msg.Presorted),
OptImportOptionsSuppressLog(true),
}
if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
opts := []ImportOption{
OptImportOptionsClear(msg.Clear),
OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
OptImportOptionsPresorted(msg.Presorted),
OptImportOptionsSuppressLog(true),
}
if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
}
if err := qcx.Commit(); err != nil {
return errors.Wrap(err, "committing write")
}
return nil
}()
if err != nil {
return err
}
case *computer.ImportRoaringShardMessage:
req := &ImportRoaringShardRequest{

151
api_test.go
View file

@ -24,15 +24,46 @@ import (
"github.com/golang-jwt/jwt"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
// mustQueryContext gets a write query context for the API and the
// specified index, or index-and-shards, or fails
// the test. The query context will be released at the end of the test.
func mustIndexQueryContext(tb testing.TB, api *pilosa.API, index string, shards ...uint64) qc.QueryContext {
tb.Helper()
// disregard a leading ^0, because that's idiomatic for "all shards"
if len(shards) > 0 && shards[0] == ^uint64(0) {
shards = shards[1:]
}
qcx, err := api.NewIndexQueryContext(context.Background(), index, shards...)
if err != nil {
tb.Fatalf("creating query context: %v", err)
}
tb.Cleanup(qcx.Release)
return qcx
}
// mustQueryContext gets a read-only query context for the API, or fails
// the test. The query context will be released at the end of the test.
func mustQueryContext(tb testing.TB, api *pilosa.API) qc.QueryContext {
tb.Helper()
qcx, err := api.NewQueryContext(context.Background())
if err != nil {
tb.Fatalf("creating query context: %v", err)
}
tb.Cleanup(qcx.Release)
return qcx
}
func TestAPI_Import(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
@ -89,12 +120,12 @@ func TestAPI_Import(t *testing.T) {
ColumnKeys: colKeys,
}
qcx := m0.API.Txf().NewQcx()
qcx := mustIndexQueryContext(t, m0.API, req.Index)
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
pql := fmt.Sprintf("Row(%s=%d)", fieldNames[false], rowIDs[0])
@ -154,13 +185,10 @@ func TestAPI_Import(t *testing.T) {
req.RowIDs = rowIDs
}
err := func() error {
qcx := m0.API.Txf().NewQcx()
defer qcx.Abort()
qcx := mustIndexQueryContext(t, m0.API, req.Index)
defer qcx.Release()
err := m0.API.Import(ctx, qcx, req.Clone())
e2 := qcx.Finish()
if e2 != nil {
t.Fatalf("unexpected error committing: %v", e2)
}
require.Nil(t, qcx.Commit())
return err
}()
if err != nil {
@ -241,11 +269,11 @@ func TestAPI_ImportValue(t *testing.T) {
Shard: 0, // inaccurate but keys override it
}
qcx := coord.API.Txf().NewQcx()
qcx := mustIndexQueryContext(t, coord.API, req.Index)
if err := coord.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
pql := fmt.Sprintf("Row(%s>0)", field)
@ -291,41 +319,41 @@ func TestAPI_ImportValue(t *testing.T) {
Index: index,
Field: field,
}
qcx1 := coord.API.Txf().NewQcx()
defer qcx1.Abort()
qcx1 := mustIndexQueryContext(t, coord.API, req.Index)
defer qcx1.Release()
// Import with empty request, should succeed
if err := coord.API.ImportValue(ctx, qcx1, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx1.Finish())
require.Nil(t, qcx1.Commit())
// Import without data but with columnkeys, verify that it errors
req.ColumnKeys = colKeys
qcx2 := coord.API.Txf().NewQcx()
defer qcx2.Abort()
qcx2 := mustIndexQueryContext(t, coord.API, req.Index)
defer qcx2.Release()
if err := coord.API.ImportValue(ctx, qcx2, req); err == nil {
t.Fatal("expected error but succeeded")
}
PanicOn(qcx2.Finish())
require.Nil(t, qcx2.Commit())
// Import with mismatch column and value lengths
req.Values = values
qcx3 := coord.API.Txf().NewQcx()
defer qcx3.Abort()
qcx3 := mustIndexQueryContext(t, coord.API, req.Index)
defer qcx3.Release()
if err := coord.API.ImportValue(ctx, qcx3, req); err == nil {
t.Fatal("expected error but succeeded")
}
PanicOn(qcx3.Finish())
require.Nil(t, qcx3.Commit())
// Import with data but no columns
req.ColumnKeys = make([]string, 0)
qcx4 := coord.API.Txf().NewQcx()
defer qcx4.Abort()
qcx4 := mustIndexQueryContext(t, coord.API, req.Index)
defer qcx4.Release()
if err := coord.API.ImportValue(ctx, qcx4, req); err == nil {
t.Fatal("expected error but succeeded")
}
PanicOn(qcx4.Finish())
require.Nil(t, qcx4.Commit())
})
@ -357,11 +385,11 @@ func TestAPI_ImportValue(t *testing.T) {
ColumnIDs: colIDs,
FloatValues: values,
}
qcx := m0.API.Txf().NewQcx()
qcx := mustIndexQueryContext(t, m0.API, req.Index)
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
query := fmt.Sprintf("Row(%s>6)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
@ -418,11 +446,11 @@ func TestAPI_ImportValue(t *testing.T) {
TimestampValues: values,
}
qcx := m2.API.Txf().NewQcx()
qcx := mustIndexQueryContext(t, m2.API, req.Index)
if err := m2.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
query := fmt.Sprintf("Row(%s>='1833-11-24T17:31:50Z')", field) // 6s after MinTimestamp
@ -476,11 +504,11 @@ func TestAPI_ImportValue(t *testing.T) {
ColumnIDs: colIDs,
StringValues: values,
}
qcx := m0.API.Txf().NewQcx()
qcx := mustIndexQueryContext(t, m0.API, req.Index)
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
@ -667,14 +695,14 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
RowIDs: []uint64{iraRowID},
}
qcx := m0api.Txf().NewQcx()
qcx := mustIndexQueryContext(t, m0api, index)
if err := m0api.Import(ctx, qcx, ir0.Clone()); err != nil {
t.Fatal(err)
}
if err := m0api.ImportValue(ctx, qcx, ivr0.Clone()); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
bitIsSet := func() bool {
query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
@ -712,24 +740,24 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
}
// clear the bit
qcx = m0api.Txf().NewQcx()
qcx = mustIndexQueryContext(t, m0api, index)
ir0.Clear = true
if err := m0api.Import(ctx, qcx, ir0); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
if bitIsSet() {
PanicOn("IRA bit should have been cleared")
}
// clear the BSI
qcx = m0api.Txf().NewQcx()
qcx = mustIndexQueryContext(t, m0api, index)
ivr0.Clear = true
if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
if bal != 0 {
@ -883,9 +911,9 @@ func TestAPI_MutexCheck(t *testing.T) {
defer c.Close()
m0 := c.GetNode(0)
nodesByID := make(map[string]*test.Command, 3)
qcxsByID := make(map[string]*pilosa.Qcx, 3)
for i := 0; i < 3; i++ {
nodesByID := make(map[string]*test.Command, len(c.Nodes))
qcxsByID := make(map[string]qc.QueryContext, len(c.Nodes))
for i := 0; i < len(c.Nodes); i++ {
node := c.GetNode(i)
id := node.API.NodeID()
nodesByID[id] = node
@ -937,7 +965,7 @@ func TestAPI_MutexCheck(t *testing.T) {
for keyedField, fieldData := range indexData.fields {
t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) {
for id, node := range nodesByID {
qcxsByID[id] = node.API.Txf().NewQcx()
qcxsByID[id] = mustIndexQueryContext(t, node.API, indexData.indexName)
}
for shard := uint64(0); shard < nShards; shard++ {
// restore row/col ID values which can get altered by imports
@ -977,13 +1005,10 @@ func TestAPI_MutexCheck(t *testing.T) {
t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err)
}
pilosa.CorruptAMutex(t, field, qcxsByID[id])
err = qcxsByID[id].Finish()
if err != nil {
t.Fatalf("closing out transaction on node %s: %v", id, err)
}
require.Nil(t, qcxsByID[id].Commit())
}
qcx := m0.API.Txf().NewQcx()
defer qcx.Abort()
qcx := mustQueryContext(t, m0.API)
defer qcx.Release()
// first two shards of each group of 4 should have a collision in
// position 1
@ -1076,9 +1101,6 @@ func TestAPI_MutexCheck(t *testing.T) {
indexData = indexes[true]
for keyedField, fieldData := range indexData.fields {
t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) {
for id, node := range nodesByID {
qcxsByID[id] = node.API.Txf().NewQcx()
}
req := &pilosa.ImportRequest{
Index: indexData.indexName,
IndexCreatedAt: indexData.createdAt,
@ -1105,12 +1127,11 @@ func TestAPI_MutexCheck(t *testing.T) {
} else {
req.RowIDs = rowIDs
}
var id string
var node *test.Command
for id, node = range nodesByID {
for _, node = range nodesByID {
break
}
if err := node.API.Import(ctx, qcxsByID[id], req); err != nil {
if err := node.API.Import(ctx, nil, req); err != nil {
t.Fatalf("importing data: %v", err)
}
expected, err := node.API.FindIndexKeys(ctx, indexData.indexName, colKeys...)
@ -1156,14 +1177,15 @@ func TestAPI_MutexCheck(t *testing.T) {
if err != nil {
t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err)
}
pilosa.CorruptAMutex(t, field, qcxsByID[id])
err = qcxsByID[id].Finish()
if err != nil {
t.Fatalf("closing out transaction on node %s: %v", id, err)
}
qcx := mustIndexQueryContext(t, node.API, indexData.indexName)
pilosa.CorruptAMutex(t, field, qcx)
require.Nil(t, qcx.Commit())
}
qcx := m0.API.Txf().NewQcx()
defer qcx.Abort()
qcx := mustQueryContext(t, m0.API)
if err != nil {
t.Fatalf("creating query context: %v", err)
}
defer qcx.Release()
results, err := m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName, true, 0)
if err != nil {
@ -1355,21 +1377,6 @@ func TestAPI_CreateField(t *testing.T) {
}
}
func TestAPI_RBFDebugInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1)
defer c.Close()
coord := c.GetPrimary()
if _, err := coord.API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if infos := coord.API.RBFDebugInfo(); infos == nil {
t.Fatal("expected info")
}
}
// makeUser makes an authnUserInfo from groups and a name and a secret key
func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.UserInfo {
tkn := jwt.New(jwt.SigningMethodHS256)

58
apimethod_string.go
View file

@ -16,38 +16,38 @@ func _() {
_ = x[apiDeleteIndex-5]
_ = x[apiDeleteView-6]
_ = x[apiExportCSV-7]
_ = x[apiFragmentBlockData-8]
_ = x[apiFragmentBlocks-9]
_ = x[apiFragmentData-10]
_ = x[apiTranslateData-11]
_ = x[apiFieldTranslateData-12]
_ = x[apiField-13]
_ = x[apiImport-14]
_ = x[apiImportValue-15]
_ = x[apiIndex-16]
_ = x[apiQuery-17]
_ = x[apiRecalculateCaches-18]
_ = x[apiSchema-19]
_ = x[apiShardNodes-20]
_ = x[apiState-21]
_ = x[apiViews-22]
_ = x[apiApplySchema-23]
_ = x[apiStartTransaction-24]
_ = x[apiFinishTransaction-25]
_ = x[apiTransactions-26]
_ = x[apiGetTransaction-27]
_ = x[apiActiveQueries-28]
_ = x[apiPastQueries-29]
_ = x[apiIDReserve-30]
_ = x[apiIDCommit-31]
_ = x[apiIDReset-32]
_ = x[apiPartitionNodes-33]
_ = x[apiMutexCheck-34]
_ = x[apiFragmentData-8]
_ = x[apiTranslateData-9]
_ = x[apiFieldTranslateData-10]
_ = x[apiField-11]
_ = x[apiImport-12]
_ = x[apiImportValue-13]
_ = x[apiIndex-14]
_ = x[apiQuery-15]
_ = x[apiRecalculateCaches-16]
_ = x[apiSchema-17]
_ = x[apiShardNodes-18]
_ = x[apiState-19]
_ = x[apiViews-20]
_ = x[apiApplySchema-21]
_ = x[apiStartTransaction-22]
_ = x[apiFinishTransaction-23]
_ = x[apiTransactions-24]
_ = x[apiGetTransaction-25]
_ = x[apiActiveQueries-26]
_ = x[apiPastQueries-27]
_ = x[apiIDReserve-28]
_ = x[apiIDCommit-29]
_ = x[apiIDReset-30]
_ = x[apiPartitionNodes-31]
_ = x[apiMutexCheck-32]
_ = x[apiApplyChangeset-33]
_ = x[apiDeleteDataframe-34]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheckapiApplyChangesetapiDeleteDataframe"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 136, 152, 173, 181, 190, 204, 212, 220, 240, 249, 262, 270, 278, 292, 311, 331, 346, 363, 379, 393, 405, 416, 426, 443, 456, 473, 491}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

5
apply.go
View file

@ -15,6 +15,7 @@ import (
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/tracing"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
@ -117,7 +118,7 @@ func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx con
}
// executeApply executes a Apply() call.
func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
func (e *executor) executeApply(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
if !e.dataframeEnabled {
return nil, errors.New("Dataframe support not enabled")
}
@ -189,7 +190,7 @@ func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter
return indexResolver, nil
}
func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
func (e *executor) executeApplyShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64) (value.Value, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
defer span.Finish()

7
arrow.go
View file

@ -19,6 +19,7 @@ import (
"github.com/apache/arrow/go/v10/parquet/pqarrow"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
)
@ -33,7 +34,7 @@ Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
*/
// executeApply executes a Arrow() call.
func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
func (e *executor) executeArrow(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
if !e.dataframeEnabled {
return nil, errors.New("Dataframe support not enabled")
}
@ -353,7 +354,7 @@ func filterColumns(filters []string, table arrow.Table) arrow.Table {
return array.NewTable(filterdSchema, cols, table.NumRows())
}
func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
func (e *executor) executeArrowShard(ctx context.Context, qcx qc.QueryContext, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
name := fmt.Sprintf("a. %v", shard)
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
defer span.Finish()
@ -468,7 +469,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error)
return nil, err
}
defer rr.Close()
records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
records := make([]arrow.Record, rr.NumRecords())
i := 0
for {
rec, err := rr.Read()

221
catcher.go
View file

@ -1,221 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
)
// catcher is useful to report error locations with a
// Stack dump before the complexity
// of the executor_test swallows up
// the location of a PanicOn.
type catcherTx struct {
b Tx
}
func newCatcherTx(b Tx) *catcherTx {
return &catcherTx{b: b}
}
func init() {
// keep golangci-lint happy
_ = newCatcherTx
}
var _ Tx = (*catcherTx)(nil)
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *catcherTx) Rollback() {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
c.b.Rollback()
}
func (c *catcherTx) Commit() error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Commit()
}
func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
}
func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Container(index, field, view, shard, key)
}
func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
}
func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Add(index, field, view, shard, a...)
}
func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
}
func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
}
func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
}
func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Count(index, field, view, shard)
}
func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Max(index, field, view, shard)
}
func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Min(index, field, view, shard)
}
func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
}
func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *catcherTx) Type() string {
return c.b.Type()
}
func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}
func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
}
func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
return c.b.GetSortedFieldViewList(idx, shard)
}
func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
return 0, nil
}

16
ctl/rbf_pages.go
View file

@ -9,7 +9,6 @@ import (
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/rbf"
"github.com/molecula/featurebase/v3/txkey"
)
// RBFPagesCommand represents a command for printing a list of RBF page metadata.
@ -100,21 +99,21 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
case *rbf.LeafPageInfo:
fmt.Fprintf(cmd.stdout, "%-10s ", "leaf")
if cmd.WithTree {
fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree))
fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree))
}
fmt.Fprintf(cmd.stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
case *rbf.BranchPageInfo:
fmt.Fprintf(cmd.stdout, "%-10s ", "branch")
if cmd.WithTree {
fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree))
fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree))
}
fmt.Fprintf(cmd.stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
case *rbf.BitmapPageInfo:
fmt.Fprintf(cmd.stdout, "%-10s ", "bitmap")
if cmd.WithTree {
fmt.Fprintf(cmd.stdout, "%-30q ", prefixToString(info.Tree))
fmt.Fprintf(cmd.stdout, "%-30q ", rbf.PrefixToString(info.Tree))
}
fmt.Fprintf(cmd.stdout, "-\n")
@ -132,12 +131,3 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
return nil
}
func prefixToString(s string) (ret string) {
defer func() {
if err := recover(); err != nil {
ret = s
}
}()
return txkey.PrefixToString([]byte(s))
}

33
ctl/restore.go
View file

@ -2,6 +2,7 @@
package ctl
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
@ -20,6 +21,7 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/server"
"github.com/pkg/errors"
@ -350,34 +352,15 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er
for _, node := range nodes {
logger.Printf("shard %v %v", shard, indexName)
f, err := os.Open(filename)
data, err := os.ReadFile(filename)
if err != nil {
return err
}
defer f.Close()
url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
req, err := retryablehttp.NewRequest("POST", url, f)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream")
token, ok := authn.GetAccessToken(ctx)
if ok && token != "" {
req.Header.Set("Authorization", token)
}
client := cmd.newClient()
resp, err := client.Do(req)
if err != nil {
return err
} else if err := resp.Body.Close(); err != nil {
return err
} else if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
client := pilosa.NewInternalClientFromURI(&node.URI,
pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(time.Second*3)),
pilosa.WithClientRetryPeriod(cmd.RetryPeriod),
pilosa.WithSerializer(proto.Serializer{}))
return client.RestoreShard(ctx, indexName, shard, bytes.NewBuffer(data))
}
return nil
}

9
dataframe_test.go
View file

@ -12,6 +12,7 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
"github.com/stretchr/testify/require"
)
func TestExecutor_Apply(t *testing.T) {
@ -39,16 +40,12 @@ func TestExecutor_Apply(t *testing.T) {
req.Field = fieldName
req.ColumnIDs = []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
req.Values = []int64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
qcx := api.Txf().NewQcx()
defer qcx.Abort()
qcx := mustIndexQueryContext(t, api, req.Index, req.Shard)
if err := api.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
t.Run("dataframe ingest", func(t *testing.T) {
// func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetRequest, shard uint64) (map[string]interface{}, error) {

1
dax/computer/logmessage.go
View file

@ -180,7 +180,6 @@ type ImportRoaringMessage struct {
Shard uint64 `json:"shard"`
Clear bool `json:"clear"`
Action string `json:"action"` // [set, clear, overwrite]
Block int `json:"block"`
Views map[string][]byte `json:"views"`
UpdateExistence bool `json:"update-existence"`
}

702
dbshard.go
View file

@ -1,702 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/storage"
"github.com/pkg/errors"
"github.com/molecula/featurebase/v3/vprint"
)
var _ = sort.Sort
const (
// backendsDir is the default backends directory used to store the
// data for each backend.
backendsDir = "backends"
)
// types to support a database file per shard
type DBHolder struct {
Index map[string]*DBIndex
}
func NewDBHolder() *DBHolder {
return &DBHolder{
Index: make(map[string]*DBIndex),
}
}
type DBIndex struct {
Shard map[uint64]*DBShard
}
type DBWrapper interface {
NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error)
Close() error
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
DeleteField(index, field, fieldPath string) error
OpenListString() string
Path() string
HasData() (has bool, err error)
SetHolder(h *Holder)
//needed for restore
CloseDB() error
OpenDB() error
}
type DBRegistry interface {
OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error)
}
type DBShard struct {
HolderPath string
Index string
Shard uint64
Open bool
typ txtype
styp string
W DBWrapper
ParentDBIndex *DBIndex
idx *Index
per *DBPerShard
closed bool
}
func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) {
if index != dbs.Index {
return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index)
}
if shard != dbs.Shard {
return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard)
}
return dbs.W.DeleteFragment(index, field, view, shard, frag)
}
func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
if index != dbs.Index {
return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index)
}
return dbs.W.DeleteField(index, field, fieldPath)
}
func (dbs *DBShard) Close() (err error) {
dbs.closed = true
return dbs.W.Close()
}
func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
if initialIndexName != dbs.Index {
return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName)
}
if o.dbs != dbs {
return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)
}
if o.Shard != dbs.Shard {
return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))
}
return dbs.W.NewTx(write, initialIndexName, o)
}
type flatkey struct {
index string
shard uint64
}
type DBPerShard struct {
Mu sync.Mutex
HolderDir string
dbh *DBHolder
// just flat, not buried within the Node heirarchy.
// Easily see how many we have.
Flatmap map[flatkey]*DBShard
typ txtype
txf *TxFactory
holder *Holder
// cache the shards per index to avoid excessive
// directory scans of the index directory.
// Keep it up-to-date as we add shards to avoid doing
// a filesystem rescan on new shard creation.
//
// index -> *shardSet
index2shards map[string]*shardSet
StorageConfig *storage.Config
RBFConfig *rbfcfg.Config
}
func newIndex2Shards() (r map[string]*shardSet) {
r = make(map[string]*shardSet)
return
}
type shardSet struct {
shardsMap map[uint64]struct{}
shardsVer int64 // increment with each change.
// give out readonly to repeated consumers if
// readonlyVer == shardsVer
readonly map[uint64]struct{}
readonlyVer int64
}
func (a *shardSet) unionInPlace(b *shardSet) {
shards := b.CloneMaybe()
for shard := range shards {
a.add(shard)
}
}
func (a *shardSet) equals(b *shardSet) bool {
if len(a.shardsMap) != len(b.shardsMap) {
return false
}
for shardInA := range a.shardsMap {
_, ok := b.shardsMap[shardInA]
if !ok {
return false
}
}
return true
}
func (a *shardSet) shards() []uint64 {
s := make([]uint64, 0, len(a.shardsMap))
for si := range a.shardsMap {
s = append(s, si)
}
return s
}
func (ss *shardSet) String() (r string) {
r = "["
for k := range ss.shardsMap {
r += fmt.Sprintf("%v, ", k)
}
r += "]"
return
}
func (ss *shardSet) add(shard uint64) {
_, already := ss.shardsMap[shard]
if !already {
ss.shardsMap[shard] = struct{}{}
ss.shardsVer++
}
}
// CloneMaybe maintains a re-usable readonly version
// ss.shards that can be returned to multiple goroutine
// reads as it will never change. A copy is only made
// once for each change in the shard set.
func (ss *shardSet) CloneMaybe() map[uint64]struct{} {
if ss.readonlyVer == ss.shardsVer {
return ss.readonly
}
// readonlyVer is out of date.
// readonly needs update. We cannot
// modify the readonly map in place;
// must make a fully new copy here.
ss.readonly = make(map[uint64]struct{})
for k := range ss.shardsMap {
ss.readonly[k] = struct{}{}
}
ss.readonlyVer = ss.shardsVer
return ss.readonly
}
func newShardSet() *shardSet {
return &shardSet{
shardsMap: make(map[uint64]struct{}),
}
}
func (per *DBPerShard) LoadExistingDBs() (err error) {
idxs := per.holder.Indexes()
for _, idx := range idxs {
shardset, err := per.txf.GetShardsForIndex(idx, "", true)
if err != nil {
return err
}
for shard := range shardset {
_, err := per.GetDBShard(idx.name, shard, idx)
if err != nil {
return errors.Wrap(err, "DBPerShard.LoadExistingDBs GetDBShard()")
}
}
}
return
}
func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) {
if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil {
vprint.PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
}
d = &DBPerShard{
typ: typ,
HolderDir: holderDir,
holder: holder,
dbh: NewDBHolder(),
Flatmap: make(map[flatkey]*DBShard),
txf: txf,
index2shards: newIndex2Shards(),
StorageConfig: holder.cfg.StorageConfig,
RBFConfig: holder.cfg.RBFConfig,
}
return
}
func (per *DBPerShard) DeleteIndex(index string) (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
dbi, ok := per.dbh.Index[index]
if !ok {
// since we lazily make indexes upon use by a Tx now, we won't
// have an index for server/ TestQuerySQLUnary/test-20 to delete.
// Don't freak out. Just return nil.
return nil
}
for _, dbs := range dbi.Shard {
err = dbs.Close()
if err != nil {
return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()")
}
path := dbs.pathForType(per.typ)
err = os.RemoveAll(path)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path))
}
delete(per.index2shards, index)
}
// allow the index to be created again anew.
delete(per.dbh.Index, index)
return
}
func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
per.Mu.Lock()
defer func() {
if fieldPath != "" {
_ = os.RemoveAll(fieldPath)
}
per.Mu.Unlock()
}()
dbi, ok := per.dbh.Index[index]
if !ok {
// TestIndex_Existence_Delete in index_internal_test.go
// will call us without having ever created a Tx or DB,
// so we can't complain here.
return nil
}
for _, dbs := range dbi.Shard {
if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil {
err = errors.Wrap(e, "DeleteFieldFromStore()")
}
}
return err
}
func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
idx := per.txf.holder.Index(index)
dbs, err := per.GetDBShard(index, shard, idx)
if err != nil {
return err
}
return dbs.DeleteFragment(index, field, view, shard, frag)
}
// if you know the shard, you can use this
// pathForType and prefixForType must be kept in sync!
func (dbs *DBShard) pathForType(ty txtype) string {
// top level paths will end in "@@"
// what here for roaring? well, roaringRegistrar.OpenDBWrapper()
// is a no-op anyhow. so doesn't need to be correct atm.
path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard)
return path
}
// if you don't know the shard, you have to use this.
// prefixForType and pathForType must be kept in sync!
func (per *DBPerShard) prefixForType(idx *Index, ty txtype) string {
// top level paths will end in "@@"
return per.HolderDir + sep + idx.name + sep + backendsDir + sep + ty.DirectoryName() + sep
}
var ErrNoData = fmt.Errorf("no data")
// keep our cache of shards up-to-date in memory; after the initial
// directory scan, this is all we should we need. Prevents us from
// doing additional, expensive, directory scans.
//
// Caller must hold per.Mu.Lock() already.
func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) {
shardset, ok := per.index2shards[dbs.Index]
if !ok {
shardset = newShardSet()
per.index2shards[dbs.Index] = shardset
}
// INVAR: shardset is present, not nil; a map that can be added to.
shardset.add(dbs.Shard)
}
func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
return per.unprotectedGetDBShard(index, shard, idx)
}
func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
dbi, ok := per.dbh.Index[index]
if !ok {
dbi = &DBIndex{
Shard: make(map[uint64]*DBShard),
}
per.dbh.Index[index] = dbi
}
dbs, ok = dbi.Shard[shard]
if dbs != nil && dbs.closed {
vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ))
}
if !ok {
dbs = &DBShard{
typ: per.typ,
ParentDBIndex: dbi,
Index: index,
Shard: shard,
HolderPath: per.HolderDir,
idx: idx,
per: per,
}
dbs.styp = per.typ.String()
dbi.Shard[shard] = dbs
per.updateIndex2ShardCacheWithNewShard(dbs)
}
if !dbs.Open {
var registry DBRegistry
switch dbs.typ {
case rbfTxn:
registry = globalRbfDBReg
registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig)
default:
vprint.PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ))
}
path := dbs.pathForType(dbs.typ)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig)
vprint.PanicOn(err)
h := idx.Holder()
w.SetHolder(h)
dbs.Open = true
per.Flatmap[flatkey{index: index, shard: shard}] = dbs
dbs.W = w
}
return dbs, nil
}
func (per *DBPerShard) Close() (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
for _, dbi := range per.dbh.Index {
for _, dbs := range dbi.Shard {
err = dbs.Close()
vprint.PanicOn(err)
}
}
return
}
// DBPerShardGetShardsForIndex returns the shards for idx.
// If requireData, we open the database and see that it has a key, rather
// than assume that the database file presence is enough.
func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]struct{}, error) {
return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData)
}
// requireData means open the database file and verify that at least one key is set.
// The returned sliceOfShards should not be modified. We will cache it for subsequent
// queries.
//
// when a new DBShard is made, we will update the list of shards then. Thus
// the per.index2shard should always be up to date AFTER the first call here.
func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]struct{}, err error) {
// use the cache, always
per.Mu.Lock()
defer per.Mu.Unlock()
i2ss := per.index2shards
ss, ok := i2ss[idx.name]
if ok {
return ss.CloneMaybe(), nil
}
// INVAR: cache miss, and index2shards[ty] exists.
// gotta read shards from disk directory layout.
setOfShards := newShardSet()
per.index2shards[idx.name] = setOfShards
// Upon return, cache the setOfShards value and reuse it next time
path := per.prefixForType(idx, ty)
ignoreEmpty := false
includeRoot := true
dbf, err := listDirUnderDir(path, includeRoot, ignoreEmpty)
vprint.PanicOn(err)
for _, nm := range dbf {
base := filepath.Base(nm)
// We're only interested in "shard.*" files, so skip everything else.
const shardPrefix = "shard."
const lenOfShardPrefix = len(shardPrefix)
if !strings.HasPrefix(base, shardPrefix) {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(base[lenOfShardPrefix:], 10, 64)
if err != nil {
vprint.PanicOn(err)
continue
}
// exclude those without data?
hasData := false
if requireData {
hasData, err = per.unprotectedTypedIndexShardHasData(ty, idx, shard)
if err != nil {
return nil, err
}
if hasData {
setOfShards.add(shard)
}
} else {
// file presence is enough
setOfShards.add(shard)
}
}
return setOfShards.CloneMaybe(), nil
}
func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) {
if ty != per.typ {
return
}
// make the dbs if it doesn't get exist
dbs, err := per.unprotectedGetDBShard(idx.name, shard, idx)
if err != nil {
return false, errors.Wrap(err, fmt.Sprintf("DBPerShard.TypedIndexShardHasData() "+
"per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String()))
}
return dbs.W.HasData()
}
func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) {
if !dirExists(root) {
return
}
n := len(root) + 1
if includeRoot {
n = 0
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if len(path) < n {
// ignore
} else {
if info == nil {
// re-opening an RBF database hit this, racing with a directory rename.
// Don't freak out.
return nil
}
if !info.IsDir() {
// ignore files
} else {
if ignoreEmpty && info.Size() == 0 {
return nil
}
files = append(files, path[n:])
}
}
return nil
})
return
}
type FieldView2Shards struct {
// field -> view -> *shardSet
m map[string]map[string]*shardSet
}
func (vs *FieldView2Shards) getViewsForField(field string) map[string]*shardSet {
return vs.m[field]
}
func (vs *FieldView2Shards) addViewShardSet(fv txkey.FieldView, ss *shardSet) {
f, ok := vs.m[fv.Field]
if !ok {
f = make(map[string]*shardSet)
vs.m[fv.Field] = f
}
// INVAR: f is ready to take ss.
// existing stuff to merge with?
prior, ok := f[fv.View]
if !ok {
f[fv.View] = ss
return
}
// merge ss and prior. No need to put the union back into f[fv.View]
// because prior is a pointer.
prior.unionInPlace(ss)
}
func (a *FieldView2Shards) equals(b *FieldView2Shards) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a.m) != len(b.m) {
return false
}
for field, viewmapA := range a.m {
viewmapB, ok := b.m[field]
if !ok {
return false
}
if len(viewmapB) != len(viewmapA) {
return false
}
for k, va := range viewmapA {
vb, ok := viewmapB[k]
if !ok {
return false
}
if !va.equals(vb) {
return false
}
}
}
return true
}
func NewFieldView2Shards() *FieldView2Shards {
return &FieldView2Shards{
m: make(map[string]map[string]*shardSet), // expected response from GetView2ShardMapForIndex
}
}
func (vs *FieldView2Shards) addShard(fv txkey.FieldView, shard uint64) {
viewmap, ok := vs.m[fv.Field]
if !ok {
viewmap = make(map[string]*shardSet)
vs.m[fv.Field] = viewmap
}
ss, ok := viewmap[fv.View]
if !ok {
ss = newShardSet()
viewmap[fv.View] = ss
}
ss.add(shard)
}
func (vs *FieldView2Shards) String() (r string) {
r = "\n"
for field, viewmap := range vs.m {
for view, shards := range viewmap {
r += fmt.Sprintf("field '%v' view:'%v' shards:%v\n", field, view, shards)
}
}
r += "\n"
return
}
func (vs *FieldView2Shards) removeField(name string) {
delete(vs.m, name)
}
func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) {
ty := per.typ
switch ty {
default:
vs = NewFieldView2Shards()
shardMap, err := per.TypedDBPerShardGetShardsForIndex(ty, idx, "", true)
if err != nil {
return nil, err
}
for shard := range shardMap {
dbs, err := per.GetDBShard(idx.name, shard, idx)
if err != nil {
return nil, errors.Wrap(err, "DBPerShard.GetFieldView2ShardsMapForIndex GetDBShard()")
}
fieldviews, err := dbs.AllFieldViews()
if err != nil {
return nil, errors.Wrap(err, "DBPerShard.GetFieldView2ShardsMapForIndex dbs.AllFieldViews()")
}
for _, fv := range fieldviews {
vs.addShard(fv, shard)
}
}
}
return
}
func (dbs *DBShard) AllFieldViews() (fvs []txkey.FieldView, err error) {
tx, err := dbs.NewTx(!writable, dbs.idx.name, Txo{Write: !writable, Shard: dbs.Shard, Index: dbs.idx, dbs: dbs})
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("dbshard.NewTx for index '%v', shard %v", dbs.idx.name, dbs.Shard))
}
defer tx.Rollback()
return tx.GetSortedFieldViewList(dbs.idx, dbs.Shard)
}

281
dbshard_internal_test.go
View file

@ -1,281 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/molecula/featurebase/v3/rbf"
"github.com/molecula/featurebase/v3/shardwidth"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/testhook"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
// Shard per db evaluation
func TestShardPerDB_SetBit(t *testing.T) {
f, idx, tx := mustOpenFragment(t)
_ = idx
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(tx, 120, 1); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(tx, 120, 6); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(tx, 121, 0); err != nil {
t.Fatal(err)
}
// should have two containers set in the fragment.
// Verify counts on rows.
if n := f.mustRow(tx, 120).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// commit the change, and verify it is still there
PanicOn(tx.Commit())
// Close and reopen the fragment & verify the data.
err := f.Reopen() // roaring data not being flushed? red on roaring
if err != nil {
t.Fatal(err)
}
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if n := f.mustRow(tx, 120).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
// test that we find all *local* shards
func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly")
PanicOn(err)
defer os.RemoveAll(tmpdir)
v2s := NewFieldView2Shards()
stdShardSet := newShardSet()
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
stdShardSet.add(shard)
}
for _, field := range []string{"f", "_exists"} {
v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
}
for _, src := range []string{"rbf"} {
holder := newTestHolder(t)
index := "rick"
idx := makeSampleRoaringDir(t, tmpdir, index, src, 1, holder, v2s)
if idx == nil {
idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
PanicOn(err)
}
std := "rick/fields/f/views/standard"
shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if _, ok := shards[shard]; !ok {
t.Fatalf("missing shard=%v from shards='%#v'", shard, shards)
}
}
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
PanicOn(err)
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
t.Fatalf("fvs should be len 2, got '%#v' (%s)", fvs, src)
}
if fvs[0] != expect0 {
t.Fatalf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])
}
if fvs[1] != expect1 {
t.Fatalf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1])
}
tx.Rollback()
}
}
}
// data for Test_DBPerShard_GetShardsForIndex
var sampleRoaringDirList = map[string]string{"roaring": `
rick/fields/f/views/standard/fragments/215.cache
rick/fields/f/views/standard/fragments/221.cache
rick/fields/f/views/standard/fragments/223.cache
rick/fields/f/views/standard/fragments/93.cache
rick/fields/f/views/standard/fragments/217.cache
rick/fields/f/views/standard/fragments/219.cache
rick/fields/f/views/standard/fragments/217
rick/fields/f/views/standard/fragments/219
rick/fields/f/views/standard/fragments/215
rick/fields/f/views/standard/fragments/221
rick/fields/f/views/standard/fragments/223
rick/fields/f/views/standard/fragments/93
rick/fields/_exists/views/standard/fragments/221
rick/fields/_exists/views/standard/fragments/215
rick/fields/_exists/views/standard/fragments/217
rick/fields/_exists/views/standard/fragments/93
rick/fields/_exists/views/standard/fragments/219
rick/fields/_exists/views/standard/fragments/223
`,
"rbf": `
rick/backends/backend-rbf/shard.0093-rbf
rick/backends/backend-rbf/shard.0215-rbf
rick/backends/backend-rbf/shard.0217-rbf
rick/backends/backend-rbf/shard.0219-rbf
rick/backends/backend-rbf/shard.0221-rbf
rick/backends/backend-rbf/shard.0223-rbf
`,
}
func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
shards := []uint64{0, 93, 215, 217, 219, 221, 223}
fns := strings.Split(sampleRoaringDirList[backend], "\n")
firstDone := false
for i, fn := range fns {
// This check is here because in sampleRoaringDirList, the first entry
// of each map value is a line feed, so the strings.Split() above
// results in a blank entry for the first item. This means that the
// slice of shards above has an initial entry "0" which is not used.
if fn == "" {
continue
}
var shard uint64
switch backend {
case "rbf":
shard = shards[i]
idx = helperCreateDBShard(h, index, shard)
// first time only, we'll actually make all the shards at this point because
// view2shards has them all anyway.
if !firstDone {
firstDone = true
makeTxTestDBWithViewsShards(t, h, idx, view2shards)
}
continue
case "roaring":
default:
t.Fatalf("invalid backend: %s", backend)
}
path := root + sep + filepath.Dir(fn)
PanicOn(os.MkdirAll(path, 0755))
fd, err := os.Create(root + sep + fn)
PanicOn(err)
if minBytes > 0 {
_, err := fd.Write(make([]byte, minBytes))
PanicOn(err)
}
fd.Close()
}
return
}
func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
idx, err := h.CreateIndexIfNotExists(index, "", IndexOptions{})
PanicOn(err)
// TODO: It's not clear that this is actually doing anything.
dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
PanicOn(err)
_ = dbs
return idx
}
// keep the ocd linter happy
var _ = makeRBFtestDB
func makeRBFtestDB(path string, h *Holder, shard uint64) {
i := uint64(1)
db := rbf.NewDB(path, nil)
err := db.Open()
PanicOn(err)
defer db.Close()
tx, err := db.Begin(true)
PanicOn(err)
err = tx.CreateBitmap("x")
PanicOn(err)
_, err = tx.Add("x", i)
PanicOn(err)
err = tx.Commit()
PanicOn(err)
}
func makeTxTestDBWithViewsShards(tb testing.TB, holder *Holder, idx *Index, exp *FieldView2Shards) {
// TODO(jea): need date time quantum views!!
for field, viewmap := range exp.m {
for view, shset := range viewmap {
ss := shset.CloneMaybe()
for shard := range ss {
// simply write 1 bit to each shard to force its creation.
bits := []uint64{(shard << shardwidth.Exponent) + 1}
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
changeCount, err := tx.Add(idx.name, field, view, shard, bits...)
PanicOn(err)
if changeCount != len(bits) {
tb.Fatalf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount)
}
PanicOn(tx.Commit())
}
}
}
}
// test that rbf can give us a map[view]*shardSet
func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
holder := newTestHolder(t)
index := "rick"
field := "f"
idx, err := holder.CreateIndex(index, "", IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
exp := NewFieldView2Shards()
stdShardSet := newShardSet()
stdShardSet.add(12)
stdShardSet.add(15)
exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
hrShardSet := newShardSet()
hrShardSet.add(7)
exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard_2019092416"}, hrShardSet)
makeTxTestDBWithViewsShards(t, holder, idx, exp)
// setup is done
view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx)
PanicOn(err)
// compare against setup
if !view2shard.equals(exp) {
t.Fatalf("expected '%v' but got view2shard '%v'", exp, view2shard)
}
}

90
dbshard_test.go
View file

@ -1,90 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (
"context"
"fmt"
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
m0 := c.GetNode(0)
t.Run("RowIDColumnKey", func(t *testing.T) {
ctx := context.Background()
indexName := c.Idx()
fieldName := "f"
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
if index.CreatedAt() == 0 {
t.Fatal("index createdAt is empty")
}
field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if field.CreatedAt() == 0 {
t.Fatal("field createdAt is empty")
}
rowID := uint64(1)
timestamp := int64(0)
// Generate some keyed records.
rowIDs := []uint64{}
timestamps := []int64{}
for i := 1; i <= 10; i++ {
rowIDs = append(rowIDs, rowID)
timestamps = append(timestamps, timestamp)
}
// Keys are sharded so ordering is not guaranteed.
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0
req := &pilosa.ImportRequest{
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
Field: fieldName,
FieldCreatedAt: field.CreatedAt(),
Shard: 0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
qcx := m0.API.Txf().NewQcx()
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
//select {}
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
// Query node0.
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
keys := res.Results[0].(*pilosa.Row).Keys
if !sameStringSlice(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
}
})
}

43
delete_test.go
View file

@ -113,7 +113,48 @@ func TestExecutor_DeleteRecords(t *testing.T) {
}
require := require.New(t)
t.Run("DeleteRecords", func(t *testing.T) {
t.Run("DeleteRace", func(t *testing.T) {
setup(t, require, c)
defer tearDown(t, require, c)
for i := 0; i < 100; i++ {
// imagine that we start with bsi set to 3 in column 0.
// We then execute two operations:
// (1) we set bsi to 1 in column 0
// (2) we delete everything with bsi > 2
// No matter which order these happen in, we should see
// bsi set to 1 in column 0.
// If the set happens first, the delete doesn't touch it.
// If the set happens second, the delete deletes the previous
// value, then the set happens.
// Let's find out...
c.ImportIntID(t, indexName, "bsi", []test.IntID{
{ID: 0, Val: 3},
})
ch := make(chan error)
go func() {
// Make sure we close the channel even if we're failing out of the test.
defer close(ch)
// We don't actually care whether the delete succeeds or fails...
_ = c.Query(t, indexName, `Delete(Row(bsi>2))`)
// we don't try to handle an error from that, at this time.
}()
c.ImportIntID(t, indexName, "bsi", []test.IntID{
{ID: 0, Val: 1},
})
// wait for the async delete
<-ch
resp := c.Query(t, indexName, `Row(bsi<2)`)
// we expect to always find 0 in this row
row, ok := resp.Results[0].(*pilosa.Row)
if !ok {
t.Fatalf("expected row return")
}
cols := row.Columns()
if len(cols) < 1 || cols[0] != 0 {
t.Fatalf("expected columns including 0, got %d on try %d", cols, i)
}
}
})
t.Run("Delete", func(t *testing.T) {
setup(t, require, c)
defer tearDown(t, require, c)

2
encoding/proto/proto.go
View file

@ -450,7 +450,6 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *
FieldCreatedAt: m.FieldCreatedAt,
Clear: m.Clear,
Action: m.Action,
Block: uint64(m.Block),
Views: views,
UpdateExistence: m.UpdateExistence,
}
@ -1191,7 +1190,6 @@ func (s Serializer) decodeImportRoaringRequest(pb *pb.ImportRoaringRequest, m *p
}
m.Clear = pb.Clear
m.Action = pb.Action
m.Block = int(pb.Block)
m.Views = views
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt

783
executor.go
View file

File diff suppressed because it is too large Load diff

15
executor_internal_test.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/molecula/featurebase/v3/pql"
"github.com/stretchr/testify/require"
)
// AssertEqual checks a given RowIdentifiers against expected values.
@ -51,8 +52,7 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
t.Fatalf("creating index: %v", err)
}
qcx := holder.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := holder.MustIndexQueryContext(t, "i")
fb, errb := idx.CreateField("b", "", OptFieldTypeBool())
_, errbk := idx.CreateField("bk", "", OptFieldTypeBool(), OptFieldKeys())
@ -569,8 +569,7 @@ func TestExecutor_DeleteRows(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
qcx := holder.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := holder.MustIndexQueryContext(t, "i")
if _, err = f.SetBit(qcx, 1, 1, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
@ -580,14 +579,18 @@ func TestExecutor_DeleteRows(t *testing.T) {
if err != nil {
t.Fatalf("failed to read row: %v", err)
}
require.Nil(t, qcx.Commit())
ctx := context.Background()
changed, err := DeleteRows(ctx, row, idx, 0)
// make a new query context for the deletes. we use the same context
// for both, because deleteRows is supposed to handle this.
qcx = holder.MustIndexQueryContext(t, "i")
changed, err := holder.deleteRows(ctx, qcx, row, idx, 0)
if !changed || err != nil {
t.Fatalf("failed to delete row: %v", err)
}
changed, err = DeleteRows(ctx, row, idx, 0)
changed, err = holder.deleteRows(ctx, qcx, row, idx, 0)
if err != nil {
t.Fatalf("deleting rows: %v", err)
}

81
executor_test.go
View file

@ -35,6 +35,8 @@ import (
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
var (
@ -57,6 +59,54 @@ func TestExecutor(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
// Ensure a row query can be executed.
t.Run("QcxError", func(t *testing.T) {
n := c.GetNode(0)
a := n.API
h := c.GetHolder(0)
i, err := h.CreateIndex("i", "", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = i.CreateField("f", "", pilosa.OptFieldTypeInt(0, 1000))
if err != nil {
t.Fatalf("creating field: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
eg, ctx := errgroup.WithContext(ctx)
defer cancel()
eg.Go(func() error {
for i := 0; i < 1000; i++ {
_, err = a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
if err != nil {
return err
}
_, err = a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=3)"})
if err != nil {
return err
}
}
return nil
})
for i := 0; i < 1000; i++ {
res, err := a.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Intersect(Row(f>2),Row(f<2))\nSet(1, f=2)"})
if err != nil {
t.Fatal(err)
}
var row *pilosa.Row
var ok bool
if row, ok = res.Results[0].(*pilosa.Row); !ok {
t.Fatalf("expected row, got %T", res.Results[0])
}
cols := row.Columns()
if len(cols) != 0 {
t.Fatalf("expected empty row, got %d", cols)
}
}
err = eg.Wait()
if err != nil {
t.Fatalf("background errored")
}
})
t.Run("ExecuteRow", func(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {
writeQuery := `` +
@ -1645,9 +1695,8 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
t.Fatal(err)
}
// Obtain transaction.
qcx := hldr.Txf().NewQcx()
defer qcx.Abort()
qcx := hldr.MustIndexQueryContext(t, c.Idx())
defer qcx.Release()
f := hldr.Field(c.Idx(), "f")
if value, exists, err := f.Value(qcx, 10); err != nil {
@ -1716,9 +1765,8 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
t.Fatal(err)
}
// Obtain transaction.
qcx := hldr.Txf().NewQcx()
defer qcx.Abort()
qcx := hldr.MustIndexQueryContext(t, c.Idx())
defer qcx.Release()
f := hldr.Field(c.Idx(), "f")
if value, exists, err := f.Value(qcx, 10); err != nil {
@ -4371,7 +4419,11 @@ func TestExecutor_Execute_All(t *testing.T) {
m0 := c.GetNode(0)
// the request gets altered by the Import operation now...
reqs := req.Clone().SortToShards()
qcx := m0.API.Txf().NewQcx()
shards := make([]uint64, 0, len(reqs))
for shardID := range reqs {
shards = append(shards, shardID)
}
qcx := mustIndexQueryContext(t, m0.API, req.Index, shards...)
for _, r := range reqs {
// we can ignore the key (which is the shard) because each req
// also got its internal key set.
@ -4379,7 +4431,7 @@ func TestExecutor_Execute_All(t *testing.T) {
t.Fatal(err)
}
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
i0, err := m0.API.Index(context.Background(), c.Idx())
PanicOn(err)
@ -4452,11 +4504,12 @@ func TestExecutor_Execute_All(t *testing.T) {
req.ColumnKeys[i] = fmt.Sprintf("c%d", i)
}
qcx := c.GetNode(0).API.Txf().NewQcx()
// note, the shard specified in the import request is wrong.
qcx := mustIndexQueryContext(t, c.GetNode(0).API, req.Index)
if err := c.GetNode(0).API.Import(context.Background(), qcx, req); err != nil {
t.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
tests := []struct {
qry string
@ -4850,11 +4903,11 @@ func benchmarkExistence(nn bool, b *testing.B) {
b.ResetTimer()
nodeAPI := c.GetNode(0).API
for i := 0; i < b.N; i++ {
qcx := nodeAPI.Txf().NewQcx()
qcx := mustIndexQueryContext(b, nodeAPI, req.Index, req.Shard)
if err := nodeAPI.Import(context.Background(), qcx, req); err != nil {
b.Fatal(err)
}
PanicOn(qcx.Finish())
require.Nil(b, qcx.Commit())
}
}
@ -5390,8 +5443,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
t.Fatalf("importing: %v", err)
}
m0 := c.GetNode(0)
qcx := m0.API.Txf().NewQcx()
defer qcx.Abort()
qcx := mustIndexQueryContext(t, m0.API, c.Idx())
var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4
@ -5446,6 +5498,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
}); err != nil {
t.Fatalf("importing: %v", err)
}
require.Nil(t, qcx.Commit())
tests := []struct {
query string

187
field.go
View file

@ -15,7 +15,9 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
@ -719,17 +721,17 @@ func (f *Field) TTL() time.Duration {
return f.options.TTL
}
func (f *Field) bitDepth() (uint64, error) {
func (f *Field) bitDepth(qcx qc.QueryContext) (uint64, error) {
var maxBitDepth uint64
view2shards := f.idx.fieldView2shard.getViewsForField(f.name)
view2shards := f.holder.dbContents[keys.Index(f.index)][keys.Field(f.name)]
for name, shardset := range view2shards {
view := f.view(name)
view := f.view(string(name))
if view == nil {
continue
}
bd, err := view.bitDepth(shardset.shards())
bd, err := view.bitDepth(qcx, shardset.Shards())
if err != nil {
return 0, errors.Wrapf(err, "getting view(%s) bit depth", name)
}
@ -763,14 +765,14 @@ func (f *Field) cacheBitDepth(bd uint64) error {
// openViews opens and initializes the views inside the field.
func (f *Field) openViews() error {
view2shards := f.idx.fieldView2shard.getViewsForField(f.name)
if view2shards == nil {
viewShards := f.holder.dbContents[keys.Index(f.index)][keys.Field(f.name)]
if viewShards == nil {
// no data
return nil
}
for name, shardset := range view2shards {
view := f.newView(f.viewPath(name), name)
for name, shardset := range viewShards {
view := f.newView(f.viewPath(string(name)), string(name))
if err := view.openWithShardSet(shardset); err != nil {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
@ -1052,7 +1054,7 @@ func (f *Field) viewsByTimeRange(from, to time.Time) (views []string, err error)
// RowTime gets the row at the particular time with the granularity specified by
// the quantum.
func (f *Field) RowTime(qcx *Qcx, rowID uint64, time time.Time, quantum string) (*Row, error) {
func (f *Field) RowTime(qcx qc.QueryContext, rowID uint64, time time.Time, quantum string) (*Row, error) {
if !TimeQuantum(quantum).Valid() {
return nil, ErrInvalidTimeQuantum
}
@ -1203,7 +1205,7 @@ func (f *Field) deleteView(name string) error {
// package, and the fact that it's only allowed on
// `set`,`mutex`, and `bool` fields is odd. This may
// be considered for deprecation in a future version.
func (f *Field) Row(qcx *Qcx, rowID uint64) (*Row, error) {
func (f *Field) Row(qcx qc.QueryContext, rowID uint64) (*Row, error) {
switch f.Type() {
case FieldTypeSet, FieldTypeMutex, FieldTypeBool:
view := f.view(viewStandard)
@ -1218,7 +1220,7 @@ func (f *Field) Row(qcx *Qcx, rowID uint64) (*Row, error) {
// mutexCheck performs a sanity-check on the available fragments for a
// field. The return is map[column]map[shard][]values for collisions only.
func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
func (f *Field) MutexCheck(ctx context.Context, qcx qc.QueryContext, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
if f.Type() != FieldTypeMutex {
return nil, errors.New("mutex check only valid for mutex fields")
}
@ -1240,7 +1242,7 @@ func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit in
}
// SetBit sets a bit on a view within the field.
func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed bool, err error) {
func (f *Field) SetBit(qcx qc.QueryContext, rowID, colID uint64, t *time.Time) (changed bool, err error) {
viewName := viewStandard
if !f.options.NoStandardView {
// Retrieve view. Exit if it doesn't exist.
@ -1280,7 +1282,7 @@ func (f *Field) SetBit(qcx *Qcx, rowID, colID uint64, t *time.Time) (changed boo
}
// ClearBit clears a bit within the field.
func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error) {
func (f *Field) ClearBit(qcx qc.QueryContext, rowID, colID uint64) (changed bool, err error) {
viewName := viewStandard
// Retrieve view. Exit if it doesn't exist.
@ -1325,29 +1327,31 @@ func (f *Field) ClearBit(qcx *Qcx, rowID, colID uint64) (changed bool, err error
return changed, nil
}
// ClearBits clears all bits corresponding to the given record IDs in standard
// or BSI views. It does not delete bits from time quantum views.
func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error {
bsig := f.bsiGroup(f.name)
var v *view
if bsig != nil {
// looks like we're a BSI field?
v = f.view(viewBSIGroupPrefix + f.name)
} else {
v = f.view(viewStandard)
}
// it's fine if we never actually created the view, that means the
// bits are all clear!
if v == nil {
return nil
}
frag := v.Fragment(shard)
if frag == nil {
return nil
}
_, err := frag.ClearRecords(tx, recordIDs)
return err
}
// ClearBits is probably unused. Leaving it commented out for now but if things
// still work without it it should go away.
// // ClearBits clears all bits corresponding to the given record IDs in standard
// // or BSI views. It does not delete bits from time quantum views.
// func (f *Field) ClearBits(tx Tx, shard uint64, recordIDs ...uint64) error {
// bsig := f.bsiGroup(f.name)
// var v *view
// if bsig != nil {
// // looks like we're a BSI field?
// v = f.view(viewBSIGroupPrefix + f.name)
// } else {
// v = f.view(viewStandard)
// }
// // it's fine if we never actually created the view, that means the
// // bits are all clear!
// if v == nil {
// return nil
// }
// frag := v.Fragment(shard)
// if frag == nil {
// return nil
// }
// _, err := frag.ClearRecords(tx, recordIDs)
// return err
// }
func groupCompare(a, b string, offset int) (lt, eq bool) {
if len(a) > offset {
@ -1392,7 +1396,7 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) {
// StringValue reads an integer field value for a column, and converts
// it to a string based on a foreign index string key.
func (f *Field) StringValue(qcx *Qcx, columnID uint64) (value string, exists bool, err error) {
func (f *Field) StringValue(qcx qc.QueryContext, columnID uint64) (value string, exists bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return value, false, ErrBSIGroupNotFound
@ -1406,7 +1410,7 @@ func (f *Field) StringValue(qcx *Qcx, columnID uint64) (value string, exists boo
}
// Value reads a field value for a column.
func (f *Field) Value(qcx *Qcx, columnID uint64) (value int64, exists bool, err error) {
func (f *Field) Value(qcx qc.QueryContext, columnID uint64) (value int64, exists bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return 0, false, ErrBSIGroupNotFound
@ -1428,7 +1432,7 @@ func (f *Field) Value(qcx *Qcx, columnID uint64) (value int64, exists bool, err
}
// SetValue sets a field value for a column.
func (f *Field) SetValue(qcx *Qcx, columnID uint64, value int64) (changed bool, err error) {
func (f *Field) SetValue(qcx qc.QueryContext, columnID uint64, value int64) (changed bool, err error) {
// Fetch bsiGroup & validate min/max.
bsig := f.bsiGroup(f.name)
if bsig == nil {
@ -1480,7 +1484,7 @@ func (f *Field) SetValue(qcx *Qcx, columnID uint64, value int64) (changed bool,
}
// ClearValue removes a field value for a column.
func (f *Field) ClearValue(qcx *Qcx, columnID uint64) (changed bool, err error) {
func (f *Field) ClearValue(qcx qc.QueryContext, columnID uint64) (changed bool, err error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return false, ErrBSIGroupNotFound
@ -1500,9 +1504,7 @@ func (f *Field) ClearValue(qcx *Qcx, columnID uint64) (changed bool, err error)
return false, nil
}
func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, error) {
tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: f.idx, Shard: shard})
defer finisher(&err)
func (f *Field) MaxForShard(qcx qc.QueryContext, shard uint64, filter *Row) (ValCount, error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return ValCount{}, ErrBSIGroupNotFound
@ -1518,7 +1520,11 @@ func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro
return ValCount{}, nil
}
max, cnt, err := fragment.max(tx, filter, bsig.BitDepth)
qr, err := fragment.qcxRead(qcx)
if err != nil {
return ValCount{}, err
}
max, cnt, err := fragment.max(qr, filter, bsig.BitDepth)
if err != nil {
return ValCount{}, errors.Wrap(err, "calling fragment.max")
}
@ -1530,9 +1536,7 @@ func (f *Field) MaxForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro
// MinForShard returns the minimum value which appears in this shard
// (this field must be an Int or Decimal field). It also returns the
// number of times the minimum value appears.
func (f *Field) MinForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, error) {
tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: f.idx, Shard: shard})
defer finisher(&err)
func (f *Field) MinForShard(qcx qc.QueryContext, shard uint64, filter *Row) (ValCount, error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return ValCount{}, ErrBSIGroupNotFound
@ -1548,7 +1552,11 @@ func (f *Field) MinForShard(qcx *Qcx, shard uint64, filter *Row) (ValCount, erro
return ValCount{}, nil
}
min, cnt, err := fragment.min(tx, filter, bsig.BitDepth)
qr, err := fragment.qcxRead(qcx)
if err != nil {
return ValCount{}, err
}
min, cnt, err := fragment.min(qr, filter, bsig.BitDepth)
if err != nil {
return ValCount{}, errors.Wrap(err, "calling fragment.min")
}
@ -1590,7 +1598,7 @@ func (f *Field) valCountize(val int64, cnt uint64, bsig *bsiGroup) (ValCount, er
}
// Range performs a conditional operation on Field.
func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Row, error) {
func (f *Field) Range(qcx qc.QueryContext, name string, op pql.Token, predicate int64) (*Row, error) {
// Retrieve and validate bsiGroup.
bsig := f.bsiGroup(name)
if bsig == nil {
@ -1614,7 +1622,7 @@ func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Ro
}
// Import bulk imports data.
func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) {
func (f *Field) Import(qcx qc.QueryContext, rowIDs, columnIDs []uint64, timestamps []int64, shard uint64, options *ImportOptions) (err0 error) {
// Determine quantum if timestamps are set.
q := f.TimeQuantum()
if len(timestamps) > 0 {
@ -1636,12 +1644,6 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64,
}
}
}
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: f.idx, Shard: shard})
if err != nil {
return errors.Wrap(err, "qcx.GetTx")
}
var err1 error
defer finisher(&err1)
view, err := f.createViewIfNotExists(viewStandard)
if err != nil {
return errors.Wrapf(err, "creating view %s", viewStandard)
@ -1652,8 +1654,11 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64,
return errors.Wrap(err, "creating fragment")
}
err1 = frag.bulkImport(tx, rowIDs, columnIDs, options)
return err1
qw, err := frag.qcxWrite(qcx)
if err != nil {
return err
}
return frag.bulkImport(qw, rowIDs, columnIDs, options)
}
fieldType := f.Type()
@ -1717,12 +1722,10 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64,
}
}
}
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: f.idx, Shard: shard})
if err != nil {
return errors.Wrap(err, "qcx.GetTx")
}
var err1 error
defer finisher(&err1)
// in the Qcx/Tx era, we grabbed a single top-level Tx, because we secretly
// knew that Tx were shard-based, and didn't care about views. Now we request
// a QueryWrite per frag.
for viewName, data := range views {
view, err := f.createViewIfNotExists(viewName)
if err != nil {
@ -1734,9 +1737,13 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64,
return errors.Wrap(err, "creating fragment")
}
err1 = frag.bulkImport(tx, data.RowIDs, data.ColumnIDs, options)
if err1 != nil {
return err1
qw, err := frag.qcxWrite(qcx)
if err != nil {
return err
}
err = frag.bulkImport(qw, data.RowIDs, data.ColumnIDs, options)
if err != nil {
return err
}
}
return nil
@ -1745,7 +1752,7 @@ func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []int64,
// importFloatValue imports floating point values. In current usage, this
// should only ever be called with data for a single shard; the API calls
// around this are splitting it up per shard.
func (f *Field) importFloatValue(qcx *Qcx, columnIDs []uint64, values []float64, shard uint64, options *ImportOptions) error {
func (f *Field) importFloatValue(qcx qc.QueryContext, columnIDs []uint64, values []float64, shard uint64, options *ImportOptions) error {
// convert values to int64 values based on scale
ivalues := make([]int64, len(values))
bsig := f.bsiGroup(f.name)
@ -1763,7 +1770,7 @@ func (f *Field) importFloatValue(qcx *Qcx, columnIDs []uint64, values []float64,
// importTimestampValue imports timestamp values. In current usage, this
// should only ever be called with data for a single shard; the API calls
// around this are splitting it up per shard.
func (f *Field) importTimestampValue(qcx *Qcx, columnIDs []uint64, values []time.Time, shard uint64, options *ImportOptions) error {
func (f *Field) importTimestampValue(qcx qc.QueryContext, columnIDs []uint64, values []time.Time, shard uint64, options *ImportOptions) error {
ivalues := make([]int64, len(values))
bsig := f.bsiGroup(f.name)
if bsig == nil {
@ -1779,7 +1786,7 @@ func (f *Field) importTimestampValue(qcx *Qcx, columnIDs []uint64, values []time
// importValue bulk imports range-encoded value data. This function should
// only be called with data for a single shard; the API calls that wrap
// this handle splitting the data up per-shard.
func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, shard uint64, options *ImportOptions) (err0 error) {
func (f *Field) importValue(qcx qc.QueryContext, columnIDs []uint64, values []int64, shard uint64, options *ImportOptions) error {
// no data to import
if len(columnIDs) == 0 {
return nil
@ -1870,19 +1877,15 @@ func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, shard
}
}
// now we know which shard we discovered.
tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: f.idx, Shard: frag.shard})
// request a QueryWrite and write to it
qw, err := frag.qcxWrite(qcx)
if err != nil {
return err
}
// defer the finisher, so it will check the error returned and
// possibly rollback.
defer finisher(&err0)
return frag.importValue(tx, columnIDs, values, requiredDepth, options.Clear)
return frag.importValue(qw, columnIDs, values, requiredDepth, options.Clear)
}
func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, clear bool) error {
func (f *Field) importRoaring(ctx context.Context, qcx qc.QueryContext, data []byte, shard uint64, viewName string, clear bool) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaring")
defer span.Finish()
@ -1899,7 +1902,11 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin
if err != nil {
return errors.Wrap(err, "creating fragment")
}
if err := frag.importRoaring(ctx, tx, data, clear); err != nil {
qw, err := frag.qcxWrite(qcx)
if err != nil {
return err
}
if err := frag.importRoaring(ctx, qw, data, clear); err != nil {
return err
}
@ -1910,7 +1917,7 @@ func (f *Field) GetIndex() *Index {
return f.idx
}
func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error {
func (f *Field) importRoaringOverwrite(ctx context.Context, qcx qc.QueryContext, data []byte, shard uint64, viewName string) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite")
defer span.Finish()
@ -1927,7 +1934,11 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
if err != nil {
return errors.Wrap(err, "creating fragment")
}
if err := frag.importRoaringOverwrite(ctx, tx, data, block); err != nil {
qw, err := frag.qcxWrite(qcx)
if err != nil {
return errors.Wrap(err, "creating writer")
}
if err := frag.importRoaringOverwrite(ctx, qw, data); err != nil {
return err
}
@ -1935,8 +1946,12 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
// field.options.BitDepth and bsiGroup.BitDepth based on the imported data.
switch f.Options().Type {
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
qr, err := frag.qcxRead(qcx)
if err != nil {
return err
}
frag.mu.Lock()
maxRowID, _, err := frag.maxRow(tx, nil)
maxRowID, _, err := frag.maxRow(qr, nil)
frag.mu.Unlock()
if err != nil {
return err
@ -2368,7 +2383,7 @@ func CheckEpochOutOfRange(epoch, min, max time.Time) error {
return nil
}
func (f *Field) SortShardRow(tx Tx, shard uint64, filter *Row, sort_desc bool) (*SortedRow, error) {
func (f *Field) SortShardRow(qcx qc.QueryContext, shard uint64, filter *Row, sort_desc bool) (*SortedRow, error) {
bsig := f.bsiGroup(f.name)
if bsig == nil {
return nil, errors.New("bsig is nil")
@ -2384,5 +2399,9 @@ func (f *Field) SortShardRow(tx Tx, shard uint64, filter *Row, sort_desc bool) (
return nil, errors.New("fragment is nil")
}
return fragment.sortBsiData(tx, filter, bsig.BitDepth, sort_desc)
qr, err := fragment.qcxRead(qcx)
if err != nil {
return nil, err
}
return fragment.sortBsiData(qr, filter, bsig.BitDepth, sort_desc)
}

138
field_internal_test.go
View file

@ -2,7 +2,6 @@
package pilosa
import (
"context"
"fmt"
"math"
"reflect"
@ -12,9 +11,10 @@ import (
"time"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/shardwidth"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
"github.com/stretchr/testify/require"
)
// CorruptAMutex breaks a mutex in order to test the mutex-corruption stuff.
@ -25,7 +25,7 @@ import (
//
// This always sets row 3 in column 0 of each shard it finds. Populate the
// field with existing shards first.
func CorruptAMutex(tb testing.TB, field *Field, qcx *Qcx) {
func CorruptAMutex(tb testing.TB, field *Field, qcx qc.QueryContext) {
v := field.view(viewStandard)
if v == nil {
tb.Fatalf("creating view failed")
@ -33,14 +33,13 @@ func CorruptAMutex(tb testing.TB, field *Field, qcx *Qcx) {
frags := v.allFragments()
for _, frag := range frags {
func() {
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: field.idx, Shard: frag.shard})
defer finisher(&err)
qw, err := frag.qcxWrite(qcx)
if err != nil {
tb.Fatalf("getting tx: %v", err)
}
// set a bonus bit, bypassing the mutex handling
frag.mu.Lock()
_, err = frag.unprotectedSetBit(tx, 3, (frag.shard<<shardwidth.Exponent)+1)
_, err = frag.unprotectedSetBit(qw, 3, (frag.shard<<shardwidth.Exponent)+1)
frag.mu.Unlock()
if err != nil {
tb.Fatalf("setting bit: %v", err)
@ -232,26 +231,25 @@ func TestField_DeleteView(t *testing.T) {
// reopens it using its cached schema, and returns the corresponding
// field data structure from the reopened index.
func reopenTestField(t testing.TB, f *Field) (*Field, error) {
index := f.index
name := f.Name()
if err := f.idx.Close(); err != nil {
f.idx = nil
return nil, err
require.Nil(t, f.holder.Close())
require.Nil(t, f.holder.Open())
idx := f.holder.Index(index)
if idx == nil {
t.Fatalf("index %q disappeared during reopen", index)
}
schema, err := f.holder.Schemator.Schema(context.Background())
if err != nil {
return nil, err
f = idx.Field(name)
if f == nil {
t.Fatalf("field %q/%q disappeared during reopen", index, name)
}
if err := f.idx.OpenWithSchema(schema[f.idx.name]); err != nil {
f.idx = nil
return nil, err
}
return f.idx.Field(name), nil
return f, nil
}
// testFieldSetBit sets a bit and checks for an error, using a provided qcx and an
// optional timestamp or series of timestamps. if multiple times are provided,
// the underlying set bit operation is repeated for all of them.
func testFieldSetBit(tb testing.TB, qcx *Qcx, f *Field, row, col uint64, ts ...time.Time) {
func testFieldSetBit(tb testing.TB, qcx qc.QueryContext, f *Field, row, col uint64, ts ...time.Time) {
if len(ts) == 0 {
_, err := f.SetBit(qcx, row, col, nil)
if err != nil {
@ -309,12 +307,9 @@ func TestField_SetTimeQuantum(t *testing.T) {
}
func TestField_RowTime(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0"))
// Obtain transaction.
qcx := f.holder.Txf().NewWritableQcx()
defer qcx.Abort()
h, idx, f := newTestField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0"))
qcx := h.MustIndexQueryContext(t, idx.name)
testFieldSetBit(t, qcx, f, 1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
testFieldSetBit(t, qcx, f, 1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
testFieldSetBit(t, qcx, f, 1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
@ -323,14 +318,9 @@ func TestField_RowTime(t *testing.T) {
// Warning: Right now this is misleading, and doesn't really do anything. We
// already committed each change as we got there. SOME DAY we will fix this.
PanicOn(qcx.Finish())
require.Nil(t, qcx.Commit())
qcx = f.holder.Txf().NewQcx()
defer qcx.Abort()
// obtain 2nd transaction to read it back.
qcx = f.holder.Txf().NewQcx()
defer qcx.Abort()
qcx = h.MustQueryContext(t)
if r, err := f.RowTime(qcx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
t.Fatal(err)
@ -453,10 +443,7 @@ func TestField_ApplyOptions(t *testing.T) {
// into consideration. This would cause an import of 1/8/1
// to result in a value of 9 instead of 1.
func TestBSIGroup_importValue(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeInt(-100, 200))
qcx := f.idx.holder.txf.NewQcx()
defer qcx.Abort()
h, idx, f := newTestField(t, OptFieldTypeInt(-100, 200))
options := &ImportOptions{}
for i, tt := range []struct {
@ -484,24 +471,24 @@ func TestBSIGroup_importValue(t *testing.T) {
[]uint64{100},
},
} {
qcx := h.MustIndexQueryContext(t, idx.name)
if err := f.importValue(qcx, tt.columnIDs, tt.values, 0, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
PanicOn(qcx.Finish())
qcx.Reset()
require.Nil(t, qcx.Commit())
qcx = h.MustIndexQueryContext(t, idx.name)
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
PanicOn(qcx.Finish())
qcx.Reset()
require.Nil(t, qcx.Commit())
} // loop
}
// benchmarkImportValues is a helper function to explore, very roughly, the cost
// of setting values using the special setter used for imports.
func benchmarkFieldImportValues(b *testing.B, qcx *Qcx, bitDepth uint64, f *Field, cfunc func(uint64) uint64) {
func benchmarkFieldImportValues(b *testing.B, qcx qc.QueryContext, bitDepth uint64, f *Field, cfunc func(uint64) uint64) {
batches := makeBenchmarkImportValueData(b, bitDepth, cfunc)
for _, req := range batches {
// NOTE: We assume everything's in Shard 0 for now.
@ -517,10 +504,10 @@ func BenchmarkField_ImportValue(b *testing.B) {
depths := []uint64{4, 8, 16, 32}
for _, bitDepth := range depths {
_, _, f := newTestField(b, OptFieldTypeInt(0, 1<<bitDepth))
h, idx, f := newTestField(b, OptFieldTypeInt(0, 1<<bitDepth))
qcx := f.idx.holder.txf.NewQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(b, idx.name)
defer qcx.Release()
name := fmt.Sprintf("Depth%d", bitDepth)
b.Run(name+"_Sparse", func(b *testing.B) {
benchmarkFieldImportValues(b, qcx, bitDepth, f, func(u uint64) uint64 { return (u + 19) & (ShardWidth - 1) })
@ -532,10 +519,7 @@ func BenchmarkField_ImportValue(b *testing.B) {
}
func TestIntField_MinMaxForShard(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeInt(-100, 200))
qcx := f.idx.holder.txf.NewQcx()
defer qcx.Abort()
h, idx, f := newTestField(t, OptFieldTypeInt(-100, 200))
options := &ImportOptions{}
for i, test := range []struct {
@ -580,11 +564,13 @@ func TestIntField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
qcx := h.MustIndexQueryContext(t, idx.name)
if err := f.importValue(qcx, test.columnIDs, test.values, 0, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
PanicOn(qcx.Finish())
qcx.Reset()
require.Nil(t, qcx.Commit())
qcx = h.MustQueryContext(t)
shard := uint64(0)
// Rollback below manually, because we are in a loop.
@ -689,7 +675,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
}
func TestDecimalField_MinMaxForShard(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeDecimal(3))
h, idx, f := newTestField(t, OptFieldTypeDecimal(3))
options := &ImportOptions{}
for i, test := range []struct {
@ -734,19 +720,18 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
qcx := f.idx.holder.txf.NewQcx()
qcx := h.MustIndexQueryContext(t, idx.name)
err := f.importFloatValue(qcx, test.columnIDs, test.values, 0, options)
if err != nil {
qcx.Abort()
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
qcx.Abort()
require.Nil(t, qcx.Commit())
shard := uint64(0)
qcx = f.idx.holder.txf.NewQcx()
defer qcx.Abort()
qcx = h.MustQueryContext(t)
maxvc, err := f.MaxForShard(qcx, shard, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)
@ -767,10 +752,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
}
func TestBSIGroup_TxReopenDB(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeInt(-100, 200))
qcx := f.idx.holder.txf.NewQcx()
defer qcx.Abort()
h, idx, f := newTestField(t, OptFieldTypeInt(-100, 200))
options := &ImportOptions{}
for i, tt := range []struct {
@ -798,19 +780,21 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
[]uint64{100},
},
} {
if err := f.importValue(qcx, tt.columnIDs, tt.values, 0, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
PanicOn(qcx.Finish())
qcx.Reset()
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
PanicOn(qcx.Finish())
qcx.Reset()
func() {
qcx := h.MustIndexQueryContext(t, idx.name)
defer qcx.Release()
if err := f.importValue(qcx, tt.columnIDs, tt.values, 0, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
require.Nil(t, qcx.Commit())
qcx = h.MustQueryContext(t)
defer qcx.Release()
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
}()
} // loop
// the test: can we re-open a BSI fragment under Tx store
@ -822,15 +806,14 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
// Ensure that an integer field has the same BitDepth after reopening.
func TestField_SaveMeta(t *testing.T) {
_, _, f := newTestField(t, OptFieldTypeInt(-10, 1000))
h, idx, f := newTestField(t, OptFieldTypeInt(-10, 1000))
colID := uint64(1)
val := int64(88)
expBitDepth := uint64(7)
// Obtain transaction.
qcx := f.holder.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.name)
if changed, err := f.SetValue(qcx, colID, val); err != nil {
t.Fatal(err)
@ -850,9 +833,7 @@ func TestField_SaveMeta(t *testing.T) {
t.Fatalf("expected value to be: %d, got: %d", val, rslt)
}
if err := qcx.Finish(); err != nil {
t.Fatalf("error finishing qcx: %v", err)
}
require.Nil(t, qcx.Commit())
// Reload field and verify that it is persisted.
f, err := reopenTestField(t, f)
@ -860,8 +841,7 @@ func TestField_SaveMeta(t *testing.T) {
t.Fatal(err)
}
qcx = f.holder.Txf().NewQcx()
defer qcx.Abort()
qcx = h.MustQueryContext(t)
if f.options.BitDepth != expBitDepth {
t.Fatalf("expected BitDepth after reopen to be: %d, got: %d", expBitDepth, f.options.BitDepth)

21
field_test.go
View file

@ -26,8 +26,7 @@ func TestField_SetValue(t *testing.T) {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// You're going to note the lack of any commits here. That's
// because, when you have a writable Qcx, *every individual
// sub-transaction commits immediately*. In theory, we ought
@ -67,8 +66,7 @@ func TestField_SetValue(t *testing.T) {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set value.
if changed, err := f.SetValue(qcx, 100, 21); err != nil {
@ -102,8 +100,7 @@ func TestField_SetValue(t *testing.T) {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set value.
if _, err := f.SetValue(qcx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
@ -118,8 +115,7 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set value.
if _, err := f.SetValue(qcx, 100, 15); !errors.Is(err, pilosa.ErrBSIGroupValueTooLow) {
t.Fatalf("unexpected error: %s", err)
@ -134,8 +130,7 @@ func TestField_SetValue(t *testing.T) {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set value.
if _, err := f.SetValue(qcx, 100, 31); !errors.Is(err, pilosa.ErrBSIGroupValueTooHigh) {
@ -193,8 +188,7 @@ func TestField_AvailableShards(t *testing.T) {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set values on shards 0 & 2, and verify.
if _, err := f.SetBit(qcx, 0, 100, nil); err != nil {
@ -233,8 +227,7 @@ func TestField_ClearValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, idx.Name())
// Set value on field.
if changed, err := f.SetValue(qcx, 100, 21); err != nil {

753
fragment.go
View file

File diff suppressed because it is too large Load diff

1547
fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

1
handler.go
View file

@ -448,7 +448,6 @@ type ImportRoaringRequest struct {
FieldCreatedAt int64
Clear bool
Action string // [set, clear, overwrite]
Block int
Views map[string][]byte
UpdateExistence bool
SuppressLog bool

356
holder.go
View file

@ -14,13 +14,15 @@ import (
"github.com/molecula/featurebase/v3/dax"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/logger"
qc "github.com/molecula/featurebase/v3/querycontext"
rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/task"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -127,7 +129,7 @@ type Holder struct {
Auditor testhook.Auditor
txf *TxFactory
txStore qc.TxStore
lookupDB *sql.DB
@ -146,6 +148,35 @@ type Holder struct {
// snapshotter/writelogger; then MDS should only start directing queries to
// that computer once it has completed applying the snapshot.
directiveApplied bool
dbContents keys.DBContents // used during startup to determine which views to open, etc
}
func (h *Holder) NewQueryContext(ctx context.Context) (qc.QueryContext, error) {
return h.TxStore().NewQueryContext(ctx)
}
func (h *Holder) NewWriteQueryContext(ctx context.Context, scope qc.QueryScope) (qc.QueryContext, error) {
return h.TxStore().NewWriteQueryContext(ctx, scope)
}
// NewIndexQueryContext is a helper to create a scope for a given index, and
// optional list of shards. If no shards are provided, the context is
// index-wide.
func (h *Holder) NewIndexQueryContext(ctx context.Context, index string, shards ...uint64) (qc.QueryContext, error) {
txs := h.TxStore()
var typeShifted []keys.Shard
// helpfully treat a shard of -1 as no shard
if len(shards) > 0 && shards[0] == ^uint64(0) {
shards = shards[1:]
}
if len(shards) > 0 {
typeShifted = make([]keys.Shard, len(shards))
for i, v := range shards {
typeShifted[i] = keys.Shard(v)
}
}
return txs.NewWriteQueryContext(ctx, txs.Scope().AddIndexShards(keys.Index(index), typeShifted...))
}
// HolderOpts holds information about the holder which other things might want
@ -166,6 +197,16 @@ func (h *Holder) Directive() dax.Directive {
return *h.directive
}
// TxStore yields the backing TxStore used by this holder. If no TxStore is
// set, it yields a NopTxStore which errors out on usage, rather than panicing.
func (h *Holder) TxStore() qc.TxStore {
if h.txStore != nil {
return h.txStore
}
// if you somehow didn't pick a TxStore, we want to error peacefully rather than panicing
return qc.NopTxStore
}
func (h *Holder) SetDirective(d *dax.Directive) {
if d == nil {
return
@ -303,7 +344,7 @@ func TestHolderConfig() *HolderConfig {
}
// NewHolder returns a new instance of Holder for the given path.
func NewHolder(path string, cfg *HolderConfig) *Holder {
func NewHolder(path string, cfg *HolderConfig) (*Holder, error) {
if cfg == nil {
cfg = DefaultHolderConfig()
}
@ -335,6 +376,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
Schemator: cfg.Schemator,
Logger: cfg.Logger,
Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend},
txStore: qc.NopTxStore,
Auditor: NewAuditor(),
@ -343,12 +385,8 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
indexes: make(map[string]*Index),
}
txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h)
vprint.PanicOn(err)
h.txf = txf
_ = testhook.Created(h.Auditor, h, nil)
return h
return h, nil
}
// Path returns the path directory the holder was created with.
@ -361,6 +399,228 @@ func (h *Holder) IndexesPath() string {
return filepath.Join(h.path, IndexesDir)
}
// transactExistRow atomically grabs a currently-unused row of the existence
// field to store some bits in. These bits are used to denote records which
// we are in the process of deleting. By "atomically" we mean that this
// operation gets its own QueryContext, which it commits before returning.
// You cannot use this while you already have a live QueryContext referring
// to this shard.
func (h *Holder) transactExistRow(ctx context.Context, qcx qc.QueryContext, idx *Index, shard uint64, frag *fragment, src *Row) (uint64, error) {
qw, err := frag.qcxWrite(qcx)
if err != nil {
return 0, err
}
rows, err := frag.rows(ctx, qw, 1)
if err != nil {
return 0, err
}
// obtain a rowID which is higher than any currently present row ID.
rowID := uint64(1)
if len(rows) > 0 {
rowID = rows[len(rows)-1] + 1
}
_, err = frag.setRow(qw, src, rowID)
if err != nil {
return 0, err
}
return rowID, qcx.Flush(keys.Index(idx.name), keys.Shard(shard))
}
// deleteRows deletes the everything from the given index/shard matching a provided
// Row.
func (h *Holder) deleteRows(ctx context.Context, qcx qc.QueryContext, src *Row, idx *Index, shard uint64) (bool, error) {
return h.deleteRowsWithFlow(ctx, qcx, src, idx, shard, false)
}
// deleteRowsWithFlowWithKeys deletes the given columns from every field, for the given
// index/shard. The "normalFlow" parameter tells whether we're trying to do a recovery
// of an interrupted delete.
func (h *Holder) deleteRowsWithFlowWithKeys(ctx context.Context, qcx qc.QueryContext, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (bool, error) {
var existenceFragment *fragment
var deletedRowID uint64
var commitor Commitor = &NopCommitor{}
var err error // store columns in exits field ToBeDelete row commited
if normalFlow { // normalFlow is the standard path, "not normal" is recoverory
existenceFragment = h.fragment(idx.Name(), existenceFieldName, viewStandard, shard)
if existenceFragment == nil {
// no exists field
return false, errors.New("can't bulk delete without existence field")
}
src := NewRowFromBitmap(columns)
deletedRowID, err = h.transactExistRow(ctx, qcx, idx, shard, existenceFragment, src)
if err != nil {
return false, err
}
}
commitor, err = deleteKeyTranslation(ctx, idx, shard, columns)
if err != nil {
return false, err
}
changed := false
defer func() {
// if there is an error on the bit clearing rollback the keys
if err != nil {
changed = false
commitor.Rollback()
return
}
// if there is an error in the key commit, then rollback the delete
// write records before keys to remove possiblity of unmatch keys=records
err = qcx.Flush(keys.Index(idx.name), keys.Shard(shard))
if err != nil {
changed = false
commitor.Rollback()
return
}
if er := commitor.Commit(); er != nil {
err = er
}
if err != nil {
h.Logger.Errorf("problems committing delete in rbf %v shard %v", err, shard)
}
}()
for _, field := range idx.Fields() {
for _, view := range field.views() {
frag := view.Fragment(shard)
if frag == nil {
continue
}
qw, err := frag.qcxWrite(qcx)
if err != nil {
return false, err
}
c, err := frag.clearRecordsByBitmap(qw, columns)
if err != nil {
return false, err
}
changed = changed || c
}
}
if existenceFragment == nil {
return changed, nil
}
// a string keys have been deleted and the deleteRow was created
qw, err := existenceFragment.qcxWrite(qcx)
if err != nil {
return false, err
}
if normalFlow {
existenceFragment.clearRow(qw, deletedRowID)
} else {
// this is if we are recovering from failure and cleaning up
rows, err := existenceFragment.rows(ctx, qw, 1)
if err != nil {
return false, err
}
for _, rowId := range rows {
existenceFragment.clearRow(qw, rowId)
}
}
// Unlike other operations, Delete wants to ensure that its operations are flushed.
return changed, qcx.Flush(keys.Index(idx.name), keys.Shard(shard))
}
// deleteRowsWithOutKeys deletes the data for a given index/shard, matching the columns
// in the given bitmap, but does not attempt to delete corresponding keys.
func (h *Holder) deleteRowsWithOutKeysFlow(ctx context.Context, qcx qc.QueryContext, columns *roaring.Bitmap, idx *Index, shard uint64, normalFlow bool) (changed bool, err error) {
var existenceFragment *fragment
var deletedRowID uint64
var commitor Commitor = &NopCommitor{}
defer func() {
// if there is an error in the key commit, then rollback the delete
// write records before keys to remove possiblity of unmatch keys=records
err := qcx.Flush(keys.Index(idx.name), keys.Shard(shard))
if err != nil {
changed = false
commitor.Rollback()
return
}
}()
for _, field := range idx.Fields() {
for _, view := range field.views() {
frag := view.Fragment(shard)
if frag == nil {
continue
}
qw, err := frag.qcxWrite(qcx)
if err != nil {
return false, err
}
c, err := frag.clearRecordsByBitmap(qw, columns)
if err != nil {
return false, err
}
if c {
changed = true
}
}
}
if existenceFragment == nil { // a string keys have been deleted and the deleteRow was created
return changed, nil
}
qw, err := existenceFragment.qcxWrite(qcx)
if err != nil {
return false, err
}
if normalFlow {
existenceFragment.clearRow(qw, deletedRowID)
return changed, nil
}
// this is if we are recovering from failure and cleaning up
rows, err := existenceFragment.rows(ctx, qw, 1)
if err != nil {
return false, err
}
for _, rowId := range rows {
existenceFragment.clearRow(qw, rowId)
}
return changed, nil
}
// deleteRowsWithFlow deletes all the entries from the index for a given
// index/shard. Note that we expect the source row to have only one
// segment, which is the right one.
func (h *Holder) deleteRowsWithFlow(ctx context.Context, qcx qc.QueryContext, src *Row, idx *Index, shard uint64, normalFlow bool) (change bool, err error) {
if len(src.Segments) == 0 { // nothing to remove
return false, nil
}
if src.Segments[0].shard != shard {
return false, fmt.Errorf("data mismatch: expected data to delete for shard %d, got shard %d", shard, src.Segments[0].shard)
}
columns := src.Segments[0].data // should only be one segment
if columns.Count() == 0 {
return false, nil
}
bits := src.Segments[0].data.Slice()
min := func(a, b int) int {
if a <= b {
return a
}
return b
}
// We may not be able to delete all of the keys at once, so we have to batch
// them.
limit := h.cfg.RBFConfig.MaxDelete
var anyChanges bool
for i := 0; i < len(bits); i += limit {
batch := roaring.NewBitmap(bits[i:min(i+limit, len(bits))]...)
if idx.Keys() {
change, err = h.deleteRowsWithFlowWithKeys(ctx, qcx, batch, idx, shard, normalFlow)
} else {
change, err = h.deleteRowsWithOutKeysFlow(ctx, qcx, batch, idx, shard, normalFlow)
}
anyChanges = anyChanges || change
if err != nil {
return anyChanges, err
}
}
return anyChanges, err
}
func (h *Holder) deletePerShard(index *Index, shard uint64) error {
inprocessRecords := NewRow()
@ -369,11 +629,18 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error {
return nil
}
tx := h.Txf().NewTx(Txo{Write: !writable, Index: index, Shard: shard})
defer tx.Rollback()
qcx, err := h.NewIndexQueryContext(context.TODO(), index.name, shard)
if err != nil {
return err
}
defer qcx.Release()
qr, err := frag.qcxRead(qcx)
if err != nil {
return err
}
// filter rows based on having _exists>=1, which is used to flag delete in-flight
rows, err := frag.rows(context.Background(), tx, 1)
rows, err := frag.rows(context.Background(), qr, 1)
if err != nil {
return err
}
@ -384,7 +651,7 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error {
}
for _, record := range rows {
row, err2 := frag.row(tx, record)
row, err2 := frag.row(qr, record)
if err2 != nil {
return fmt.Errorf("getting row IDs: %v", err2)
}
@ -392,9 +659,7 @@ func (h *Holder) deletePerShard(index *Index, shard uint64) error {
}
h.Logger.Printf("retrying delete: index=%v shard=%v record count=%v", index.name, shard, inprocessRecords.Count())
tx.Rollback() // release the read tx in case a checksum is needed in DeleteRows
_, err = DeleteRows(context.Background(), inprocessRecords, index, shard)
_, err = h.deleteRows(context.Background(), qcx, inprocessRecords, index, shard)
if err != nil {
return fmt.Errorf("deleting rows: %v", err)
}
@ -439,12 +704,27 @@ func (h *Holder) Open() error {
h.opening = true
defer func() { h.opening = false }()
if h.txf == nil {
txf, err := NewTxFactory(h.cfg.StorageConfig.Backend, h.IndexesPath(), h)
if err != nil {
return errors.Wrap(err, "Holder.Open NewTxFactory()")
}
h.txf = txf
// allow overwriting a NopTxStore, but not a real one
if h.txStore != nil && h.txStore != qc.NopTxStore {
return errors.New("holder already had previous TxStore on open")
}
var workerPool *task.Pool
// in production this can almost certainly never be nil. with test
// holders, it is often nil and there's no worker pool to worry about.
// since the worker pool is used only to notify the worker pool that
// we're blocked, that's probably harmless.
if h.executor != nil {
workerPool = h.executor.workers
}
txs, err := qc.NewRBFTxStore(h.IndexesPath(), h.cfg.RBFConfig, h.Logger, workerPool, nil)
if err != nil {
return err
}
h.txStore = txs
h.dbContents, err = h.txStore.Contents()
if err != nil {
return errors.Wrap(err, "obtaining existing fields/views from store")
}
// Reset closing in case Holder is being reopened.
@ -506,7 +786,7 @@ func (h *Holder) Open() error {
err = index.OpenWithSchema(idx)
if err != nil {
_ = h.txf.Close()
_ = h.txStore.Close()
if err == ErrName {
h.Logger.Errorf("opening index: %s, err=%s", index.Name(), err)
continue
@ -532,10 +812,6 @@ func (h *Holder) Open() error {
_ = testhook.Opened(h.Auditor, h, nil)
if err := h.txf.Open(); err != nil {
return errors.Wrap(err, "Holder.Open h.txf.Open()")
}
if h.cfg.LookupDBDSN != "" {
h.Logger.Printf("connecting to lookup database")
@ -624,10 +900,6 @@ func (h *Holder) Close() error {
return nil
}
if globalUseStatTx {
fmt.Printf("%v\n", globalCallStats.report())
}
h.Stats.Close()
// Notify goroutines of closing and wait for completion.
@ -638,15 +910,16 @@ func (h *Holder) Close() error {
return errors.Wrap(err, "closing index")
}
}
if err := h.txf.Close(); err != nil {
return errors.Wrap(err, "holder.Txf.Close()")
if err := h.txStore.Close(); err != nil {
return errors.Wrap(err, "closing database backend")
}
// set txStore to something that errors harmlessly, since it's closed now.
h.txStore = qc.NopTxStore
if err := h.ida.Close(); err != nil {
return errors.Wrap(err, "closing ID allocator")
}
// Reset opened in case Holder needs to be reopened.
h.txf = nil
h.opened.mu.Lock()
h.opened.ch = make(chan struct{})
h.opened.mu.Unlock()
@ -1047,7 +1320,6 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar
index.trackExistence = cim.Meta.TrackExistence
index.createdAt = cim.CreatedAt
index.translatePartitions = translatePartitions
if err = index.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
}
@ -1190,8 +1462,8 @@ func (h *Holder) deleteIndex(name string) error {
}
// remove any backing store.
if err := h.txf.DeleteIndex(name); err != nil {
return errors.Wrap(err, "h.Txf.DeleteIndex")
if err := h.txStore.DeleteIndex(keys.Index(name)); err != nil {
return errors.Wrap(err, "deleting index")
}
// Delete index directory.
@ -1745,18 +2017,6 @@ func (h *Holder) addIndex(idx *Index) {
h.imu.Unlock()
}
func (h *Holder) Txf() *TxFactory {
h.mu.Lock()
defer h.mu.Unlock()
return h.txf
}
// BeginTx starts a transaction on the holder. The index and shard
// must be specified.
func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) {
return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
}
func decodeCreateIndexMessage(ser Serializer, b []byte) (*CreateIndexMessage, error) {
var cim CreateIndexMessage
if err := ser.Unmarshal(b, &cim); err != nil {

42
holder_internal_test.go
View file

@ -2,9 +2,41 @@
package pilosa
import (
"context"
"testing"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/stretchr/testify/require"
)
// MustIndexQueryContext gets a query context which can write to
// the specified index (and possibly shards), or fails the test.
// The qcx will be automatically cleaned up when the test completes.
func (h *Holder) MustIndexQueryContext(tb testing.TB, index string, shards ...uint64) qc.QueryContext {
tb.Helper()
// disregard a leading ^0, because that's idiomatic for "all shards"
if len(shards) > 0 && shards[0] == ^uint64(0) {
shards = shards[1:]
}
qcx, err := h.NewIndexQueryContext(context.Background(), index, shards...)
if err != nil {
tb.Fatalf("creating query context: %v", err)
}
tb.Cleanup(qcx.Release)
return qcx
}
// MustQueryContext gets a read-only query context or fails the test.
func (h *Holder) MustQueryContext(tb testing.TB) qc.QueryContext {
tb.Helper()
qcx, err := h.NewQueryContext(context.Background())
if err != nil {
tb.Fatalf("creating query context: %v", err)
}
tb.Cleanup(qcx.Release)
return qcx
}
func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*Index, *Field) {
idx, err := h.CreateIndexIfNotExists(indexName, "", IndexOptions{TrackExistence: true})
if err != nil {
@ -16,8 +48,7 @@ func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*In
}
existencefield := idx.existenceFld
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx := h.MustIndexQueryContext(t, indexName)
for _, r := range rowCol {
_, err = f.SetBit(qcx, r.row, r.col, nil)
@ -31,9 +62,7 @@ func setupTest(t *testing.T, h *Holder, rowCol []rowCols, indexName string) (*In
}
}
if err = qcx.Finish(); err != nil {
t.Fatalf("failed to commit tx for index %v: %v", indexName, err)
}
require.Nil(t, qcx.Commit())
shardsFound := idx.AvailableShards(includeRemote).Slice()
if len(shardsFound) != 3 {
@ -76,8 +105,7 @@ func TestHolder_ProcessDeleteInflight(t *testing.T) {
for _, test := range tests {
func() {
idx, f := test.idx, test.f
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx := h.MustQueryContext(t)
for _, r := range rowCol {
row, err := f.Row(qcx, r.row)
if err != nil {

5
holder_test.go
View file

@ -162,7 +162,10 @@ func TestHolder_HasData(t *testing.T) {
// Note that we are intentionally not using test.NewHolder,
// because we want to create a Holder object with an invalid path,
// rather than creating a valid holder with a temporary path.
h := pilosa.NewHolder("bad-path", pilosa.TestHolderConfig())
h, err := pilosa.NewHolder("bad-path", pilosa.TestHolderConfig())
if err != nil {
t.Fatalf("surprisingly, got an error from NewHolder with an invalid path which we didn't expect: %v", err)
}
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)

110
http_handler.go
View file

@ -323,9 +323,6 @@ func (h *Handler) populateValidators() {
h.validators["GetStatus"] = queryValidationSpecRequired()
h.validators["GetVersion"] = queryValidationSpecRequired()
h.validators["PostClusterMessage"] = queryValidationSpecRequired()
h.validators["GetFragmentBlockData"] = queryValidationSpecRequired()
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["GetPartitionNodes"] = queryValidationSpecRequired("partition")
h.validators["GetNodes"] = queryValidationSpecRequired()
@ -572,9 +569,6 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/mem-usage", handler.chkAuthZ(handler.handleGetMemUsage, authz.Read)).Methods("GET").Name("GetUsage")
router.HandleFunc("/internal/disk-usage", handler.chkAuthZ(handler.handleGetDiskUsage, authz.Read)).Methods("GET").Name("GetUsage")
router.HandleFunc("/internal/disk-usage/{index}", handler.chkAuthZ(handler.handleGetDiskUsage, authz.Read)).Methods("GET").Name("GetUsage")
router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.chkAuthN(handler.handleGetFragmentBlocks)).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/data", handler.chkAuthN(handler.handleGetFragmentData)).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/internal/fragment/nodes", handler.chkAuthN(handler.handleGetFragmentNodes)).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/partition/nodes", handler.chkAuthN(handler.handleGetPartitionNodes)).Methods("GET").Name("GetPartitionNodes")
router.HandleFunc("/internal/translate/keys", handler.chkAuthN(handler.handlePostTranslateKeys)).Methods("POST").Name("PostTranslateKeys")
@ -2583,14 +2577,14 @@ func validateProtobufHeader(r *http.Request) (error string, code int) {
// handleGetInternalDebugRBFJSON handles /internal/debug/rbf requests.
func (h *Handler) handleGetInternalDebugRBFJSON(w http.ResponseWriter, r *http.Request) {
buf, err := json.MarshalIndent(h.api.RBFDebugInfo(), "", " ")
w.Header().Set("Content-Type", "application/octet-stream")
var buf bytes.Buffer
err := h.api.holder.TxStore().DumpDot(&buf)
if err != nil {
http.Error(w, "marshal json: "+err.Error(), http.StatusInternalServerError)
http.Error(w, "rendering DOT: "+err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(buf)
w.Write(buf.Bytes())
}
// handleGetMetricsJSON handles /metrics.json requests, translating text metrics results to more consumable JSON.
@ -2746,37 +2740,6 @@ func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) {
}
}
// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
http.Error(w, "fragment blocks feature removed", http.StatusNotFound)
}
// handleGetFragmentBlocks handles GET /internal/fragment/blocks requests.
func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) {
http.Error(w, "fragment blocks feature removed", http.StatusNotFound)
}
// handleGetFragmentData handles GET /internal/fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read shard parameter.
q := r.URL.Query()
shard, err := strconv.ParseUint(q.Get("shard"), 10, 64)
if err != nil {
http.Error(w, "shard required", http.StatusBadRequest)
return
}
// Retrieve fragment data from holder.
f, err := h.api.FragmentData(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// Stream fragment to response body.
if _, err := f.WriteTo(w); err != nil {
h.logger.Errorf("error streaming fragment data: %s", err)
}
}
// handleGetTranslateData handles GET /internal/translate/data requests.
func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
@ -3075,12 +3038,15 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re
return
}
qcx := h.api.Txf().NewQcx()
qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer qcx.Release()
err = h.api.ImportAtomicRecord(r.Context(), qcx, req, opt)
if err == nil {
err = qcx.Finish()
} else {
qcx.Abort()
err = qcx.Commit()
}
if err != nil {
switch errors.Cause(err) {
@ -3153,9 +3119,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
// mark that we don't really have a specific shard
if len(req.ColumnKeys) != 0 {
req.Shard = ^uint64(0)
}
// ^0 is special and doesn't count as a shard
qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer qcx.Release()
if err := h.api.ImportValue(r.Context(), qcx, req, opts...); err != nil {
switch errors.Cause(err) {
@ -3169,9 +3143,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
err := qcx.Finish()
err = qcx.Commit()
if err != nil {
http.Error(w, fmt.Sprintf("error in qcx.Finish(): '%v'", err.Error()), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("error committing import: '%v'", err.Error()), http.StatusInternalServerError)
return
}
} else {
@ -3183,8 +3157,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
// mark that we don't really have a specific shard
if len(req.ColumnKeys) != 0 {
req.Shard = ^uint64(0)
}
// ^0 is special and doesn't count as a shard
qcx, err := h.api.NewIndexQueryContext(r.Context(), req.Index, req.Shard)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer qcx.Release()
if err := h.api.Import(r.Context(), qcx, req, opts...); err != nil {
switch errors.Cause(err) {
@ -3197,9 +3180,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
return
}
err := qcx.Finish()
err = qcx.Commit()
if err != nil {
http.Error(w, fmt.Sprintf("error in qcx.Finish() on set,time,mutex: '%v'", err.Error()), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("error committing import: '%v'", err.Error()), http.StatusInternalServerError)
return
}
}
@ -3231,8 +3214,12 @@ func (h *Handler) handleGetMutexCheck(w http.ResponseWriter, r *http.Request) {
http.Error(w, "limit must be numeric", http.StatusBadRequest)
}
}
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
qcx, err := h.api.NewQueryContext(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer qcx.Release()
out, err := h.api.MutexCheck(r.Context(), qcx, indexName, fieldName, details, limit)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -3268,8 +3255,12 @@ func (h *Handler) handleInternalGetMutexCheck(w http.ResponseWriter, r *http.Req
http.Error(w, "limit must be numeric", http.StatusBadRequest)
}
}
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
qcx, err := h.api.NewQueryContext(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer qcx.Release()
out, err := h.api.MutexCheckNode(r.Context(), qcx, indexName, fieldName, details, limit)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -3805,6 +3796,7 @@ func (h *Handler) handleRestoreIDAlloc(w http.ResponseWriter, r *http.Request) {
}
func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)

68
index.go
View file

@ -14,6 +14,7 @@ import (
"github.com/molecula/featurebase/v3/dax"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
@ -57,9 +58,6 @@ type Index struct {
// Instantiates new translation stores
OpenTranslateStore OpenTranslateStoreFunc
// track the subset of shards available to our views
fieldView2shard *FieldView2Shards
// indicate that we're closing and should wrap up and not allow new actions
closing chan struct{}
}
@ -97,10 +95,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
return idx, nil
}
func (i *Index) NewTx(txo Txo) Tx {
return i.holder.txf.NewTx(txo)
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()
@ -168,7 +162,8 @@ func (i *Index) Open() error {
}
// OpenWithSchema opens the index and uses the provided schema to verify that
// the index's fields are expected.
// the index's fields are expected. The provided QueryContext is to be used
// if any of this requires actual database reads.
func (i *Index) OpenWithSchema(idx *disco.Index) error {
if idx == nil {
return ErrInvalidSchema
@ -208,17 +203,6 @@ func (i *Index) open(idx *disco.Index) (err error) {
}
i.closing = make(chan struct{})
// fmt.Printf("new channel %p for index %p\n", i.closing, i)
// we don't want to open *all* the views for each shard, since
// most are empty when we are doing time quantums. It slows
// down startup dramatically. So we ask for the meta data
// of what fields/views/shards are present with data up front.
fieldView2shard, err := i.holder.txf.GetFieldView2ShardsMapForIndex(i)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("i.holder.txf.GetFieldView2ShardsMapForIndex('%v')", i.name))
}
i.fieldView2shard = fieldView2shard
// Add index to a map in holder. Used by openFields.
i.holder.addIndex(i)
@ -228,15 +212,11 @@ func (i *Index) open(idx *disco.Index) (err error) {
return errors.Wrap(err, "opening fields")
}
// Set bit depths.
// This is called in Index.open() (as opposed to Field.Open()) because the
// Field.bitDepth() method uses a transaction which relies on the index and
// its entry for the field in the Index.field map. If we try to set a
// field's BitDepth in Field.Open(), which itself might be inside the
// Index.openField() loop, then the field has not yet been added to the
// Index.field map. I think it would be better if Field.bitDepth didn't rely
// on its index at all, but perhaps with transactions that not possible. I
// don't know.
// Set bit depths based on current contents of fields.
// We could in theory do this as part of opening each field,
// but what query context would they use for the actual
// database transactions? So we have our own top-level
// thing to do that.
if err := i.setFieldBitDepths(); err != nil {
return errors.Wrap(err, "setting field bitDepths")
}
@ -414,7 +394,14 @@ func (i *Index) openExistenceField() error {
}
// setFieldBitDepths sets the BitDepth for all int and decimal fields in the index.
// We do it here, rather than when each field is opened, so we can open a
// single QueryContext to handle them all.
func (i *Index) setFieldBitDepths() error {
qcx, err := i.holder.NewIndexQueryContext(context.TODO(), i.name)
if err != nil {
return err
}
defer qcx.Release()
for name, f := range i.fields {
switch f.Type() {
case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
@ -422,7 +409,7 @@ func (i *Index) setFieldBitDepths() error {
default:
continue
}
bd, err := f.bitDepth()
bd, err := f.bitDepth(qcx)
if err != nil {
return errors.Wrapf(err, "getting bit depth for field: %s", name)
}
@ -451,11 +438,6 @@ func (i *Index) Close() error {
_ = testhook.Closed(i.holder.Auditor, i, nil)
}()
err := i.holder.txf.CloseIndex(i)
if err != nil {
return errors.Wrap(err, "closing index")
}
// Close partitioned translation stores.
for _, store := range i.translateStores {
if err := store.Close(); err != nil {
@ -514,11 +496,6 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
return b
}
// Begin starts a transaction on a shard of the index.
func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) {
return i.holder.txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil
}
// fieldPath returns the path to a field in the index.
func (i *Index) fieldPath(name string) string { return filepath.Join(i.FieldsPath(), name) }
@ -956,15 +933,18 @@ func (i *Index) DeleteField(name string) error {
return errors.Wrap(err, "closing")
}
if err := i.holder.txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil {
return errors.Wrap(err, "Txf.DeleteFieldFromStore")
fieldPath := i.fieldPath(name)
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "deleting field directory")
}
if err := i.holder.txStore.DeleteField(keys.Index(i.name), keys.Field(name)); err != nil {
return errors.Wrap(err, "deleting field from store")
}
// Remove reference.
delete(i.fields, name)
// remove shard metadata for field
i.fieldView2shard.removeField(name)
return i.translationSyncer.Reset()
}

21
index_internal_test.go
View file

@ -1,21 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"testing"
)
// mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
h := newTestHolder(tb)
index, err := h.CreateIndex("i", "", opt)
if err != nil {
panic(err)
}
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
return index
}

31
internal_client.go
View file

@ -18,6 +18,7 @@ import (
"time"
fbcontext "github.com/molecula/featurebase/v3/context"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/hashicorp/go-retryablehttp"
"github.com/molecula/featurebase/v3/authn"
@ -856,7 +857,7 @@ func (c *InternalClient) importHelper(ctx context.Context, req Message, process
//
// If we get a non-nil qcx, and have an associated API, we'll use that API
// directly for the local shard.
func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error {
func (c *InternalClient) Import(ctx context.Context, qcx qc.QueryContext, req *ImportRequest, options *ImportOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import")
defer span.Finish()
@ -884,7 +885,7 @@ func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportReques
//
// If we get a non-nil qcx, and have an associated API, we'll use that API
// directly for the local shard.
func (c *InternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error {
func (c *InternalClient) ImportValue(ctx context.Context, qcx qc.QueryContext, req *ImportValueRequest, options *ImportOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import")
defer span.Finish()
if req.ColumnKeys != nil {
@ -2175,6 +2176,32 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui
return resp.Body, nil
}
func (c *InternalClient) RestoreShard(ctx context.Context, index string, shard uint64, body io.Reader) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RestoreShard")
defer span.Finish()
url := fmt.Sprintf("%s%s/internal/restore/%s/%d", c.defaultURI, c.prefix(), index, shard)
req, err := http.NewRequest("POST", url, body)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream")
token, ok := authn.GetAccessToken(ctx)
if ok && token != "" {
req.Header.Set("Authorization", token)
}
req.Header.Set("User-Agent", "pilosa/"+Version)
// Execute request.
resp, err := c.executeRequest(req)
if resp != nil {
resp.Body.Close()
}
return err
}
// IDAllocDataReader returns a reader that provides a snapshot of ID allocation data.
func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.IDAllocDataReader")

20
internal_client_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
"github.com/ricochet2200/go-disk-usage/du"
"github.com/stretchr/testify/require"
)
// Test distributed TopN Row count across 3 nodes.
@ -484,15 +485,12 @@ func TestClient_Import(t *testing.T) {
// do a clear. also, do the clear with a Qcx.
func() {
// inner function so the deferred abort isn't delayed a lot
qcx := api.Txf().NewQcx()
defer qcx.Abort()
qcx := mustIndexQueryContext(t, api, req.Index, req.Shard)
if err := c.Import(context.Background(), qcx, req.Clone(), &pilosa.ImportOptions{Clear: true}); err != nil {
t.Fatalf("%s/%s: %v",
indexName, fieldName, err)
}
if err := qcx.Finish(); err != nil {
t.Fatalf("committing write: %v", err)
}
require.Nil(t, qcx.Commit())
}()
// Now do a query to see whether it worked...
if fieldName == "keyedf" {
@ -1532,18 +1530,6 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring
}
}
// verify that serverInfo has Backend
func TestClient_ServerInfoHasBackend(t *testing.T) {
//srcs := []string{"roaring", "rbf", "lmdb"}
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
si := cmd.API.Info()
if si.StorageBackend == "" {
panic("should have gotten a StorageBackend back")
}
pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid
}
func TestClient_ImportRoaringExists(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()

2
pql/ast.go
View file

@ -317,7 +317,7 @@ func (q *Query) WriteCallN() int {
var n int
for _, call := range q.Calls {
switch call.Name {
case "Set", "Clear", "ClearRow", "Store", "SetBit":
case "Set", "Clear", "ClearRow", "Store", "SetBit", "Delete":
n++
}
}

537
rbf.go
View file

@ -1,537 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"fmt"
"io"
"math"
"os"
"strings"
"sync"
"github.com/molecula/featurebase/v3/rbf"
rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/storage"
"github.com/pkg/errors"
)
// RbfDBWrapper wraps an *rbf.DB
type RbfDBWrapper struct {
path string
db *rbf.DB
cfg *rbfcfg.Config
reg *rbfDBRegistrar
muDb sync.Mutex
openTx map[*RBFTx]bool
// make Close() idempotent, avoiding panic on double Close()
closed bool
//DeleteEmptyContainer bool // needed for roaring compat?
doAllocZero bool
}
func (w *RbfDBWrapper) Path() string {
return w.path
}
func (w *RbfDBWrapper) SetHolder(h *Holder) {
// don't need it at the moment
//w.h = h
}
func (w *RbfDBWrapper) CleanupTx(tx Tx) {
r := tx.(*RBFTx)
r.mu.Lock()
if r.done {
r.mu.Unlock()
return
}
r.done = true
r.mu.Unlock()
// try not to hold r.mu while locking w.muDb
w.muDb.Lock()
delete(w.openTx, r)
w.muDb.Unlock()
}
// rbfDBRegistrar also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
type rbfDBRegistrar struct {
mu sync.Mutex
mp map[*RbfDBWrapper]bool
path2db map[string]*RbfDBWrapper
rbfConfig *rbfcfg.Config
}
func (r *rbfDBRegistrar) SetRBFConfig(cfg *rbfcfg.Config) {
r.mu.Lock()
defer r.mu.Unlock()
r.rbfConfig = cfg
}
func (r *rbfDBRegistrar) Size() int {
r.mu.Lock()
defer r.mu.Unlock()
nmp := len(r.mp)
npa := len(r.path2db)
if nmp != npa {
panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa))
}
return nmp
}
var globalRbfDBReg *rbfDBRegistrar = newRbfDBRegistrar()
func newRbfDBRegistrar() *rbfDBRegistrar {
return &rbfDBRegistrar{
mp: make(map[*RbfDBWrapper]bool),
path2db: make(map[string]*RbfDBWrapper),
}
}
// register each rbf.DB created, so we dedup and can
// can clean them up. This is called by OpenDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
// return the existing instance.
func (r *rbfDBRegistrar) unprotectedRegister(w *RbfDBWrapper) {
r.mp[w] = true
r.path2db[w.path] = w
}
// unregister removes w from r
func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) {
r.mu.Lock()
delete(r.mp, w)
delete(r.path2db, w.path)
r.mu.Unlock()
}
// OpenDBWrapper opens the database in the path directory
// without deleting any prior content. Any
// database directory will have the "-rbf" suffix.
//
// OpenDBWrapper will check the registry and make a new instance only
// if one does not exist for its path. Otherwise it returns
// the existing instance. This insures only one RbfDBWrapper
// per bpath in this pilosa node.
func (r *rbfDBRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[path]
if ok {
// creates the effect of having only one DB open per pilosa node.
return w, nil
}
if r.rbfConfig == nil {
r.rbfConfig = rbfcfg.NewDefaultConfig()
r.rbfConfig.DoAllocZero = doAllocZero
r.rbfConfig.FsyncEnabled = cfg.FsyncEnabled
}
db := rbf.NewDB(path, r.rbfConfig)
w = &RbfDBWrapper{
reg: r,
path: path,
db: db,
doAllocZero: doAllocZero,
openTx: make(map[*RBFTx]bool),
cfg: r.rbfConfig,
}
r.unprotectedRegister(w)
err := db.Open()
if err != nil {
panic(fmt.Sprintf("cannot open rbfDB at path '%v': '%v'", path, err))
}
return w, nil
}
type RBFTx struct {
// initialIndex is only a debugging aid. Transactions
// can cross indexes. It can be left empty without consequence.
initialIndex string
tx *rbf.Tx
o Txo
Db *RbfDBWrapper
done bool
mu sync.Mutex // protect done as it changes state
}
func (tx *RBFTx) DBPath() string {
return tx.tx.DBPath()
}
func (tx *RBFTx) Type() string {
return RBFTxn
}
func (tx *RBFTx) Rollback() {
tx.tx.Rollback()
tx.Db.CleanupTx(tx)
}
func (tx *RBFTx) Commit() (err error) {
err = tx.tx.Commit()
tx.Db.CleanupTx(tx)
return err
}
func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.tx.RoaringBitmap(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
return tx.tx.Container(rbfName(index, field, view, shard), key)
}
func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
return tx.tx.PutContainer(rbfName(index, field, view, shard), key, c)
}
func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
return tx.tx.RemoveContainer(rbfName(index, field, view, shard), key)
}
// Add sets all the a bits hot in the specified fragment.
func (tx *RBFTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
return tx.addOrRemove(index, field, view, shard, false, a...)
}
// Remove clears all the specified a bits in the chosen fragment.
func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
return tx.addOrRemove(index, field, view, shard, true, a...)
}
// sortedParanoia is a flag to enable a check for unsorted inputs to addOrRemove,
// which is expensive in practice and only really useful occasionally.
const sortedParanoia = false
func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) {
if len(a) == 0 {
return 0, nil
}
name := rbfName(index, field, view, shard)
// this special case can/should possibly go away, except that it
// turns out to be by far the most common case, and we need to know
// there's at least two items to simplify the check-sorted thing.
if len(a) == 1 {
hi, lo := highbits(a[0]), lowbits(a[0])
rc, err := tx.tx.Container(name, hi)
if err != nil {
return 0, errors.Wrap(err, "failed to retrieve container")
}
if remove {
if rc.N() == 0 {
return 0, nil
}
rc1, chng := rc.Remove(lo)
if !chng {
return 0, nil
}
if rc1.N() == 0 {
err = tx.tx.RemoveContainer(name, hi)
} else {
err = tx.tx.PutContainer(name, hi, rc1)
}
if err != nil {
return 0, err
}
return 1, nil
} else {
rc2, chng := rc.Add(lo)
if !chng {
return 0, nil
}
err = tx.tx.PutContainer(name, hi, rc2)
if err != nil {
return 0, err
}
return 1, nil
}
}
var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter.
var rc *roaring.Container
var hi uint64
var lo uint16
// we can accept sorted either ascending or descending.
sign := a[1] - a[0]
prev := a[0] - sign
sign >>= 63
for i, v := range a {
// This check is noticably expensive (a few percent in some
// use cases) and as long as it passes occasionally it's probably
// not important to run it all the time, and anyway panic is
// not a good choice outside of testing.
if sortedParanoia {
if (v-prev)>>63 != sign {
explain := fmt.Sprintf("addOrRemove: %d < %d != %d < %d", v, prev, a[1], a[0])
panic(explain)
}
if v == prev {
explain := fmt.Sprintf("addOrRemove: %d twice", v)
panic(explain)
}
}
prev = v
hi, lo = highbits(v), lowbits(v)
if hi != lastHi {
// either first time through, or changed to a different container.
// do we need put the last updated container now?
if i > 0 {
// not first time through, write what we got.
if remove && (rc == nil || rc.N() == 0) {
err = tx.tx.RemoveContainer(name, lastHi)
if err != nil {
return 0, errors.Wrap(err, "failed to remove container")
}
} else {
err = tx.tx.PutContainer(name, lastHi, rc)
if err != nil {
return 0, errors.Wrap(err, "failed to put container")
}
}
}
// get the next container
rc, err = tx.tx.Container(name, hi)
if err != nil {
return 0, errors.Wrap(err, "failed to retrieve container")
}
} // else same container, keep adding bits to rct.
chng := false
// rc can be nil before, and nil after, in both Remove/Add below.
// The roaring container add() and remove() methods handle this.
if remove {
rc, chng = rc.Remove(lo)
} else {
rc, chng = rc.Add(lo)
}
if chng {
changeCount++
}
lastHi = hi
}
// write the last updates.
if remove {
if rc == nil || rc.N() == 0 {
err = tx.tx.RemoveContainer(name, hi)
if err != nil {
return 0, errors.Wrap(err, "failed to remove container")
}
} else {
err = tx.tx.PutContainer(name, hi, rc)
if err != nil {
return 0, errors.Wrap(err, "failed to put container")
}
}
} else {
if rc == nil || rc.N() == 0 {
panic("there should be no way to have an empty bitmap AFTER an Add() operation")
}
err = tx.tx.PutContainer(name, hi, rc)
if err != nil {
return 0, errors.Wrap(err, "failed to put container")
}
}
return
}
func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
return tx.tx.Contains(rbfName(index, field, view, shard), v)
}
func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
return tx.tx.ContainerIterator(rbfName(index, field, view, shard), key)
}
func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Count(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Max(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
return tx.tx.Min(rbfName(index, field, view, shard))
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
return tx.tx.CountRange(rbfName(index, field, view, shard), start, end)
}
func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end)
}
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize)
}
func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
err = tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter)
return errors.Wrap(err, fmt.Sprintf("applying filter for index %s, field %s, view %s, shard %d", index, field, view, shard))
}
func (tx *RBFTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
err = tx.tx.ApplyRewriter(rbfName(index, field, view, shard), ckey, filter)
return errors.Wrap(err, fmt.Sprintf("applying rewriter for index %s, field %s, view %s, shard %d", index, field, view, shard))
}
func (tx *RBFTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
return tx.tx.GetSortedFieldViewList()
}
func (tx *RBFTx) GetFieldSizeBytes(index, field string) (uint64, error) {
return tx.tx.GetSizeBytesWithPrefix(string(txkey.FieldPrefix(index, field)))
}
// SnapshotReader returns a reader that provides a snapshot of the current database.
func (tx *RBFTx) SnapshotReader() (io.Reader, error) {
return tx.tx.SnapshotReader()
}
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
func rbfName(index, field, view string, shard uint64) string {
return string(txkey.Prefix(index, field, view, shard))
}
// rbfFieldPrefix returns a prefix for field keys in RBF.
func rbfFieldPrefix(index, field string) string {
//return fmt.Sprintf("%s\x00%s\x00", index, field)
return string(txkey.FieldPrefix(index, field))
}
func (w *RbfDBWrapper) HasData() (has bool, err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
return w.db.HasData(false) // false => any prior attempt at write means we "have data"
}
func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error {
w.muDb.Lock()
defer w.muDb.Unlock()
if err := os.RemoveAll(fieldPath); err != nil {
return errors.Wrap(err, "removing directory")
}
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(rbfFieldPrefix(index, field)); err != nil {
return err
}
return tx.Commit()
}
func (w *RbfDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in RbfDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes", indexName)
}
prefix := txkey.IndexOnlyPrefix(indexName)
w.muDb.Lock()
defer w.muDb.Unlock()
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(string(prefix)); err != nil {
return err
}
return tx.Commit()
}
func (w *RbfDBWrapper) Close() error {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
w.reg.unregister(w)
w.closed = true
}
return w.db.Close()
}
// needed to handle the special case on reload, the close method unregisters the wrapper and all that is
// required is the backing file get reloaded
func (w *RbfDBWrapper) CloseDB() error {
w.muDb.Lock()
defer w.muDb.Unlock()
w.closed = true
return w.db.Close()
}
func (w *RbfDBWrapper) OpenDB() error {
w.muDb.Lock()
defer w.muDb.Unlock()
err := w.db.Open()
if err != nil {
return err
}
w.closed = false
return nil
}
func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) {
tx, err := w.db.Begin(write)
if err != nil {
return nil, err
}
rtx := &RBFTx{
tx: tx,
initialIndex: initialIndex,
o: o,
Db: w,
}
w.muDb.Lock()
w.openTx[rtx] = true
w.muDb.Unlock()
return rtx, nil
}
func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
err = tx.DeleteBitmapsWithPrefix(rbfName(index, field, view, shard))
if err != nil {
return err
}
return tx.Commit()
}
func (w *RbfDBWrapper) OpenListString() (r string) {
return "rbf OpenListString not implemented yet"
}

7
rbf/ingest_test.go
View file

@ -16,12 +16,15 @@ import (
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/testhook"
txkey "github.com/molecula/featurebase/v3/short_txkey"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
// This is intended to be similar to the actual field/view keys we
// generate over in querycontext for fragment keys, but it actually
// doesn't matter, as long as it's some kind of a string that reflects
// both field and view.
func rbfName(index, field, view string, shard uint64) string {
return string(txkey.Prefix(index, field, view, shard))
return fmt.Sprintf("~%s;%s<", field, view)
}
var _ = rbfName // keep linter happy

18
rbf/tx.go
View file

@ -11,13 +11,10 @@ import (
"github.com/benbjohnson/immutable"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
)
var _ = txkey.ToString
// Tx represents an RBF transaction. Transactions provide guarantees such as
// atomicity for all writes that occur as well as serializable isolation.
// Transactions can be obtained by calling DB.Begin() and provide a snapshot
@ -2247,21 +2244,6 @@ func (tx *Tx) PageData(pgno uint32) ([]byte, error) {
return buf, err
}
func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) {
records, err := tx.RootRecords()
if err != nil {
return nil, err
}
it := records.Iterator()
for !it.Done() {
k, _, _ := it.Next()
root := k
fv := txkey.FieldViewFromPrefix([]byte(root))
fvs = append(fvs, fv)
}
return
}
func (tx *Tx) DebugInfo() *TxDebugInfo {
return &TxDebugInfo{
Ptr: fmt.Sprintf("%p", tx),

30
rbf/util.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"strings"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
)
@ -52,7 +51,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
rootRecords, err := readRootRecords(page)
vprint.PanicOn(err)
for k, rr := range rootRecords {
fmt.Printf(" [%02v] Name:'%v' pgno:%v\n", k, prefixToString(rr.Name), rr.Pgno)
fmt.Printf(" [%02v] Name:'%v' pgno:%v\n", k, PrefixToString(rr.Name), rr.Pgno)
}
case *LeafPageInfo:
@ -61,7 +60,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
}
fmt.Printf("Pgno:%-8d ", pgno)
fmt.Printf("%-10s ", "leaf")
fmt.Printf("%-54q ", prefixToString(info.Tree))
fmt.Printf("%-54q ", PrefixToString(info.Tree))
fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
page, _, err := tx.readPage(uint32(pgno))
@ -76,7 +75,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
case *BranchPageInfo:
fmt.Printf("Pgno:%-8d ", pgno)
fmt.Printf("%-10s ", "branch")
fmt.Printf("%-54q ", prefixToString(info.Tree))
fmt.Printf("%-54q ", PrefixToString(info.Tree))
fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
page, _, err := tx.readPage(uint32(pgno))
@ -90,7 +89,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
case *BitmapPageInfo:
fmt.Printf("Pgno:%-8d ", pgno)
fmt.Printf("%-10s ", "bitmap")
fmt.Printf("%-54q ", prefixToString(info.Tree))
fmt.Printf("%-54q ", PrefixToString(info.Tree))
fmt.Printf("-\n")
case *FreePageInfo:
@ -195,13 +194,19 @@ func printFreePage(page *FreePage) {
fmt.Printf("Type: free\n")
}
func prefixToString(s string) (ret string) {
defer func() {
if err := recover(); err != nil {
ret = s
}
}()
return txkey.PrefixToString([]byte(s))
// PrefixToString converts a fragment key (used to denote
// a root bitmap in an RBF file) into a description of it
// suitable for printing. This behavior reflects the
// historical practice of the short_txkey package, which
// we no longer have. It's exported because the rbf_pages
// command wants to use it to display things.
func PrefixToString(s string) (ret string) {
var field, view string
n, err := fmt.Sscanf(s, "~%s;%s<", &field, &view)
if err != nil || n != 2 {
return s
}
return fmt.Sprintf("field:%s;view:%s", field, view)
}
///////////////// happy linter
@ -212,4 +217,3 @@ var _ = printLeafPage
var _ = printBranchPage
var _ = printBitmapPage
var _ = printFreePage
var _ = prefixToString

21
roaring/filter.go
View file

@ -1297,6 +1297,27 @@ func getFirstRowAsContainers(citer ContainerIterator) []containerWithKey {
return citerContainers
}
type infiniteOnesIterator struct {
count uint64
c *Container
}
func NewInfiniteOnesIterator() *infiniteOnesIterator {
return &infiniteOnesIterator{c: NewContainerRun([]Interval16{{Start: 0, Last: 65535}})}
}
func (i *infiniteOnesIterator) Next() bool {
i.count++
return true
}
func (i *infiniteOnesIterator) Value() (uint64, *Container) {
return i.count, i.c
}
func (i *infiniteOnesIterator) Close() {
}
// NewClearAndSetRewriter instantiates a ClearAndSetRewriter
func NewClearAndSetRewriter(clear, set ContainerIterator) (*ClearAndSetRewriter, error) {
curSetKey, curSet := getNextFromIterator(set)

27
roaring/roaring.go
View file

@ -2265,33 +2265,6 @@ func (r *baseRoaringIterator) Current() (key uint64, cType byte, n int, length i
return r.currentKey, r.currentType, r.currentN, r.currentLen, r.currentPointer, r.lastErr
}
// SanityCheckMapping is a debugging function which checks whether containers
// are *correctly* recorded as mapped or unmapped.
func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut int64, unmappedIn int64, errs int, err error) {
b.Containers.UpdateEvery(func(key uint64, c *Container, existed bool) (*Container, bool) {
dptr := uintptr(unsafe.Pointer(c.pointer))
if dptr >= from && dptr < to {
if c.Mapped() {
mappedIn++
} else {
err = fmt.Errorf("container key %d, addr %x, inside %x+%d",
key, dptr, from, to-from)
errs++
unmappedIn++
}
} else {
if c.Mapped() {
err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped",
key, dptr, from, to-from)
errs++
mappedOut++
}
}
return c, false
})
return mappedIn, mappedOut, unmappedIn, errs, err
}
// RemapRoaringStorage tries to update all containers to refer to
// the roaring bitmap in the provided []byte. If any containers are
// marked as mapped, but do not match the provided storage, they will

37
server.go
View file

@ -18,6 +18,7 @@ import (
daxstorage "github.com/molecula/featurebase/v3/dax/storage"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/logger"
pnet "github.com/molecula/featurebase/v3/net"
rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg"
@ -540,7 +541,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
if err != nil {
return nil, err
}
s.holder = NewHolder(path, s.holderConfig)
s.holder, err = NewHolder(path, s.holderConfig)
if err != nil {
return nil, err
}
s.holder.Stats.SetLogger(s.logger)
cwd, err := os.Getwd()
if err != nil {
@ -914,14 +918,17 @@ func (s *Server) ViewsRemoval(ctx context.Context) {
timeSince := time.Since(viewTime)
if timeSince >= field.Options().TTL {
for _, shard := range field.AvailableShards(true).Slice() {
err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil)
if err != nil {
s.logger.Errorf("view: %s, shard: %d, ttl delete fragment: %s", shard, viewName, err)
}
shardUints := field.AvailableShards(true).Slice()
shards := make([]keys.Shard, len(shardUints))
for i, v := range shardUints {
shards[i] = keys.Shard(v)
}
err := s.holder.txStore.DeleteFragments(keys.Index(index.Name()), keys.Field(field.Name()), []keys.View{keys.View(view.name)}, shards)
if err != nil {
s.logger.Errorf("view: %s ttl delete fragment: %s", viewName, err)
}
err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name)
err = s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name)
if err != nil {
s.logger.Errorf("view: %s, ttl delete view: %s", viewName, err)
}
@ -931,15 +938,17 @@ func (s *Server) ViewsRemoval(ctx context.Context) {
}
}
if field.Options().NoStandardView && field.view(viewStandard) != nil {
// delete view "standard" if NoStandardView is true and view "standard" exists
for _, shard := range field.AvailableShards(true).Slice() {
err := s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), viewStandard, shard, nil)
if err != nil {
s.logger.Errorf("delete view %s from shard %d: %s", viewStandard, shard, err)
}
shardUints := field.AvailableShards(true).Slice()
shards := make([]keys.Shard, len(shardUints))
for i, v := range shardUints {
shards[i] = keys.Shard(v)
}
err := s.holder.txStore.DeleteFragments(keys.Index(index.Name()), keys.Field(field.Name()), []keys.View{keys.View(viewStandard)}, shards)
if err != nil {
s.logger.Errorf("view: %s ttl delete fragment: %s", viewStandard, err)
}
err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard)
err = s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), viewStandard)
if err != nil {
s.logger.Errorf("view: %s, delete view: %s", viewStandard, err)
}

7
server/grpc.go
View file

@ -736,8 +736,11 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
return errToStatusError(err)
}
qcx := h.api.Holder().Txf().NewQcx()
defer qcx.Abort()
qcx, err := h.api.NewQueryContext(context.TODO())
if err != nil {
return err
}
defer qcx.Release()
var fields []*pilosa.Field
for _, field := range index.Fields() {

44
server/handler_test.go
View file

@ -23,6 +23,7 @@ import (
pb "github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
@ -175,31 +176,28 @@ func TestHandler_Endpoints(t *testing.T) {
})
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
tx0 := holder.Txf().NewWritableQcx()
defer tx0.Abort()
qcx, err := hldr.NewIndexQueryContext(context.Background(), "i0")
require.Nil(t, err)
defer qcx.Release()
if f, err := i0.CreateFieldIfNotExists("f1", "", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if err := tx0.Finish(); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
tx1 := holder.Txf().NewWritableQcx()
defer tx1.Abort()
qcx, err = hldr.NewIndexQueryContext(context.Background(), "i1")
require.Nil(t, err)
if f, err := i1.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx1, 0, 0, nil); err != nil {
t.Fatal(err)
}
if err := tx1.Finish(); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
t.Run("Schema", func(t *testing.T) {
w := httptest.NewRecorder()
@ -255,11 +253,11 @@ func TestHandler_Endpoints(t *testing.T) {
// i2 is for SchemaDetails
i2 := hldr.MustCreateIndexIfNotExists("i2", pilosa.IndexOptions{})
tx2 := holder.Txf().NewWritableQcx()
defer tx2.Abort()
qcx, err = holder.NewIndexQueryContext(context.Background(), "i2")
require.Nil(t, err)
if f, err := i2.CreateFieldIfNotExists("f0", "", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
@ -269,7 +267,7 @@ func TestHandler_Endpoints(t *testing.T) {
}
for n := 0; n < 4; n++ {
if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil {
if _, err := f.SetValue(qcx, uint64(n), int64(n)); err != nil {
t.Fatal(err)
}
}
@ -280,30 +278,27 @@ func TestHandler_Endpoints(t *testing.T) {
}
for n := 0; n < 5; n++ {
if _, err := f.SetValue(tx2, uint64(n), int64(n)); err != nil {
if _, err := f.SetValue(qcx, uint64(n), int64(n)); err != nil {
t.Fatal(err)
}
}
if f, err := i2.CreateFieldIfNotExists("f3", "", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i2.CreateFieldIfNotExists("f4", "", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000)); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i2.CreateFieldIfNotExists("f5", "", pilosa.OptFieldTypeBool()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil {
t.Fatal(err)
}
if err := tx2.Finish(); err != nil {
} else if _, err := f.SetBit(qcx, 0, 0, nil); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
t.Run("SchemaDetails", func(t *testing.T) {
w := httptest.NewRecorder()
@ -493,7 +488,6 @@ func TestHandler_Endpoints(t *testing.T) {
msg := pilosa.ImportRoaringRequest{
Action: pilosa.RequestActionOverwrite,
Block: 0,
Views: map[string][]byte{
"bsig_int-field": roaringData,
},

17
server/server_test.go
View file

@ -43,6 +43,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
for i := 0; i < 10; i++ {
i := i // make local copy of i, loop variable capture
t.Run(fmt.Sprint(i), func(t *testing.T) {
t.Parallel()
@ -677,13 +678,13 @@ func TestMain_ImportTimestamp(t *testing.T) {
}
// Import data.
qcx := m.API.Txf().NewQcx()
ctx := context.Background()
qcx, err := m.API.NewIndexQueryContext(ctx, data.Index)
require.Nil(t, err)
if err := m.API.Import(context.Background(), qcx, &data); err != nil { /// first write i/0 here. 2nd write here.
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName)
files, err := os.ReadDir(dir)
@ -732,13 +733,13 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
}
// Import data.
qcx := m.API.Txf().NewQcx()
ctx := context.Background()
qcx, err := m.API.NewIndexQueryContext(ctx, data.Index)
require.Nil(t, err)
if err := m.API.Import(context.Background(), qcx, &data); err != nil {
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
require.Nil(t, qcx.Commit())
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName)

199
short_txkey/txkey.go
View file

@ -1,199 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Package txkey consolidates in one place the use of keys to index into our
// various storage/txn back-ends. The short_txkey version omits the
// index and shard, since these are implicitly part of our database-per-shard
// in an index scheme. In other words, every database is only in exactly
// one shard of one index already. There is no need to repeat the index
// and shard in these keys.
package short_txkey
import (
"encoding/binary"
"fmt"
)
// FieldView is here to avoid circular import.
type FieldView struct {
Field string
View string
}
func FieldViewFromPrefix(prefix []byte) FieldView {
field, view := SplitPrefix(prefix)
return FieldView{Field: field, View: view}
}
func FieldViewFromFullKey(fullKey []byte) FieldView {
field, view, _ := Split(fullKey)
return FieldView{Field: field, View: view}
}
// Key produces the bytes that we use as a key to query the storage/tx engine.
// The roaringContainerKey argument to Key() is a container key into a roaring Container.
// The return value from Key() is constructed as follows:
//
// ~field;view<ckey#
//
// where ckey is always exactly 8 bytes, uint64 big-endian encoded.
//
// Keys always start with either '~' or '>'. Keys always end with '#'.
// Keys always contain exactly one each of ';' and '<', in that order.
// The field is between the '~' and the ';'. It must be at least 1 byte long.
// The view is between the ';' and the '<'. It must be at least 1 byte long.
// The ckey is the 8 bytes between the '<' and the '#'.
// The Prefix of a key ends at, and includes, the '<'. It is at least 13 bytes long.
// The index, field, and view are not allowed to contain these reserved bytes:
//
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable
// base-64 encoding.
//
// The shortest possible key is 14 bytes. It would be laid out like this:
//
// ~f;v<12345678#
// 12345678901234
//
// keys starting with '~' are regular value keys.
// keys starting with '>' are symlink keys.
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) {
prefix := Prefix(index, field, view, shard)
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
return append(prefix, ckey[:]...)
}
// KeyAndPrefix returns the equivalent of Key() and Prefix() calls.
func KeyAndPrefix(index, field, view string, shard, roaringContainerKey uint64) (key, prefix []byte) {
prefix = Prefix(index, field, view, shard)
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
key = append(prefix, ckey[:]...)
return
}
var _ = KeyAndPrefix // keep linter happy
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 14 {
panic(fmt.Sprintf("bkey too short, must have at least 14 bytes: '%v'", string(bkey)))
}
typ := bkey[0]
if typ != '~' && typ != '>' {
panic(fmt.Sprintf("bkey did not start with '~' for value nor '>' for symlink: '%v'", string(bkey)))
}
if bkey[n-10] != '<' {
panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey)))
}
if bkey[n-1] != '#' {
panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey)))
}
}
// KeyExtractContainerKey extracts the containerKey from bkey.
// key example: field;view<ckey
// shortest: ~f;v<12345678#
//
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
n := len(bkey)
MustValidateKey(bkey)
containerKey = binary.BigEndian.Uint64(bkey[(n - 9):(n - 1)])
return
}
func AllShardPrefix(index, field, view string) (r []byte) {
r = make([]byte, 0, 64)
r = append(r, '~')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, '<')
return
}
// Prefix returns everything from Key up to and
// including the '<' byte in a Key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with Key() and KeyExtractContainerKey().
func Prefix(index, field, view string, shard uint64) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, '<')
return
}
// IndexOnlyPrefix returns a "~" prefix suitable for DeleteIndex and a key-scan to
// remove all storage. We assume only one index in this database, so delete everything.
func IndexOnlyPrefix(indexName string) (r []byte) {
return []byte("~")
}
// same for deleting a whole field.
func FieldPrefix(index, field string) (r []byte) {
r = make([]byte, 0, 16)
r = append(r, '~')
r = append(r, []byte(field)...)
r = append(r, ';')
return
}
// PrefixFromKey key example: ~field;view<ckey#
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
//
// view ckey
func PrefixFromKey(bkey []byte) (prefix []byte) {
n := len(bkey)
return bkey[:(n - 9)]
}
func ToString(bkey []byte) (r string) {
field, view, ckey := Split(bkey)
return fmt.Sprintf("fld:'%v';vw:'%v';ckey@%020d", field, view, ckey)
}
func PrefixToString(pre []byte) (r string) {
field, view := SplitPrefix(pre)
return fmt.Sprintf("fld:'%v';vw:'%v';", field, view)
}
func Split(bkey []byte) (field, view string, ckey uint64) {
ckey = KeyExtractContainerKey(bkey)
n := len(bkey)
field, view = SplitPrefix(bkey[:(n - 9)])
return
}
// full key: ~field;view<ckey#
// prefix : ~field;view<
func SplitPrefix(pre []byte) (field, view string) {
n := len(pre)
// prefix: ~field;view<
beg := 1
for i := 1; i < n; i++ {
switch pre[i] {
case ';':
field = string(pre[beg:i])
beg = i + 1
view = string(pre[beg:(n - 1)])
return
}
}
panic(fmt.Sprintf("malformed prefix '%v' / '%#v', could not Split", string(pre), pre))
}

78
short_txkey/txkey_test.go
View file

@ -1,78 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package short_txkey
import (
"bytes"
"encoding/binary"
"fmt"
"testing"
)
func Test_KeyPrefix(t *testing.T) {
// Prefix() must agree with Key(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view := "i", "f", "v"
needle := Key(index, field, view, 0, 0)
// prefix example: i%f;v:12345678<
prefix := Prefix(index, field, view, 0)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("Prefix() output '%v'was not a prefix of Key() '%v'", string(needle), string(prefix)))
}
npre := len(prefix)
nneed := len(needle)
if npre+9 != nneed {
panic(fmt.Sprintf("Prefix() output len %v '%v' was not 9 characters shorter than Key() len %v '%v'", npre, string(prefix), nneed, string(needle)))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
KeyExtractContainerKey(prefix) // should panic.
}()
}
func Test_PrefixFromKey(t *testing.T) {
k := []byte("~f;v<12345678#")
x := []byte("~f;v<")
pre := PrefixFromKey(k)
if !bytes.Equal(pre, x) {
nx := len(x)
npre := len(pre)
if nx != npre {
panic(fmt.Sprintf("nx=%v, npre=%v; expected '%v', observed '%v'", nx, npre, string(x), string(pre)))
}
for i := 0; i < nx; i++ {
if x[i] != pre[i] {
panic(fmt.Sprintf("first diff at index %v, expected '%v', observed '%v'", i, string(x[:i]), string(pre[:i])))
}
}
panic(fmt.Sprintf("expected:\n%v\n, observed:\n%v\n", string(x), string(pre)))
}
}
func Test_Split(t *testing.T) {
bkey := []byte("~f;v<12345678#")
var xckey uint64 = 43
binary.BigEndian.PutUint64(bkey[5:13], xckey)
f, v, ckey := Split(bkey)
if f != "f" {
panic("wrong field")
}
if v != "v" {
panic("wrong view")
}
if ckey != xckey {
panic("wrong ckey")
}
}

496
stattx.go
View file

@ -1,496 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"fmt"
"math"
"runtime"
"sort"
"sync"
"time"
"github.com/molecula/featurebase/v3/debugstats"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/vprint"
)
// statTx is useful to profile on a
// per method basis, and to play with
// read/write locking.
type statTx struct {
b Tx
stats *callStats
}
// for now, just track call stats globally. But each statTx has
// a pointer to a callStats, so could be made per index or per shard, etc.
var globalCallStats = newCallStats()
type callStats struct {
// protect elap
mu sync.Mutex
// track how much time each call took.
elap map[kall]*elapsed
}
type elapsed struct {
dur []float64
}
func newCallStats() *callStats {
w := &callStats{}
w.reset()
return w
}
func (w *callStats) reset() {
w.mu.Lock()
defer w.mu.Unlock()
w.elap = make(map[kall]*elapsed)
for i := kall(0); i < kLast; i++ {
w.elap[i] = &elapsed{}
}
}
func (c *callStats) report() (r string) {
backend := storage.DefaultBackend
r = fmt.Sprintf("callStats: (%v)\n", backend)
c.mu.Lock()
defer c.mu.Unlock()
var lines []*debugstats.LineSorter
for i := kall(0); i < kLast; i++ {
slc := c.elap[i].dur
n := len(slc)
if n == 0 {
continue
}
mean, sd, totaltm := computeMeanSd(slc)
if n == 1 {
sd = 0
mean = slc[0]
totaltm = slc[0]
}
line := fmt.Sprintf(" %20v N=%8v avg/op: %12v sd: %12v total: %12v\n", i.String(), n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm))
lines = append(lines, &debugstats.LineSorter{Line: line, Tot: totaltm})
}
sort.Sort(debugstats.SortByTot(lines))
for i := range lines {
r += lines[i].Line
}
var m1 runtime.MemStats
runtime.ReadMemStats(&m1)
r += fmt.Sprintf("\n m1.TotalAlloc = %v\n", m1.TotalAlloc)
return
}
var NaN = math.NaN()
func computeMeanSd(slc []float64) (mean, sd, tot float64) {
if len(slc) < 2 {
return NaN, NaN, NaN
}
for _, v := range slc {
tot += v
}
n := float64(len(slc))
mean = tot / n
variance := 0.0
for _, v := range slc {
tmp := (v - mean)
variance += tmp * tmp
}
variance = variance / n // biased, but we don't care b/c we can have very small n
sd = math.Sqrt(variance)
if sd < 1e-8 {
// sd is super close to zero, NaN out the z-score rather than +/- Inf
sd = NaN
}
return
}
func (c *callStats) add(k kall, dur time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
e := c.elap[k]
e.dur = append(e.dur, float64(dur))
}
func newStatTx(b Tx) *statTx {
w := &statTx{
b: b,
// For now, just track call stats globally.
// But this could be made per-Tx by making this be stats: newCallStats(),
// for example.
stats: globalCallStats,
}
return w
}
type kall int
// constants for kall argument to callStats.add()
const (
kNewTxIterator kall = iota
kImportRoaringBits
kRollback
kCommit
kRoaringBitmap
kContainer
kPutContainer
kRemoveContainer
kAdd
kRemove
kContains
kContainerIterator
kCount
kMax
kMin
kCountRange
kOffsetRange
kLast // mark the end, always keep this last. The following aren't tracked atm:
kType
)
func (k kall) String() string {
switch k {
case kNewTxIterator:
return "kNewTxIterator"
case kImportRoaringBits:
return "kImportRoaringBits"
case kRollback:
return "kRollback"
case kCommit:
return "kCommit"
case kRoaringBitmap:
return "kRoaringBitmap"
case kContainer:
return "kContainer"
case kPutContainer:
return "kPutContainer"
case kRemoveContainer:
return "kRemoveContainer"
case kAdd:
return "kAdd"
case kRemove:
return "kRemove"
case kContains:
return "kContains"
case kContainerIterator:
return "kContainerIterator"
case kCount:
return "kCount"
case kMax:
return "kMax"
case kMin:
return "kMin"
case kCountRange:
return "kCountRange"
case kOffsetRange:
return "kOffsetRange"
case kLast:
return "kLast"
case kType:
return "kType"
}
vprint.PanicOn(fmt.Sprintf("unknown kall '%v'", int(k)))
return ""
}
var _ Tx = (*statTx)(nil)
func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
me := kImportRoaringBits
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *statTx) Rollback() {
me := kRollback
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
c.b.Rollback()
}
func (c *statTx) Commit() error {
me := kCommit
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Commit()
}
func (c *statTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
me := kRoaringBitmap
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
}
func (c *statTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
me := kContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Container(index, field, view, shard, key)
}
func (c *statTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
me := kPutContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
}
func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
me := kRemoveContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *statTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
me := kAdd
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Add(index, field, view, shard, a...)
}
func (c *statTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
me := kRemove
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
}
func (c *statTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
me := kContains
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
}
func (c *statTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
me := kContainerIterator
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
}
func (c *statTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}
func (c *statTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
}
func (c *statTx) Count(index, field, view string, shard uint64) (uint64, error) {
me := kCount
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Count(index, field, view, shard)
}
func (c *statTx) Max(index, field, view string, shard uint64) (uint64, error) {
me := kMax
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Max(index, field, view, shard)
}
func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
me := kMin
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Min(index, field, view, shard)
}
func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
me := kCountRange
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
}
func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
me := kOffsetRange
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *statTx) Type() string {
return c.b.Type()
}
func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
return c.b.GetSortedFieldViewList(idx, shard)
}
func (tx *statTx) GetFieldSizeBytes(index, field string) (uint64, error) {
return 0, nil
}

43
test/cluster.go
View file

@ -16,8 +16,10 @@ import (
"github.com/molecula/featurebase/v3/api/client"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/etcd"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/proto"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/storage"
"github.com/pkg/errors"
@ -270,8 +272,8 @@ func (c *ShareableCluster) ImportBitsWithTimestamp(t testing.TB, index, field st
continue
}
func() {
qcx := com.API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := com.IndexWideQcx(t, index)
defer done()
if len(timestamps) == 0 {
err := com.API.Import(context.Background(), qcx, &pilosa.ImportRequest{
Index: index,
@ -310,6 +312,23 @@ func (c *ShareableCluster) ImportBits(t testing.TB, index, field string, rowcols
c.ImportBitsWithTimestamp(t, index, field, rowcols, noTime)
}
// IndexWideQueryContext requests an index-wide, not shard-specific,
// QueryContext for an index on the given node. It's a helper for some
// of the test functions. It also yields a function to defer when
// done which fails the test on error.
func (c *Command) IndexWideQcx(tb testing.TB, index string) (qc.QueryContext, func()) {
txs := c.API.Holder().TxStore()
qcx, err := txs.NewWriteQueryContext(context.Background(), txs.Scope().AddIndex(keys.Index(index)))
if err != nil {
tb.Fatalf("creating query context for test setup: %v", err)
}
return qcx, func() {
if err := qcx.Commit(); err != nil {
tb.Fatalf("committing write: %v", err)
}
}
}
// ImportKeyKey imports data into an index where both the index and
// the field are using string keys.
func (c *ShareableCluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys [][2]string) {
@ -324,8 +343,8 @@ func (c *ShareableCluster) ImportKeyKey(t testing.TB, index, field string, valAn
importRequest.RowKeys[i] = vk[0]
importRequest.ColumnKeys[i] = vk[1]
}
qcx := c.GetPrimary().API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := c.GetPrimary().IndexWideQcx(t, index)
defer done()
err := c.GetPrimary().API.Import(context.Background(), qcx, importRequest)
if err != nil {
t.Fatalf("importing keykey data: %v", err)
@ -356,8 +375,8 @@ func (c *ShareableCluster) ImportTimeQuantumKey(t testing.TB, index, field strin
importRequest.Timestamps[i] = entry.Ts
}
qcx := c.GetPrimary().API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := c.GetPrimary().IndexWideQcx(t, index)
defer done()
err := c.GetPrimary().API.Import(context.Background(), qcx, importRequest)
if err != nil {
t.Fatalf("importing keykey data: %v", err)
@ -384,8 +403,8 @@ func (c *ShareableCluster) ImportIntKey(t testing.TB, index, field string, pairs
importRequest.Values[i] = pair.Val
importRequest.ColumnKeys[i] = pair.Key
}
qcx := c.GetPrimary().API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := c.GetPrimary().IndexWideQcx(t, index)
defer done()
if err := c.GetPrimary().API.ImportValue(context.Background(), qcx, importRequest); err != nil {
t.Fatalf("importing IntKey data: %v", err)
}
@ -410,8 +429,8 @@ func (c *ShareableCluster) ImportIntID(t testing.TB, index, field string, pairs
importRequest.Values[i] = pair.Val
importRequest.ColumnIDs[i] = pair.ID
}
qcx := c.GetPrimary().API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := c.GetPrimary().IndexWideQcx(t, index)
defer done()
if err := c.GetPrimary().API.ImportValue(context.Background(), qcx, importRequest); err != nil {
t.Fatalf("importing IntID data: %v", err)
}
@ -437,8 +456,8 @@ func (c *ShareableCluster) ImportIDKey(t testing.TB, index, field string, pairs
importRequest.RowIDs[i] = pair.ID
importRequest.ColumnKeys[i] = pair.Key
}
qcx := c.GetPrimary().API.Txf().NewQcx()
defer qcx.Abort()
qcx, done := c.GetPrimary().IndexWideQcx(t, index)
defer done()
err := c.GetPrimary().API.Import(context.Background(), qcx, importRequest)
if err != nil {
t.Fatalf("importing IDKey data: %v", err)

81
test/holder.go
View file

@ -2,12 +2,15 @@
package test
import (
"context"
"math"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/testhook"
)
@ -22,13 +25,15 @@ type Holder struct {
func NewHolder(tb testing.TB) *Holder {
path, err := testhook.TempDir(tb, "pilosa-holder-")
if err != nil {
panic(err)
tb.Fatalf("requesting temp dir: %v", err)
}
cfg := pilosa.TestHolderConfig()
h := &Holder{Holder: pilosa.NewHolder(path, cfg), tb: tb}
return h
holder, err := pilosa.NewHolder(path, cfg)
if err != nil {
tb.Fatalf("creating holder for path %q: %v", path, err)
}
return &Holder{Holder: holder, tb: tb}
}
// MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
@ -71,6 +76,22 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
return &Index{Index: idx}
}
// Similar to the same method on commands, IndexWideQcx returns a Qcx that can write
// to the entire index (shard-agnostic), plus a function that will commit it or
// fail the test on error. It uses the holder's innate tb.
func (h *Holder) IndexWideQcx(index string) (qc.QueryContext, func()) {
txs := h.TxStore()
qcx, err := txs.NewWriteQueryContext(context.Background(), txs.Scope().AddIndex(keys.Index(index)))
if err != nil {
h.tb.Fatalf("creating query context for test setup: %v", err)
}
return qcx, func() {
if err := qcx.Commit(); err != nil {
h.tb.Fatalf("committing write: %v", err)
}
}
}
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
@ -78,16 +99,14 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
if err != nil {
panic(err)
}
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
row, err := f.Row(qcx, rowID)
if err != nil {
h.tb.Fatalf("retrieving row: %v", err)
}
// clone it so that mmapped storage doesn't disappear from under it
// once the qcx goes away.
return row
return row.Clone()
}
// ReadRow returns a Row for a given field. If the field does not exist,
@ -101,8 +120,8 @@ func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row {
if f == nil {
h.tb.Fatalf("read row from field %q/%q: field not found", index, field)
}
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
row, err := f.Row(qcx, rowID)
if err != nil {
@ -120,8 +139,8 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum
if err != nil {
panic(err)
}
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
row, err := f.RowTime(qcx, rowID, t, quantum)
if err != nil {
@ -146,17 +165,13 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time
panic(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
_, err = f.SetBit(qcx, rowID, columnID, t)
if err != nil {
h.tb.Fatalf("setting bit: %v", err)
}
err = qcx.Finish()
if err != nil {
h.tb.Fatalf("finishing qcx: %v", err)
}
}
// ClearBit clears a bit on the given field.
@ -167,17 +182,13 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
panic(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
_, err = f.ClearBit(qcx, rowID, columnID)
if err != nil {
h.tb.Fatalf("clearing bit: %v", err)
}
err = qcx.Finish()
if err != nil {
h.tb.Fatalf("finishing qcx: %v", err)
}
}
// MustSetBits sets columns on a row. Panic on error.
@ -196,17 +207,12 @@ func (h *Holder) SetValue(index, field string, columnID uint64, value int64) *In
panic(err)
}
qcx := h.Txf().NewWritableQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
_, err = f.SetValue(qcx, columnID, value)
if err != nil {
h.tb.Fatalf("setting value: %v", err)
}
err = qcx.Finish()
if err != nil {
h.tb.Fatalf("finishing qcx: %v", err)
}
return idx
}
@ -218,12 +224,11 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) {
panic(err)
}
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
val, exists, err := f.Value(qcx, columnID)
if err != nil {
panic(err)
h.tb.Fatalf("reading value: %v", err)
}
return val, exists
}
@ -237,8 +242,8 @@ func (h *Holder) Range(index, field string, op pql.Token, predicate int64) *pilo
panic(err)
}
qcx := h.Txf().NewQcx()
defer qcx.Abort()
qcx, done := h.IndexWideQcx(index)
defer done()
row, err := f.Range(qcx, field, op, predicate)
if err != nil {

9
test/index.go
View file

@ -6,25 +6,22 @@ import (
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/testhook"
)
// Index represents a test wrapper for pilosa.Index.
type Index struct {
*pilosa.Index
holder *Holder
}
// newIndex returns a new instance of Index, and the parent holder.
func newIndex(tb testing.TB) (*Holder, *Index) {
h := NewHolder(tb)
testhook.Cleanup(tb, func() {
h.Close()
})
h := MustOpenHolder(tb)
index, err := h.CreateIndex("i", "", pilosa.IndexOptions{})
if err != nil {
panic(err)
}
return h, &Index{Index: index}
return h, &Index{Index: index, holder: h}
}
// MustOpenIndex returns a new, opened index at a temporary path, or

1
translate_boltdb.go
View file

@ -512,7 +512,6 @@ func (r *BoltTranslateEntryReader) ReadEntry(entry *TranslateEntry) error {
type boltWrapper struct {
tx *bolt.Tx
db *bolt.DB
}
func (w *boltWrapper) Commit() error {

166
tx.go
View file

@ -1,166 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
//txkey "github.com/molecula/featurebase/v3/txkey"
)
// writable initializes Tx that update, use !writable for read-only.
const writable = true
// Tx providers offer transactional storage for high-level roaring.Bitmaps and
// low-level roaring.Containers.
//
// The common 4-tuple of (index, field, view, shard) jointly specify a fragment.
// A fragment conceptually holds one roaring.Bitmap.
//
// Within the fragment, the ckey or container-key is the uint64 that specifies
// the high 48-bits of the roaring.Bitmap 64-bit space.
// The ckey is used to retrieve a specific roaring.Container that
// is either a run, array, or raw bitmap. The roaring.Container is the
// low 16-bits of the roaring.Bitmap space. Its size is at most
// 8KB (2^16 bits / (8 bits / byte) == 8192 bytes).
//
// The grain of the transaction is guaranteed to be at least at the shard
// within one index. Therefore updates to any of the fields within
// the same shard will be atomically visible only once the transaction commits.
// Reads from another, concurrently open, transaction will not see updates
// that have not been committed.
type Tx interface {
// Type returns "roaring", "rbf", or one of the other
// Tx types at the top of txfactory.go
Type() string
// Rollback must be called at the end of read-only transactions. Either
// Rollback or Commit must be called at the end of writable transactions.
// It is safe to call Rollback multiple times, but it must be
// called at least once to release resources. Any Rollback after
// a Commit is ignored, so 'defer tx.Rollback()' should be commonly
// written after starting a new transaction.
//
// If there is an error during internal Rollback processing,
// this would be quite serious, and the underlying storage is
// expected to panic. Hence there is no explicit error returned
// from Rollback that needs to be checked.
Rollback()
// Commit makes the updates in the Tx visible to subsequent transactions.
Commit() error
// ContainerIterator loops over the containers in the conceptual
// roaring.Bitmap for the specified fragment.
// Calling Next() on the returned roaring.ContainerIterator gives
// you a roaring.Container that is either run, array, or raw bitmap.
// Return value 'found' is true when the ckey container was present.
// ckey of 0 gives all containers (in the fragment).
//
// ContainerIterator must not have side-effects.
//
// citer.Close() must be called when the client is done using it.
ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
// ApplyFilter applies a roaring.BitmapFilter to a specified shard,
// starting at the given container key. The filter's ConsiderData
// method may be called with transient Container objects which *must
// not* be retained or referenced after that function exits. Similarly,
// their data must not be retained. If you need the data later, you
// must copy it into some other memory.
ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error)
// ApplyRewriter applies a roaring.BitmapRewriter to a specified shard,
// starting at the given container key. The filter's ConsiderData
// method may be called with transient Container objects which *must
// not* be retained or referenced after that function exits. Similarly,
// their data must not be retained. If you need the data later, you
// must copy it into some other memory. However, it is safe to overwrite
// the returned container; for instance, you can DifferenceInPlace on
// it.
ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error)
// RoaringBitmap retrieves the roaring.Bitmap for the entire shard.
RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error)
// Container returns the roaring.Container for the given ckey
// (container-key or highbits) in the chosen fragment.
Container(index, field, view string, shard uint64, ckey uint64) (*roaring.Container, error)
// PutContainer stores c under the given ckey (container-key) in the specified fragment.
PutContainer(index, field, view string, shard uint64, ckey uint64, c *roaring.Container) error
// RemoveContainer deletes the roaring.Container under the given ckey (container-key)
// in the specified fragment.
RemoveContainer(index, field, view string, shard uint64, ckey uint64) error
// Add adds the 'a' values to the Bitmap for the fragment.
Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error)
// Remove removes the 'a' values from the Bitmap for the fragment.
Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error)
// Contains tests if the uint64 v is stored in the fragment's Bitmap.
Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error)
// Count returns the count of hot bits on the fragment.
Count(index, field, view string, shard uint64) (uint64, error)
// Max returns the maximum value set in the Bitmap for the fragment.
Max(index, field, view string, shard uint64) (uint64, error)
// Min returns the minimum value set in the Bitmap for the fragment.
Min(index, field, view string, shard uint64) (uint64, bool, error)
// CountRange returns the count of hot bits in the [start, end) range on the
// fragment.
CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error)
// OffsetRange returns a *roaring.Bitmap containing the portion of the Bitmap for the fragment
// which is specified by a combination of (offset, [start, end)).
//
// start - The value at which to start reading. This must be the zero value
// of a container; i.e. [0, 65536, ...]
// end - The value at which to end reading. This must be the zero value
// of a container; i.e. [0, 65536, ...]
// offset - The number of positions to shift the resulting bitmap. This must
// be the zero value of a container; i.e. [0, 65536, ...]
//
// For example, if (index, field, view, shard) represents the following bitmap:
// [1, 2, 3, 65536, 65539]
//
// then the following results are achieved based on (offset, start, end):
// (0, 0, 131072) => [1, 2, 3, 65536, 65539]
// (0, 65536, 131072) => [0, 3]
// (65536, 65536, 131072) => [65536, 65539]
// (262144, 65536, 131072) => [262144, 262147]
//
OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error)
// ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator.
//
// See the roaring package for details of the RoaringIterator.
//
// If clear is true, the bits from rit are cleared, otherwise they are set in the
// specifed fragment.
//
// ImportRoaringBits return values changed and rowSet may be inaccurate if
// the data []byte is supplied (the RoaringTx implementation neglects this for speed).
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
// GetSortedFieldViewList gets the set of FieldView(s)
GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error)
GetFieldSizeBytes(index, field string) (uint64, error)
}
// GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator,
// as a convenience if a Tx backend hasn't implemented this new function yet.
func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
iter, _, err := tx.ContainerIterator(index, field, view, shard, ckey)
if err != nil {
return err
}
// ApplyFilterToIterator closes the iterator for us.
return roaring.ApplyFilterToIterator(filter, iter)
}

133
tx_internal_test.go
View file

@ -1,133 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"bytes"
"sync"
"testing"
"github.com/molecula/featurebase/v3/roaring"
)
const countRangeMaxN = 8192
var countRangeSampleData []byte
var prepareCountRangeSampleData sync.Once
// The sample data for the counter is just a series of containers,
// each with cardinality equal to its container key.
func requireCountRangeSampleData(tb testing.TB) (*fragment, Tx) {
prepareCountRangeSampleData.Do(func() {
var arraySample [4096]uint16
// This horrible hack relies on a quirk of roaring's internals: It'll
// copy the bitmap if its length isn't exactly 1024. This lets us
// request that each container get its own copy of the bitmap.
var bitmapSample [1025]uint64
for i := range arraySample {
arraySample[i] = uint16(i * 2)
}
// Put corresponding bits in the bitmap...
for i := 0; i < 4096/32; i++ {
// bit 0 is 0x1, bit 2 is 0x4, so even-numbered bits
// are 0x5555....
bitmapSample[i] = 0x5555555555555555
}
bm := roaring.NewSliceBitmap()
for n := 0; n < 4096 && n < countRangeMaxN; n++ {
c := roaring.NewContainerArray(arraySample[:n])
bm.Put(uint64(n), c)
}
// Start filling in the missing bits. This starts us out with
// bitmap containers, but then eventually converts to things
// that are more likely to be run containers. At the end of this,
// we should have exactly the first 8,192 bits set, for a single
// run of 8k.
for n := 4096; n < 8192; n++ {
c := roaring.NewContainerBitmapN(bitmapSample[:], int32(n))
bm.Put(uint64(n), c)
w := n - 4096
bitmapSample[w/32] |= 1 << (((n % 32) * 2) + 1)
}
var asBytes bytes.Buffer
n, err := bm.WriteTo(&asBytes)
if err != nil {
tb.Fatalf("writing bitmap: %v", err)
}
countRangeSampleData = asBytes.Bytes()
tb.Logf("creating bitmap: %d containers, %d bytes of data", countRangeMaxN, n)
})
f, idx, tx := mustOpenFragment(tb)
// Properly close this transaction, but not the next one we create that the
// caller will be responsible for. The deferred callback will
// be a nop if the Commit happened.
defer tx.Rollback()
err := f.importRoaringT(tx, countRangeSampleData, false)
if err != nil {
tb.Fatalf("importing sample data: %v", err)
}
err = tx.Commit()
if err != nil {
tb.Fatalf("committing sample data: %v", err)
}
tx = idx.holder.txf.NewTx(Txo{Write: false, Index: idx, Fragment: f, Shard: 0})
return f, tx
}
func TestTx_CountRange(t *testing.T) {
f, tx := requireCountRangeSampleData(t)
defer f.Clean(t)
defer tx.Rollback()
// CountRange accesses the fragment without locking. Normally we only
// call it from inside a fragment routine with locking. Otherwise, you
// can have a race condition with snapshots, for instance.
f.mu.Lock()
defer f.mu.Unlock()
expected := uint64(0)
j := uint64(0)
for i := uint64(0); i < countRangeMaxN; i += 7 {
expected += i
if i%4 == 3 {
expected -= (j * 7) + 21
j += 7
}
// Every other bit gets set, for a total of i bits in container
// i, so they're all in the first (i*2) bits of the container.
got, err := tx.CountRange("i", "f", "v", 0, uint64(j)<<16, (uint64(i)<<16)+(i*2))
if err != nil {
t.Fatalf("counting range: %v", err)
}
if got != expected {
t.Fatalf("counting from container %d to %d, expected %d, got %d",
j, i, expected, got)
}
// The -i here undoes the +i at the top of this loop.
expected += (i * 7) + 21 - i
}
}
func BenchmarkTx_CountRange(b *testing.B) {
f, tx := requireCountRangeSampleData(b)
defer f.Clean(b)
defer tx.Rollback()
for k := 0; k < b.N; k++ {
expected := uint64(0)
j := uint64(0)
for i := uint64(0); i < countRangeMaxN; i += 7 {
if i%4 == 3 {
expected -= (j * 7) + 21
j += 7
}
got, err := tx.CountRange("i", "f", "v", 0, uint64(j)<<16, uint64(i)<<16)
if err != nil {
b.Fatalf("counting range: %v", err)
}
if got != expected {
b.Fatalf("counting from container %d to %d, expected %d, got %d",
j, i, expected, got)
}
expected += (i * 7) + 21
}
}
}

244
tx_test.go
View file

@ -1,244 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (
"context"
"fmt"
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
)
func queryIRABit(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowID uint64, index string) (bit bool) {
query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) // acctOwnerID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
if err != nil {
t.Fatalf("querying IRA bit: %v", err)
}
cols := res.Results[0].(*pilosa.Row).Columns()
for i := range cols {
if cols[i] == acctOwnerID {
return true
}
}
return false
}
func mustQueryAcct(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
if err != nil {
t.Fatalf("querying account: %v", err)
}
if len(res.Results) == 0 {
return 0
}
valCount := res.Results[0].(pilosa.ValCount)
return valCount.Val
}
func queryBalances(t *testing.T, m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, index string) (acct0bal, acct1bal int64) {
acct0bal = mustQueryAcct(t, m0api, acctOwnerID, fldAcct0, index)
acct1bal = mustQueryAcct(t, m0api, acctOwnerID, fldAcct1, index)
return
}
func TestAPI_ImportAtomicRecord(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
m0 := c.GetNode(0)
m0api := m0.API
ctx := context.Background()
index := c.Idx()
fieldAcct0 := "acct0"
fieldAcct1 := "acct1"
transferUSD := int64(100)
_ = transferUSD
opts := pilosa.OptFieldTypeInt(-1000, 1000)
_, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0api.CreateField(ctx, index, fieldAcct0, opts)
if err != nil {
t.Fatalf("creating fieldAcct0: %v", err)
}
_, err = m0api.CreateField(ctx, index, fieldAcct1, opts)
if err != nil {
t.Fatalf("creating fieldAcct1: %v", err)
}
iraField := "ira" // set field.
iraRowID := uint64(3)
_, err = m0api.CreateField(ctx, index, iraField)
if err != nil {
t.Fatalf("creating fieldIRA: %v", err)
}
acctOwnerID := uint64(78) // ColumnID
shard := acctOwnerID / ShardWidth
// setup 500 USD in acct1 and 700 USD in acct2.
// transfer 100 USD.
// should see 400 USD in acct, and 800 USD in acct2.
//
// setup initial balances
createAIRUpdate := func(acct0bal, acct1bal int64) (air *pilosa.AtomicRecord) {
ivr0 := &pilosa.ImportValueRequest{
Index: index,
Field: fieldAcct0,
Shard: shard,
ColumnIDs: []uint64{acctOwnerID},
Values: []int64{acct0bal},
}
ivr1 := &pilosa.ImportValueRequest{
Index: index,
Field: fieldAcct1,
Shard: shard,
ColumnIDs: []uint64{acctOwnerID},
Values: []int64{acct1bal},
}
ir0 := &pilosa.ImportRequest{
Index: index,
Field: iraField,
Shard: shard,
ColumnIDs: []uint64{acctOwnerID},
RowIDs: []uint64{iraRowID},
}
air = &pilosa.AtomicRecord{
Index: index,
Shard: shard,
Ivr: []*pilosa.ImportValueRequest{
ivr0, ivr1,
},
Ir: []*pilosa.ImportRequest{ir0},
}
return
}
expectedBalStartingAcct0 := int64(500)
expectedBalStartingAcct1 := int64(700)
air := createAIRUpdate(expectedBalStartingAcct0, expectedBalStartingAcct1)
//vv("BEFORE the first ImportAtomicRecord!")
qcx := m0api.Txf().NewQcx()
if err := m0api.ImportAtomicRecord(ctx, qcx, air); err != nil {
qcx.Abort()
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
//vv("AFTER the first ImportAtomicRecord!")
iraBit := queryIRABit(t, m0api, acctOwnerID, iraField, iraRowID, index)
if !iraBit {
t.Fatal("IRA bit should have been set")
}
startingBalanceAcct0, startingBalanceAcct1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
//vv("starting balance: acct0=%v, acct1=%v", startingBalanceAcct0, startingBalanceAcct1)
if startingBalanceAcct0 != expectedBalStartingAcct0 {
t.Fatalf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, startingBalanceAcct0)
}
if startingBalanceAcct1 != expectedBalStartingAcct1 {
t.Fatalf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, startingBalanceAcct1)
}
//vv("sad path: transferUSD %v from %v -> %v, with power loss half-way through", transferUSD, fieldAcct0, fieldAcct1)
opt := func(o *pilosa.ImportOptions) error {
o.SimPowerLossAfter = 1
return nil
}
expectedBalEndingAcct0 := expectedBalStartingAcct0 - 100
expectedBalEndingAcct1 := expectedBalStartingAcct1 + 100
air = createAIRUpdate(expectedBalEndingAcct0, expectedBalEndingAcct1)
qcx = m0api.Txf().NewQcx()
//vv("just before the SECOND ImportAtomicRecord, qcx is %p, should NOT BE NIL", qcx)
err = m0api.ImportAtomicRecord(ctx, qcx, air.Clone(), opt)
//err = m0api.ImportAtomicRecord(ctx, nil, air, opt)
if err != pilosa.ErrAborted {
t.Fatalf("expected ErrTxnAborted but got err='%#v'", err)
}
// sad path, cleanup
qcx.Abort()
qcx = nil
b0, b1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
//vv("after power failure tx, balance: acct0=%v, acct1=%v", b0, b1)
if b0 != expectedBalStartingAcct0 {
t.Fatalf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, b0)
}
if b1 != expectedBalStartingAcct1 {
t.Fatalf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, b1)
}
//vv("good: with power loss half-way, no change in account balances; acct0=%v; acct1=%v", b0, b1)
// next part of the test, just make sure we do the update.
//vv("happy path: transferUSD %v from %v -> %v, with no interruption.", transferUSD, fieldAcct0, fieldAcct1)
// happy path with no power failure half-way through.
qcx = m0api.Txf().NewQcx()
err = m0api.ImportAtomicRecord(ctx, qcx, air.Clone())
if err != nil {
t.Fatalf("importing record: %v", err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
eb0, eb1 := queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
// should have been applied this time.
if eb0 != expectedBalEndingAcct0 ||
eb1 != expectedBalEndingAcct1 {
t.Fatalf("problem: transaction did not get committed/applied. transferUSD=%v, but we see: startingBalanceAcct0=%v -> endingBalanceAcct0=%v; startingBalanceAcct1=%v -> endingBalanceAcct1=%v", transferUSD, startingBalanceAcct0, eb0, startingBalanceAcct1, eb1)
}
//vv("ending balance: acct0=%v, acct1=%v", eb0, eb1)
// clear all the bits
air.Ivr[0].Clear = true
air.Ivr[1].Clear = true
air.Ir[0].Clear = true
qcx = m0api.Txf().NewQcx()
err = m0api.ImportAtomicRecord(ctx, qcx, air)
if err != nil {
t.Fatalf("importing record: %v", err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
eb0, eb1 = queryBalances(t, m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
if eb0 != 0 ||
eb1 != 0 {
t.Fatal("problem: bits did not clear")
}
//vv("cleared balances: acct0=%v, acct1=%v", eb0, eb1)
iraBit = queryIRABit(t, m0api, acctOwnerID, iraField, iraRowID, index)
if iraBit {
t.Fatal("IRA bit should have been cleared")
}
}

702
txfactory.go
View file

@ -1,702 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"fmt"
"os"
"strings"
"sync"
"github.com/molecula/featurebase/v3/task"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
)
// public strings that pilosa/server/config.go can reference
const (
RBFTxn string = "rbf"
)
// DetectMemAccessPastTx true helps us catch places in api and executor
// where mmapped memory is being accessed after the point in time
// which the transaction has committed or rolled back. Since
// memory segments will be recycled by the underlying databases,
// this can lead to corruption. When DetectMemAccessPastTx is true,
// code in bolt.go will copy the transactionally viewed memory before
// returning it for bitmap reading, and then zero it or overwrite it
// with -2 when the Tx completes.
//
// Should be false for production.
const DetectMemAccessPastTx = false
var sep = string(os.PathSeparator)
// Qcx is a (Pilosa) Query Context.
//
// It flexibly expresses the desired grouping of Tx for mass
// rollback at a query's end. It provides one-time commit for
// an atomic import write Tx that involves multiple fragments.
//
// The most common use of Qcx is to call GetTx() to obtain a Tx locally,
// once the index/shard pair is known:
//
// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) {
// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...})
// defer finisher(&err0)
// ...
// }
//
// Qcx reuses read-only Tx on the same index/shard pair. See
// the Qcx.GetTx() for further discussion. The caveat is of
// course that your "new" read Tx actually has an "old" view
// of the database.
//
// At the moment, most
// writes to individual shards are commited eagerly and locally
// when the `defer finisher(&err0)` is run.
// This is done by returning a finisher that actually Commits,
// thus freeing the one write slot for re-use. A single
// writer is also required by RBF, so this design accomodates
// both.
//
// In contrast, the default read Tx generated (or re-used) will
// return a no-op finisher and the group of reads as a whole
// will be rolled back (mmap memory released) en-mass when
// Qcx.Abort() is called at the top-most level.
//
// Local use of a (Tx, finisher) pair obtained from Qcx.GetTx()
// doesn't need to care about these details. Local use should
// always invoke finisher(&err0) or finisher(nil) to complete
// the Tx within the local function scope.
//
// In summary write Tx are typically "local"
// and are never saved into the TxGroup. The parallelism
// supplied by TxGroup typically applies only to read Tx.
//
// The one exception is this rule is for the one write Tx
// used during the api.ImportAtomicRecord routine. There
// we make a special write Tx and use it for all matching writes.
// This is then committed at the final, top-level, Qcx.Finish() call.
//
// See also the Qcx.GetTx() example and the TxGroup description below.
type Qcx struct {
Grp *TxGroup
Txf *TxFactory
workers *task.Pool
// if we go back to using Qcx values, this must become a pointer,
// or otherwise be dealt with because copies of Mutex are a no-no.
mu sync.Mutex
// RequiredForAtomicWriteTx is used by api.ImportAtomicRecord
// to ensure that all writes happen on this one Tx.
RequiredForAtomicWriteTx *Tx
// efficient access to the options for RequiredForAtomicWriteTx
RequiredTxo *Txo
isRoaring bool
// top-level context is for a write, so re-use a
// writable tx for all reads and writes on each given
// shard
write bool
// don't allow automatic reuse now. Must manually call Reset, or NewQcx().
done bool
}
// Finish commits/rollsback all stored Tx. It no longer resets the
// Qcx for further operations automatically. User must call Reset()
// or NewQxc() again.
func (q *Qcx) Finish() (err error) {
q.mu.Lock()
defer q.mu.Unlock()
if q.RequiredForAtomicWriteTx != nil {
if q.RequiredTxo.Write {
err = (*q.RequiredForAtomicWriteTx).Commit() // PanicOn here on 2nd. is this a double commit?
} else {
(*q.RequiredForAtomicWriteTx).Rollback()
}
}
err2 := q.Grp.FinishGroup()
// drop the old group so we aren't holding references to all those Tx
q.Grp = q.Txf.NewTxGroup()
if !q.done {
_ = testhook.Closed(q.Txf.holder.Auditor, q, nil)
}
q.done = true
if err != nil {
return err
}
return err2
}
// Abort rolls back all Tx generated and stored within the Qcx.
// The Qcx is then reset and can be used again immediately.
func (q *Qcx) Abort() {
q.mu.Lock()
defer q.mu.Unlock()
if q.RequiredForAtomicWriteTx != nil {
(*q.RequiredForAtomicWriteTx).Rollback()
}
q.Grp.AbortGroup()
// drop the old group so we aren't holding references to all those Tx
q.Grp = q.Txf.NewTxGroup()
if !q.done {
_ = testhook.Closed(q.Txf.holder.Auditor, q, nil)
}
q.done = true
}
// Reset forgets everything are starts fresh with an empty
// group, ready for use again as if NewQcx() had been called.
func (q *Qcx) Reset() {
q.mu.Lock()
defer q.mu.Unlock()
q.unprotected_reset()
}
func (q *Qcx) unprotected_reset() {
q.RequiredForAtomicWriteTx = nil
q.RequiredTxo = nil
q.Grp = q.Txf.NewTxGroup()
q.done = false
}
// NewQcx allocates a freshly allocated and empty Grp.
// The top-level Qcx is not marked writable. Non-writable
// Qcx should not be used to request write Tx.
func (f *TxFactory) NewQcx() (qcx *Qcx) {
qcx = &Qcx{
Grp: f.NewTxGroup(),
Txf: f,
}
if f.typeOfTx == "roaring" {
qcx.isRoaring = true
}
if f.holder != nil {
if f.holder.executor != nil {
qcx.workers = f.holder.executor.workers
}
_ = testhook.Opened(f.holder.Auditor, qcx, nil)
}
return
}
// NewWritableQcx allocates a freshly allocated and empty Grp.
// The resulting Qcx is marked writable.
func (f *TxFactory) NewWritableQcx() (qcx *Qcx) {
qcx = &Qcx{
Grp: f.NewTxGroup(),
Txf: f,
}
if f.holder != nil && f.holder.executor != nil {
qcx.workers = f.holder.executor.workers
}
if f.typeOfTx == "roaring" {
qcx.isRoaring = true
}
_ = testhook.Opened(f.holder.Auditor, qcx, nil)
qcx.write = true
return
}
var NoopFinisher = func(perr *error) {}
var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset before re-use")
// GetTx is used like this:
//
// someFunc(ctx context.Context, shard uint64) (_ interface{}, err0 error) {
//
// tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
// defer finisher(&err0)
//
// return e.executeIncludesColumnCallShard(ctx, tx, index, c, shard, col)
// }
//
// Note we are tracking the returned err0 error value of someFunc(). An option instead is to say
//
// defer finisher(nil)
//
// This means always Commit writes, ignoring if there were errors. This style
// is expected to be rare compared to the typical
//
// defer finisher(&err0)
//
// invocation, where err0 is your return from the enclosing function error.
// If the Tx is local and not a part of a group, then the finisher
// consults that error to decides whether to Commit() or Rollback().
//
// If instead the Tx becomes part of a group, then the local finisher() is
// always a no-op, in deference to the Qcx.Finish()
// or Qcx.Abort() calls.
//
// Take care the finisher(&err) is capturing the address of the
// enclosing function's err and that it has not been shadowed
// locally by another _, err := f() call. For this reason, it can
// be clearer (and much safer) to rename the enclosing functions 'err' to 'err0',
// to make it clear we are referring to the first and final error.
func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
if qcx.workers != nil {
qcx.workers.Block()
defer qcx.workers.Unblock()
}
qcx.mu.Lock()
defer qcx.mu.Unlock()
if qcx.done {
return nil, nil, ErrQcxDone
}
// roaring uses finer grain, a file per fragment rather than
// db per shard. So we can't re-use the readTx. Moreover,
// roaring Tx are No-ops anyway, so just give it a new Tx
// everytime.
if qcx.isRoaring {
return qcx.Txf.NewTx(o), NoopFinisher, nil
}
// qcx.write reflects the top executor determination
// if a write will be happen at some point, in which case, to avoid
// locking problems with multi-shard things, we (probably incorrectly)
// treat every Tx as its own individual separate Tx.
//
// But we still want to open non-write transactions individually, we
// just can't recycle them (because write operations will come in and
// we want them to work and commit right away so we're not holding a write
// lock for long).
writeLogic := o.Write || qcx.write
// In general, we make ALL write transactions local, and never reuse them
// below. Previously this was to help lmdb.
//
// *However* there is one exception: when we have set RequiredForAtomicWriteTx
// for the importing of an AtomicRequest, then we must use that
// our single RequiredForAtomicWriteTx for all writes until it
// is cleared. This one is kept separately from the read TxGroup.
//
if o.Write && qcx.RequiredForAtomicWriteTx != nil {
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
vprint.PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
vprint.PanicOn("o.Index annot be nil")
}
if ro.Index == nil {
vprint.PanicOn("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
vprint.PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
return *qcx.RequiredForAtomicWriteTx, NoopFinisher, nil
}
if !writeLogic && qcx.Grp != nil {
// read, with a group in place.
finisher = func(perr *error) {} // finisher is a returned value
already := false
tx, already = qcx.Grp.AlreadyHaveTx(o)
if already {
return
}
tx = qcx.Txf.NewTx(o)
qcx.Grp.AddTx(tx, o)
return
}
// non atomic writes or not grouped reads
tx = qcx.Txf.NewTx(o)
if o.Write {
finisherDone := false
finisher = func(perr *error) {
if finisherDone {
return
}
finisherDone = true // only Commit once.
// so defer finisher(nil) means always Commit writes, ignoring
// the enclosing functions return status.
if perr == nil || *perr == nil {
vprint.PanicOn(tx.Commit())
} else {
tx.Rollback()
}
}
} else {
// read-only txn
finisher = func(perr *error) {
tx.Rollback()
}
}
return
}
// StartAtomicWriteTx allocates a Tx and stores it
// in qcx.RequiredForAtomicWriteTx. All subsequent writes
// to this shard/index will re-use it.
func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
if !o.Write {
vprint.PanicOn("must have o.Write true")
}
qcx.mu.Lock()
defer qcx.mu.Unlock()
if qcx.RequiredForAtomicWriteTx == nil {
// new Tx needed
tx := qcx.Txf.NewTx(o)
qcx.RequiredForAtomicWriteTx = &tx
qcx.RequiredTxo = &o
return
}
// re-using existing
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
vprint.PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
vprint.PanicOn("o.Index annot be nil")
}
if ro.Index == nil {
vprint.PanicOn("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
vprint.PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
}
func (qcx *Qcx) ListOpenTx() string {
return qcx.Grp.String()
}
// TxFactory abstracts the creation of Tx interface-level
// transactions so that RBF, or Roaring-fragment-files, or several
// of these at once in parallel, is used as the storage and transction layer.
type TxFactory struct {
typeOfTx string
typ txtype
dbsClosed bool // idemopotent CloseDB()
dbPerShard *DBPerShard
holder *Holder
}
// integer types for fast switch{}
type txtype int
const (
noneTxn txtype = 0
rbfTxn txtype = 2
)
// DirectoryName just returns a string version of the transaction type. We
// really need to consolidate the storage backend and tx stuff because it's
// currently rather confusing. This method should be addressed (i.e.
// replaced/removed) during that refactor.
func (ty txtype) DirectoryName() string {
switch ty {
case rbfTxn:
return "rbf"
}
vprint.PanicOn(fmt.Sprintf("unkown txtype %v", int(ty)))
return ""
}
func MustBackendToTxtype(backend string) (typ txtype) {
if strings.Contains(backend, "_") {
panic("blue-green comparisons removed")
}
switch backend {
case RBFTxn: // "rbf"
return rbfTxn
}
panic(fmt.Sprintf("unknown backend '%v'", backend))
}
// NewTxFactory always opens an existing database. If you
// want to a fresh database, os.RemoveAll on dir/name ahead of time.
// We always store files in a subdir of holderDir.
func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) {
typ := MustBackendToTxtype(backend)
f = &TxFactory{
typ: typ,
typeOfTx: backend,
holder: holder,
}
f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder)
if f.hasRBF() {
holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig)
}
return f, err
}
// Open should be called only once the index metadata is loaded
// from Holder.Open(), so we find all of our indexes.
func (f *TxFactory) Open() error {
return f.dbPerShard.LoadExistingDBs()
}
// Txo holds the transaction options
type Txo struct {
Write bool
Field *Field
Index *Index
Fragment *fragment
Shard uint64
dbs *DBShard
}
func (f *TxFactory) TxType() string {
return f.typeOfTx
}
func (f *TxFactory) TxTyp() txtype {
return f.typ
}
func (f *TxFactory) DeleteIndex(name string) (err error) {
return f.dbPerShard.DeleteIndex(name)
}
func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
return f.dbPerShard.DeleteFieldFromStore(index, field, fieldPath)
}
func (f *TxFactory) DeleteFragmentFromStore(
index, field, view string, shard uint64, frag *fragment,
) (err error) {
return f.dbPerShard.DeleteFragment(index, field, view, shard, frag)
}
// CloseIndex is a no-op. This seems to be in place for debugging purposes.
func (f *TxFactory) CloseIndex(idx *Index) error {
return nil
}
func (f *TxFactory) Close() (err error) {
if f.dbsClosed {
return nil
}
f.dbsClosed = true
return f.dbPerShard.Close()
}
var globalUseStatTx = false
func init() {
v := os.Getenv("PILOSA_CALLSTAT")
if v != "" {
globalUseStatTx = true
}
}
// TxGroup holds a set of read transactions
// that will en-mass have Rollback() (for the read set) called on
// them when TxGroup.Finish() is invoked.
// Alternatively, TxGroup.Abort() will call Rollback()
// on all Tx group memebers.
//
// It used to have writes but we never actually used that because
// of the Qcx needing to make every commit get its own transaction.
type TxGroup struct {
mu sync.Mutex
fac *TxFactory
reads []Tx
finished bool
all map[grpkey]Tx
}
type grpkey struct {
index string
shard uint64
}
func mustHaveIndexShard(o *Txo) {
if o.Index == nil || o.Index.name == "" {
vprint.PanicOn("index must be set on Txo")
}
}
func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) {
mustHaveIndexShard(&o)
g.mu.Lock()
defer g.mu.Unlock()
key := grpkey{index: o.Index.name, shard: o.Shard}
tx, already = g.all[key]
return
}
func (g *TxGroup) String() (r string) {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.reads) == 0 {
return "<empty-TxGroup>"
}
r += "\n"
for i, tx := range g.reads {
r += fmt.Sprintf("[%v]read: %#v,\n", i, tx)
}
return r
}
// NewTxGroup
func (f *TxFactory) NewTxGroup() (g *TxGroup) {
g = &TxGroup{
fac: f,
all: make(map[grpkey]Tx),
}
return
}
// AddTx adds tx to the group.
func (g *TxGroup) AddTx(tx Tx, o Txo) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
vprint.PanicOn("in TxGroup.Finish(): TxGroup already finished")
}
g.reads = append(g.reads, tx)
key := grpkey{index: o.Index.name, shard: o.Shard}
prior, ok := g.all[key]
if ok {
vprint.PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx))
}
g.all[key] = tx
}
// Finish commits the write tx and calls Rollback() on
// the read tx contained in the group. Either Abort() or Finish() must
// be called on the TxGroup exactly once.
func (g *TxGroup) FinishGroup() (err error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
vprint.PanicOn("in TxGroup.Finish(): TxGroup already finished")
}
g.finished = true
for _, r := range g.reads {
r.Rollback()
}
return
}
// Abort calls Rollback() on all the group Tx, and marks
// the group as finished. Either Abort() or Finish() must
// be called on the TxGroup.
func (g *TxGroup) AbortGroup() {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
// defer Abort() probably gets here often by default, just ignore.
return
}
g.finished = true
for _, r := range g.reads {
r.Rollback()
}
}
func (f *TxFactory) NewTx(o Txo) (txn Tx) {
defer func() {
if globalUseStatTx {
txn = newStatTx(txn)
}
}()
indexName := ""
if o.Index != nil {
indexName = o.Index.name
}
if o.Fragment != nil {
if o.Fragment.index() != indexName {
vprint.PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.index='%v' but indexName='%v'", o.Fragment.index(), indexName))
}
if o.Fragment.shard != o.Shard {
vprint.PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.shard='%v' but o.Shard='%v'", o.Fragment.shard, o.Shard))
}
}
// look up in the collection of open databases, and get our
// per-shard database. Opens a new one if needed.
dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index)
vprint.PanicOn(err)
if dbs.Shard != o.Shard {
vprint.PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard)))
}
//vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W)
o.dbs = dbs
tx, err := dbs.NewTx(o.Write, indexName, o)
if err != nil {
vprint.PanicOn(errors.Wrap(err, "dbs.NewTx transaction errored"))
}
return tx
}
// has to match the const strings at the top of the file.
func (ty txtype) String() string {
switch ty {
case noneTxn:
return "noneTxn"
case rbfTxn:
return "rbf"
}
vprint.PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
return ""
}
func dirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
var _ = anyGlobalDBWrappersStillOpen // happy linter
func anyGlobalDBWrappersStillOpen() bool {
return globalRbfDBReg.Size() != 0
}
func (f *TxFactory) hasRBF() bool {
return f.typ == rbfTxn
}
func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) {
dbs, err := f.dbPerShard.GetDBShard(index, shard, idx)
if err != nil {
return "", errors.Wrap(err, fmt.Sprintf("GetDBShardPath(index='%v', shard='%v', ty='%v')", index, shard, ty.String()))
}
shardPath = dbs.pathForType(ty)
return
}
func (txf *TxFactory) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) {
return txf.dbPerShard.GetFieldView2ShardsMapForIndex(idx)
}

19
txfactory_internal_test.go
View file

@ -1,19 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"testing"
)
func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) {
// txtype.String() method MUST return strings that match
// our const definitions at the top of txfactory.go.
check := []txtype{rbfTxn}
expect := []string{RBFTxn}
for i, chk := range check {
obs := chk.String()
if obs != expect[i] {
t.Fatalf("expected '%v' but got '%v'", expect[i], obs)
}
}
}

253
txkey/txkey.go
View file

@ -1,253 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Package txkey consolidates in one place the use of keys to index into our
// various storage/txn back-ends. Databases LMDB and rbfDB both use it,
// so that debug Dumps are comparable.
package txkey
import (
"encoding/binary"
"fmt"
)
// FieldView is here to avoid circular import.
type FieldView struct {
Field string
View string
}
func FieldViewFromPrefix(prefix []byte) FieldView {
_, field, view, _ := SplitPrefix(prefix)
return FieldView{Field: field, View: view}
}
func FieldViewFromFullKey(fullKey []byte) FieldView {
_, field, view, _, _ := Split(fullKey)
return FieldView{Field: field, View: view}
}
// Key produces the bytes that we use as a key to query the storage/tx engine.
// The roaringContainerKey argument to Key() is a container key into a roaring Container.
// The return value from Key() is constructed as follows:
//
// ~index%field;view:shard<ckey#
//
// where shard and ckey are always exactly 8 bytes, uint64 big-endian encoded.
//
// Keys always start with either '~' or '>'. Keys always end with '#'.
// Keys always contain exactly one each of '%', ';', ':' and '<', in that order.
// The index is between the first byte and the '%'. It must be at least 1 byte long.
// The field is between the '%' and the ';'. It must be at least 1 byte long.
// The view is between the ';' and the ':'. It must be at least 1 byte long.
// The shard is the 8 bytes between the ':' and the '<'.
// The ckey is the 8 bytes between the '<' and the '#'.
// The Prefix of a key ends at, and includes, the '<'. It is at least 16 bytes long.
// The index, field, and view are not allowed to contain these reserved bytes:
//
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable
// base-64 encoding.
//
// The shortest possible key is 25 bytes. It would be laid out like this:
//
// ~i%f;v:12345678<12345678#
// 1234567890123456789012345
//
// keys starting with '~' are regular value keys.
// keys starting with '>' are symlink keys.
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) {
prefix := Prefix(index, field, view, shard)
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
return append(prefix, ckey[:]...)
}
// ShardFromKey key example: index/field;view:shard<ckey
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
//
// shard ckey
func ShardFromKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// ckey is always exactly 8 bytes long
// shard is always exactly 8 bytes long
shard = binary.BigEndian.Uint64(bkey[(n - 18):(n - 10)])
return
}
func ShardFromPrefix(prefix []byte) (shard uint64) {
n := len(prefix)
shard = binary.BigEndian.Uint64(prefix[(n - 9):(n - 1)])
return
}
// KeyAndPrefix returns the equivalent of Key() and Prefix() calls.
func KeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = Prefix(index, field, view, shard)
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
key = append(prefix, ckey[:]...)
return
}
var _ = KeyAndPrefix // keep linter happy
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 25 {
panic(fmt.Sprintf("bkey too short, must have at least 25 bytes: '%v'", string(bkey)))
}
typ := bkey[0]
if typ != '~' && typ != '>' {
panic(fmt.Sprintf("bkey did not start with '~' for value nor '>' for symlink: '%v'", string(bkey)))
}
if bkey[n-10] != '<' {
panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey)))
}
if bkey[n-19] != ':' {
panic(fmt.Sprintf("bkey did not have '<' at 18 bytes from the end: '%v'", string(bkey)))
}
if bkey[n-1] != '#' {
panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey)))
}
}
// KeyExtractContainerKey extracts the containerKey from bkey.
// key example: index/field;view:shard<ckey
// shortest: =i%f;v:12345678<12345678#
//
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
n := len(bkey)
MustValidateKey(bkey)
containerKey = binary.BigEndian.Uint64(bkey[(n - 9):(n - 1)])
return
}
func AllShardPrefix(index, field, view string) (r []byte) {
r = make([]byte, 0, 64)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, ':')
return
}
// Prefix returns everything from Key up to and
// including the '<' byte in a Key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with Key() and KeyExtractContainerKey().
func Prefix(index, field, view string, shard uint64) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, ':')
var sh [8]byte
binary.BigEndian.PutUint64(sh[:], shard)
r = append(r, sh[:]...)
r = append(r, '<')
return
}
// IndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
// remove all storage associated with one index.
//
// The full name of the index must be provided, no partial index names will work.
//
// The returned prefix is terminated by '%' and so DeleteIndex("i") will not delete the index "i2".
func IndexOnlyPrefix(indexName string) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(indexName)...)
r = append(r, '%')
return
}
// same for deleting a whole field.
func FieldPrefix(index, field string) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
return
}
// PrefixFromKey key example: index/field;view:shard<ckey
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
//
// shard ckey
func PrefixFromKey(bkey []byte) (prefix []byte) {
n := len(bkey)
return bkey[:(n - 9)]
}
func ToString(bkey []byte) (r string) {
index, field, view, shard, ckey := Split(bkey)
return fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@%020d", index, field, view, shard, ckey)
}
func PrefixToString(pre []byte) (r string) {
index, field, view, shard := SplitPrefix(pre)
return fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';", index, field, view, shard)
}
func Split(bkey []byte) (index, field, view string, shard, ckey uint64) {
ckey = KeyExtractContainerKey(bkey)
n := len(bkey)
index, field, view, shard = SplitPrefix(bkey[:(n - 9)])
return
}
// full key: ~index%field;view:shard<ckey#
// prefix : ~index%field;view:shard<
func SplitPrefix(pre []byte) (index, field, view string, shard uint64) {
n := len(pre)
shard = binary.BigEndian.Uint64(pre[(n - 9):(n - 1)])
// prefix: =index%field;view:shard<
goal := byte('%')
beg := 1
for i := 1; i < n; i++ {
c := pre[i]
switch goal {
case '%':
if c == goal {
index = string(pre[beg:i])
beg = i + 1
goal = byte(';')
}
case ';':
if c == goal {
field = string(pre[beg:i])
beg = i + 1
view = string(pre[beg:(n - 10)])
return
}
}
}
panic(fmt.Sprintf("malformed prefix '%v' / '%#v', could not Split", string(pre), pre))
}

125
txkey/txkey_test.go
View file

@ -1,125 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package txkey
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"testing"
)
func Test_KeyPrefix(t *testing.T) {
// Prefix() must agree with Key(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
needle := Key(index, field, view, shard, 0)
// prefix example: i%f;v:12345678<
prefix := Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("Prefix() output '%v'was not a prefix of Key() '%v'", string(needle), string(prefix)))
}
npre := len(prefix)
nneed := len(needle)
if npre+9 != nneed {
panic(fmt.Sprintf("Prefix() output len %v '%v' was not 9 characters shorter than Key() len %v '%v'", npre, string(prefix), nneed, string(needle)))
}
// validate assumption that KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key<"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
KeyExtractContainerKey(prefix) // should panic.
}()
}
func Test_ShardFromKey(t *testing.T) {
key := []byte("~i%f;v:12345678<12345678#")
binary.BigEndian.PutUint64(key[7:15], 1)
if ShardFromKey(key) != 1 {
panic("problem")
}
binary.BigEndian.PutUint64(key[7:15], 0)
if ShardFromKey(key) != 0 {
panic("problem")
}
binary.BigEndian.PutUint64(key[7:15], 18446744073709551615)
if ShardFromKey(key) != 18446744073709551615 {
panic("problem")
}
func() {
defer func() {
r := recover()
if r == nil {
panic("should have panic-ed")
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
ShardFromKey([]byte("~i%f;v:12345678<1234567#"))
}()
}
func Test_PrefixFromKey(t *testing.T) {
k := []byte("~i%f;v:12345678<12345678#")
x := []byte("~i%f;v:12345678<")
pre := PrefixFromKey(k)
if !bytes.Equal(pre, x) {
nx := len(x)
npre := len(pre)
if nx != npre {
panic(fmt.Sprintf("nx=%v, npre=%v; expected '%v', observed '%v'", nx, npre, string(x), string(pre)))
}
for i := 0; i < nx; i++ {
if x[i] != pre[i] {
panic(fmt.Sprintf("first diff at index %v, expected '%v', observed '%v'", i, string(x[:i]), string(pre[:i])))
}
}
panic(fmt.Sprintf("expected:\n%v\n, observed:\n%v\n", string(x), string(pre)))
}
}
func Test_Split(t *testing.T) {
bkey := []byte("~i%f;v:12345678<12345678#")
var xshard uint64 = 18446744073709551615
var xckey uint64 = 43
binary.BigEndian.PutUint64(bkey[7:15], xshard)
binary.BigEndian.PutUint64(bkey[16:24], xckey)
i, f, v, shard, ckey := Split(bkey)
if i != "i" {
panic("wrong index")
}
if f != "f" {
panic("wrong field")
}
if v != "v" {
panic("wrong view")
}
if shard != xshard {
panic("wrong shard")
}
if ckey != xckey {
panic("wrong ckey")
}
}

3
util.go
View file

@ -27,9 +27,6 @@ const LeftShifted16MaxContainerKey = uint64(0xffffffffffff0000) // or math.MaxUi
//////////////////////////////////
// helper utility functions
func highbits(v uint64) uint64 { return v >> 16 }
func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
// GetLoopProgress returns the estimated remaining time to iterate through some
// items as well as the loop completion percentage with the following
// parameters:

146
view.go
View file

@ -13,7 +13,9 @@ import (
"sync/atomic"
"time"
"github.com/molecula/featurebase/v3/keys"
"github.com/molecula/featurebase/v3/pql"
qc "github.com/molecula/featurebase/v3/querycontext"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
@ -111,20 +113,11 @@ func (v *view) addKnownShard(shard uint64) {
vprint.PanicOn(err)
}
// removeKnownShard removes a known shard from v. See the notes on addKnownShard.
func (v *view) removeKnownShard(shard uint64) {
if atomic.LoadUint32(&v.knownShardsCopied) == 1 {
v.knownShards = v.knownShards.Clone()
atomic.StoreUint32(&v.knownShardsCopied, 0)
}
_, _ = v.knownShards.Remove(shard)
}
// openWithShardSet opens the view. Importantly, it
// only opens the fragments that have data. This saves
// a ton of time. If you have no data and want a new
// view, call view.openEmpty().
func (v *view) openWithShardSet(ss *shardSet) error {
func (v *view) openWithShardSet(ss keys.ViewContents) error {
if v.knownShards == nil {
v.knownShards = roaring.NewSliceBitmap()
}
@ -134,19 +127,18 @@ func (v *view) openWithShardSet(ss *shardSet) error {
v.cacheType = CacheTypeNone
}
shards := ss.CloneMaybe()
var frags []*fragment
for shard := range shards {
frag := v.newFragment(shard)
for shard := range ss {
frag := v.newFragment(uint64(shard))
frags = append(frags, frag)
v.fragments[frag.shard] = frag
}
nGoro := runtime.NumCPU()
if v.idx.holder.txf.TxType() != "roaring" {
nGoro = nGoro / 4
}
// We used to only divide by 4 if we weren't using the
// roaring backend, but we no longer have it, so this is
// unconditional for now. If we add new backends, this may
// want reconsidering.
nGoro := runtime.NumCPU() / 4
if nGoro < 4 {
nGoro = 4
}
@ -179,8 +171,8 @@ func (v *view) openWithShardSet(ss *shardSet) error {
// serial, not parallel, because no locking inside addKnownShard at the moment.
// TODO(jea): is this slow on a cluster? can we optimize it
// by running it on a goroutine in the background?
for shard := range shards {
v.addKnownShard(shard)
for shard := range ss {
v.addKnownShard(uint64(shard))
}
_ = testhook.Opened(v.holder.Auditor, v, nil)
@ -401,38 +393,15 @@ func (v *view) newFragment(shard uint64) *fragment {
return frag
}
// deleteFragment removes the fragment from the view.
func (v *view) deleteFragment(shard uint64) error {
v.mu.Lock()
defer v.mu.Unlock()
f := v.fragments[shard]
if f == nil {
return ErrFragmentNotFound
}
v.holder.Logger.Infof("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
idx := f.holder.Index(v.index)
f.Close()
if err := idx.holder.txf.DeleteFragmentFromStore(f.index(), f.field(), f.view(), f.shard, f); err != nil {
return errors.Wrap(err, "DeleteFragment")
}
delete(v.fragments, shard)
v.removeKnownShard(shard)
return nil
}
// row returns a row for a shard of the view.
func (v *view) row(qcx *Qcx, rowID uint64) (*Row, error) {
func (v *view) row(qcx qc.QueryContext, rowID uint64) (*Row, error) {
row := NewRow()
for _, frag := range v.allFragments() {
tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard})
qr, err := frag.qcxRead(qcx)
if err != nil {
return nil, err
}
defer finisher(&err)
fr, err := frag.row(tx, rowID)
fr, err := frag.row(qr, rowID)
if err != nil {
return nil, err
} else if fr == nil {
@ -446,7 +415,7 @@ func (v *view) row(qcx *Qcx, rowID uint64) (*Row, error) {
// mutexCheck checks all available fragments for duplicate values. The return
// is map[column]map[shard][]values for collisions only.
func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
func (v *view) mutexCheck(ctx context.Context, qcx qc.QueryContext, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
// We don't need the context, we just want the context-awareness on the error groups.
// It would be nice if the inner functions could use this too...
eg, _ := errgroup.WithContext(ctx)
@ -462,12 +431,14 @@ func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int
defer func() {
<-throttle
}()
tx, finisher, err := qcx.GetTx(Txo{Index: v.idx, Shard: frag.shard})
qr, err := frag.qcxRead(qcx)
if err != nil {
return err
}
defer finisher(&err)
results[i], err = frag.mutexCheck(tx, details, limit)
if err != nil {
return err
}
results[i], err = frag.mutexCheck(qr, details, limit)
if err != nil {
return err
}
@ -500,83 +471,86 @@ func (v *view) mutexCheck(ctx context.Context, qcx *Qcx, details bool, limit int
}
// setBit sets a bit within the view.
func (v *view) setBit(qcx *Qcx, rowID, columnID uint64) (changed bool, err error) {
func (v *view) setBit(qcx qc.QueryContext, rowID, columnID uint64) (changed bool, err error) {
shard := columnID / ShardWidth
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard})
defer finisher(&err)
var frag *fragment
frag, err = v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}
return frag.setBit(tx, rowID, columnID)
qw, err := frag.qcxWrite(qcx)
if err != nil {
return changed, err
}
return frag.setBit(qw, rowID, columnID)
}
// clearBit clears a bit within the view.
func (v *view) clearBit(qcx *Qcx, rowID, columnID uint64) (changed bool, err error) {
func (v *view) clearBit(qcx qc.QueryContext, rowID, columnID uint64) (changed bool, err error) {
shard := columnID / ShardWidth
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard})
defer finisher(&err)
frag := v.Fragment(shard)
if frag == nil {
return false, nil
}
return frag.clearBit(tx, rowID, columnID)
qw, err := frag.qcxWrite(qcx)
if err != nil {
return changed, err
}
return frag.clearBit(qw, rowID, columnID)
}
// value uses a column of bits to read a multi-bit value.
func (v *view) value(qcx *Qcx, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) {
func (v *view) value(qcx qc.QueryContext, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) {
shard := columnID / ShardWidth
tx, finisher, err := qcx.GetTx(Txo{Write: false, Index: v.idx, Shard: shard})
defer finisher(&err)
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return value, exists, err
}
qr, err := frag.qcxRead(qcx)
if err != nil {
return value, exists, err
}
return frag.value(tx, columnID, bitDepth)
return frag.value(qr, columnID, bitDepth)
}
// setValue uses a column of bits to set a multi-bit value.
func (v *view) setValue(qcx *Qcx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
func (v *view) setValue(qcx qc.QueryContext, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard})
defer finisher(&err)
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}
return frag.setValue(tx, columnID, bitDepth, value)
qw, err := frag.qcxWrite(qcx)
if err != nil {
return changed, err
}
return frag.setValue(qw, columnID, bitDepth, value)
}
// clearValue removes a specific value assigned to columnID
func (v *view) clearValue(qcx *Qcx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
func (v *view) clearValue(qcx qc.QueryContext, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: v.idx, Shard: shard})
defer finisher(&err)
frag := v.Fragment(shard)
if frag == nil {
return false, nil
}
return frag.clearValue(tx, columnID, bitDepth, value)
qw, err := frag.qcxWrite(qcx)
if err != nil {
return changed, err
}
return frag.clearValue(qw, columnID, bitDepth, value)
}
// rangeOp returns rows with a field value encoding matching the predicate.
func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint64, predicate int64) (_ *Row, err0 error) {
func (v *view) rangeOp(qcx qc.QueryContext, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) {
r := NewRow()
for _, frag := range v.allFragments() {
tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: v.idx, Shard: frag.shard})
qr, err := frag.qcxRead(qcx)
if err != nil {
return nil, err
}
defer finisher(&err0)
other, err := frag.rangeOp(tx, op, bitDepth, predicate)
other, err := frag.rangeOp(qr, op, bitDepth, predicate)
if err != nil {
return nil, err
}
@ -585,18 +559,22 @@ func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint64, predicate int64)
return r, nil
}
func (v *view) bitDepth(shards []uint64) (uint64, error) {
func (v *view) bitDepth(qcx qc.QueryContext, shards keys.Shards) (uint64, error) {
var maxBitDepth uint64
for _, shard := range shards {
for shard := range shards {
v.mu.RLock()
frag, ok := v.fragments[shard]
frag, ok := v.fragments[uint64(shard)]
v.mu.RUnlock()
if !ok || frag == nil {
continue
}
qr, err := frag.qcxRead(qcx)
if err != nil {
return 0, err
}
bd, err := frag.bitDepth()
bd, err := frag.bitDepth(qr)
if err != nil {
return 0, errors.Wrapf(err, "getting fragment(%d) bit depth", shard)
}

32
view_internal_test.go
View file

@ -17,38 +17,6 @@ func mustOpenView(tb testing.TB) *view {
return v
}
// Ensure view can open and retrieve a fragment.
func TestView_DeleteFragment(t *testing.T) {
v := mustOpenView(t)
shard := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.deleteFragment(shard)
if err != nil {
t.Fatal(err)
}
if v.Fragment(shard) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same shard, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {
t.Fatal("failed to create new fragment")
}
}
// Ensure that simultaneous attempts to grab a new fragment don't clash even
// if the broadcast operation takes a bit of time.
func TestView_CreateFragmentRace(t *testing.T) {