scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

database per shard, HolderConfig, rbf bit-wise import speedups.

Browse source 
- introduce Query Context (Qcx) for managing database-per-shard.
- replaces the MultiTx, so mtx.go is retired and removed.
- introduces the HolderConfig struct and all Holders now have
  a path from birth.
- rbf speedups on bitwise writes
- badgerdb is removed due to unresolvable write conflicts.

fixes #703 #676
This commit is contained in:
Ben Johnson 2020-09-02 08:31:02 -06:00 • committed by Jason Aten
parent 60da12a8e5
commit 150c8a5b06
80 changed files with 4329 additions and 5550 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
Makefile
View file

@ -57,7 +57,7 @@ testvsub:
set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i"; \
cd $$i; pwd; \
go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v || break; \
go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
echo; echo "999 done testing subpkg $$i"; \
cd ..; \
done
@ -66,7 +66,7 @@ testvsub-race:
set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i -race"; \
cd $$i; pwd; \
go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race || break; \
go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
echo; echo "999 done testing subpkg $$i -race"; \
cd ..; \
done
@ -184,6 +184,14 @@ docker-tag-push: vendor
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
pilosa-keydump:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
pilosa-chk:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
@ -216,7 +224,7 @@ topt-rbf:
topt-rbf-race:
mv log.topt.rbf-race log.topt.rbf-race.prev || true
PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf-race
PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m 2>&1 | tee log.topt.rbf-race
@echo " log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l
@echo " log.topt.rbf-race red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l

174
api.go
View file

@ -137,6 +137,10 @@ func (api *API) Close() error {
return nil
}
func (api *API) Txf() *TxFactory {
return api.holder.Txf()
}
// Query parses a PQL query out of the request and executes it.
func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.Query")
@ -330,7 +334,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) {
type importJob struct {
ctx context.Context
tx Tx
qcx *Qcx
req *ImportRoaringRequest
shard uint64
field *Field
@ -339,7 +343,7 @@ type importJob struct {
func importWorker(importWork chan importJob) {
for j := range importWork {
err := func() error {
err := func() (err0 error) {
for viewName, viewData := range j.req.Views {
// The logic here corresponds to the logic in fragment.cleanViewName().
// Unfortunately, the logic in that method is not completely exclusive
@ -368,10 +372,14 @@ func importWorker(importWork chan importJob) {
}
}
tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard})
defer finisher(&err0)
var doClear bool
switch doAction {
case RequestActionOverwrite:
if err := j.field.importRoaringOverwrite(j.ctx, j.tx, viewData, j.shard, viewName, j.req.Block); err != nil {
err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block)
if err != nil {
return errors.Wrap(err, "importing roaring as overwrite")
}
case RequestActionClear:
@ -380,7 +388,8 @@ func importWorker(importWork chan importJob) {
case RequestActionSet:
fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
if fileMagic == roaring.MagicNumber { // if pilosa roaring format
if err := j.field.importRoaring(j.ctx, j.tx, viewData, j.shard, viewName, doClear); err != nil {
err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear)
if err != nil {
return errors.Wrap(err, "importing pilosa roaring")
}
} else {
@ -388,7 +397,9 @@ func importWorker(importWork chan importJob) {
// field.importRoaring changes the standard roaring run format to pilosa roaring
data := make([]byte, len(viewData))
copy(data, viewData)
if err := j.field.importRoaring(j.ctx, j.tx, data, j.shard, viewName, doClear); err != nil {
err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
if err != nil {
return errors.Wrap(err, "importing standard roaring")
}
}
@ -421,12 +432,12 @@ func importWorker(importWork chan importJob) {
// (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all
// of the rows in this shard of this field concatenated together in one long
// bitmap.
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) {
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err0 error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring")
span.LogKV("index", indexName, "field", fieldName)
defer span.Finish()
if err = api.validate(apiField); err != nil {
if err := api.validate(apiField); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -439,9 +450,8 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return newPreconditionFailedError(err)
}
// Obtain transaction.
tx := index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
qcx := api.Txf().NewQcx()
defer qcx.Abort()
nodes := api.cluster.shardNodes(indexName, shard)
errCh := make(chan error, len(nodes))
@ -450,7 +460,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
if node.ID == api.server.nodeID {
api.importWork <- importJob{
ctx: ctx,
tx: tx,
qcx: qcx,
req: req,
shard: shard,
field: field,
@ -482,7 +492,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
// Exit once all nodes are processed.
if maxNode == len(nodes) {
return tx.Commit()
return qcx.Finish()
}
}
}
@ -590,7 +600,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Obtain transaction
tx := index.Txf.NewTx(Txo{Write: !writable, Index: index})
tx := index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Shard: shard})
defer tx.Rollback()
// Wrap writer with a CSV writer.
@ -1057,7 +1067,18 @@ func OptImportOptionsPresorted(b bool) ImportOption {
var ErrAborted = fmt.Errorf("error: update was aborted")
func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts ...ImportOption) error {
func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRecord, opts ...ImportOption) error {
// this is because some of the tests pass nil qcx for convenience.
isLocalQcx := false
if qcx == nil {
isLocalQcx = true
qcx = api.Txf().NewQcx()
defer func() {
qcx.Abort()
}()
}
simPowerLoss := false
lossAfter := -1
var opt ImportOptions
@ -1080,10 +1101,8 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts
}
// the whole point is to run this part of the import atomically.
// So make a Tx.
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
defer tx.Rollback()
// Begin that Tx now!
qcx.StartAtomicWriteTx(Txo{Write: writable, Index: idx, Shard: req.Shard})
tot := 0
// BSIs (Values)
@ -1093,7 +1112,7 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts
return ErrAborted
}
opts0 := append(opts, OptImportOptionsClear(ivr.Clear))
err := api.ImportValueWithTx(ctx, tx, ivr, opts0...)
err := api.ImportValueWithTx(ctx, qcx, ivr, opts0...)
if err != nil {
return errors.Wrap(err, "ImportAtomicRecord ImportValueWithTx")
}
@ -1106,12 +1125,17 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts
return ErrAborted
}
opts0 := append(opts, OptImportOptionsClear(ir.Clear))
err := api.ImportWithTx(ctx, tx, ir, opts0...)
err := api.ImportWithTx(ctx, qcx, ir, opts0...)
if err != nil {
return errors.Wrap(err, "ImportAtomicRecord ImportWithTx")
}
}
return tx.Commit()
// got to the end succesfully, so commit if we made the qcx
if isLocalQcx {
return qcx.Finish()
}
return nil
}
func addClearToImportOptions(opts []ImportOption) []ImportOption {
@ -1129,15 +1153,33 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption {
return append(opts, OptImportOptionsClear(true))
}
func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
// Import avoids re-writing a bajillion tests to be transaction-aware by allowing a nil pQcx.
// It is convenient for some tests, particularly those in loops, to pass a nil qcx and
// treat the Import as having been commited when we return without error. We make it so.
func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) {
if req.Clear {
opts = addClearToImportOptions(opts)
}
return api.ImportWithTx(ctx, nil, req, opts...)
isLocalQcx := false
if qcx == nil {
isLocalQcx = true
qcx = api.Txf().NewQcx()
defer func() {
qcx.Abort()
}()
}
err = api.ImportWithTx(ctx, qcx, req, opts...)
if err != nil {
return err
}
if isLocalQcx {
return qcx.Finish()
}
return nil
}
// Import bulk imports data into a particular index,field,shard.
func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opts ...ImportOption) error {
func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
defer span.Finish()
@ -1145,12 +1187,12 @@ func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opt
return errors.Wrap(err, "validating api method")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
if err := req.ValidateWithTimestamp(idx.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
@ -1179,7 +1221,7 @@ func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opt
}
// Translate column keys.
if index.Keys() {
if idx.Keys() {
span.LogKV("columnKeys", true)
if len(req.ColumnIDs) != 0 {
return errors.New("column ids cannot be used because index uses string keys")
@ -1191,7 +1233,7 @@ func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opt
// For translated data, map the columnIDs to shards. If
// this node does not own the shard, forward to the node that does.
if index.Keys() || field.Keys() {
if idx.Keys() || field.Keys() {
m := make(map[uint64][]Bit)
for i, colID := range req.ColumnIDs {
@ -1240,16 +1282,11 @@ func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opt
timestamps[i] = &t
}
isLocalTx := false
if tx == nil {
isLocalTx = true
tx = index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
}
// Import columnIDs into existence field.
// Note: req.Shard may not be the only shard imported into here,
// so don't expect it to be invariant.
if !options.Clear {
if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil {
if err := importExistenceColumns(qcx, idx, req.ColumnIDs); err != nil {
api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
return err
}
@ -1259,27 +1296,25 @@ func (api *API) ImportWithTx(ctx context.Context, tx Tx, req *ImportRequest, opt
}
// Import into fragment.
err = field.Import(tx, req.RowIDs, req.ColumnIDs, timestamps, opts...)
err = field.Import(qcx, req.RowIDs, req.ColumnIDs, timestamps, opts...)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
return errors.Wrap(err, "importing")
}
if isLocalTx {
err = tx.Commit()
}
return errors.Wrap(err, "committing")
}
func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error {
// ImportValue avoids re-writing a bajillion tests by allowing a nil pQcx.
// Then we will commit before returning.
func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
if req.Clear {
opts = addClearToImportOptions(opts)
}
return api.ImportValueWithTx(ctx, nil, req, opts...)
return api.ImportValueWithTx(ctx, qcx, req, opts...)
}
// ImportValue bulk imports values into a particular field.
func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRequest, opts ...ImportOption) error {
func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) (err0 error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue")
defer span.Finish()
@ -1287,12 +1322,12 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
return errors.Wrap(err, "validating api method")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("getting index '%v' and field '%v'; shard=%v", req.Index, req.Field, req.Shard))
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
if err := req.ValidateWithTimestamp(idx.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
@ -1302,7 +1337,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
return errors.Wrap(err, "setting up import options")
}
index, field, err = api.indexField(req.Index, req.Field, req.Shard)
idx, field, err = api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
@ -1314,7 +1349,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
// check to see if keys need translation.
if !options.IgnoreKeyCheck {
// Translate column keys.
if index.Keys() {
if idx.Keys() {
span.LogKV("columnKeys", true)
if len(req.ColumnIDs) != 0 {
return errors.New("column ids cannot be used because index uses string keys")
@ -1348,16 +1383,17 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
if !options.Presorted {
sort.Sort(req)
}
isLocalQcx := false
if qcx == nil {
isLocalQcx = true
qcx = api.Txf().NewQcx()
defer func() {
qcx.Abort()
}()
}
isLocalTx := false
// if we're importing into a specific shard
if req.Shard != math.MaxUint64 {
// Obtain transaction.
if tx == nil {
isLocalTx = true
tx = index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
}
// Check that column IDs match the stated shard.
if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard {
@ -1370,7 +1406,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
}
// Import columnIDs into existence field.
if !options.Clear {
if err := importExistenceColumns(tx, index, req.ColumnIDs); err != nil {
if err := importExistenceColumns(qcx, idx, req.ColumnIDs); err != nil {
api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
return errors.Wrap(err, "importing existence columns")
}
@ -1378,21 +1414,19 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
// Import into fragment.
if len(req.Values) > 0 {
err = field.importValue(tx, req.ColumnIDs, req.Values, options)
err = field.importValue(qcx, req.ColumnIDs, req.Values, options)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
} else if len(req.FloatValues) > 0 {
err = field.importFloatValue(tx, req.ColumnIDs, req.FloatValues, options)
err = field.importFloatValue(qcx, req.ColumnIDs, req.FloatValues, options)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
}
if err == nil && isLocalTx {
err = tx.Commit()
}
return errors.Wrap(err, "importing value")
}
} // end if req.Shard != math.MaxUint64
options.IgnoreKeyCheck = true
start := 0
@ -1436,8 +1470,14 @@ func (api *API) ImportValueWithTx(ctx context.Context, tx Tx, req *ImportValueRe
// in the client implementation.
return api.server.defaultClient.ImportValue2(ctx, subreq, options)
})
return eg.Wait()
err = eg.Wait()
if err != nil {
return err
}
if isLocalQcx {
return qcx.Finish()
}
return nil
}
func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error {
@ -1470,14 +1510,14 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq
return nil
}
func importExistenceColumns(tx Tx, index *Index, columnIDs []uint64) error {
func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64) error {
ef := index.existenceField()
if ef == nil {
return nil
}
existenceRowIDs := make([]uint64, len(columnIDs))
return ef.Import(tx, existenceRowIDs, columnIDs, nil)
return ef.Import(qcx, existenceRowIDs, columnIDs, nil)
}
// MaxShards returns the maximum shard number for each index in a map.
@ -1645,6 +1685,7 @@ func (api *API) Info() serverInfo {
CPUMHz: mhz,
CPUType: si.CPUModel(),
Memory: mem,
TxSrc: api.holder.txf.TxType(),
}
}
@ -1894,6 +1935,7 @@ type serverInfo struct {
CPUPhysicalCores int `json:"cpuPhysicalCores"`
CPULogicalCores int `json:"cpuLogicalCores"`
CPUMHz int `json:"cpuMHz"`
TxSrc string `json:"txSrc"`
}
type apiMethod int

48
api_test.go
View file

@ -214,7 +214,8 @@ func TestAPI_Import(t *testing.T) {
// Generate some keyed records.
rowIDs := []uint64{}
timestamps := []int64{}
for i := 1; i <= 10; i++ {
N := 10
for i := 1; i <= N; i++ {
rowIDs = append(rowIDs, rowID)
timestamps = append(timestamps, timestamp)
}
@ -222,6 +223,8 @@ func TestAPI_Import(t *testing.T) {
// Keys are sharded so ordering is not guaranteed.
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
colKeys = colKeys[:N]
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
@ -229,14 +232,18 @@ func TestAPI_Import(t *testing.T) {
IndexCreatedAt: index.CreatedAt(),
Field: fieldName,
FieldCreatedAt: field.CreatedAt(),
Shard: 0,
Shard: 0, // import is all on shard 0, why are we making lots of other shards? b/c this is not a restriction.
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
if err := m0.API.Import(ctx, req); err != nil {
qcx := m0.API.Txf().NewQcx()
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
@ -244,7 +251,7 @@ func TestAPI_Import(t *testing.T) {
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
}
// Query node1.
@ -327,9 +334,12 @@ func TestAPI_ImportValue(t *testing.T) {
ColumnKeys: colKeys,
Values: values,
}
if err := m0.API.ImportValue(ctx, req); err != nil {
qcx := m0.API.Txf().NewQcx()
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
pql := fmt.Sprintf("Row(%s>0)", field)
@ -383,9 +393,12 @@ func TestAPI_ImportValue(t *testing.T) {
ColumnIDs: colIDs,
FloatValues: values,
}
if err := m1.API.ImportValue(ctx, req); err != nil {
qcx := m1.API.Txf().NewQcx()
if err := m1.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
query := fmt.Sprintf("Row(%s>6)", field)
@ -393,7 +406,7 @@ func TestAPI_ImportValue(t *testing.T) {
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
t.Fatalf("unexpected column keys: %+v", ids)
t.Fatalf("unexpected column keys: observerd %+v; expected '%+v'", ids, colIDs[6:])
}
})
@ -453,9 +466,11 @@ func TestAPI_ImportValue(t *testing.T) {
ColumnIDs: colIDs,
StringValues: values,
}
if err := m0.API.ImportValue(ctx, req); err != nil {
qcx := m0.API.Txf().NewQcx()
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
@ -463,7 +478,7 @@ func TestAPI_ImportValue(t *testing.T) {
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected columns: %+v", ids)
t.Fatalf("unexpected columns: observerd %+v; expected '%+v'", ids, []uint64{1})
}
})
}
@ -536,12 +551,14 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
RowIDs: []uint64{iraRowID},
}
if err := m0api.Import(ctx, ir0); err != nil {
qcx := m0api.Txf().NewQcx()
if err := m0api.Import(ctx, qcx, ir0); err != nil {
t.Fatal(err)
}
if err := m0api.ImportValue(ctx, ivr0); err != nil {
if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
bitIsSet := func() bool {
query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
@ -579,20 +596,25 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
}
// clear the bit
qcx = m0api.Txf().NewQcx()
ir0.Clear = true
if err := m0api.Import(ctx, ir0); err != nil {
if err := m0api.Import(ctx, qcx, ir0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
if bitIsSet() {
panic("IRA bit should have been cleared")
}
// clear the BSI
qcx = m0api.Txf().NewQcx()
ivr0.Clear = true
if err := m0api.ImportValue(ctx, ivr0); err != nil {
if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
if bal != 0 {
panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))

6
audit_test.go
View file

@ -89,19 +89,19 @@ func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) err
}
func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
path := o.(*pilosa.Holder).Path
path := o.(*pilosa.Holder).Path()
if path == "" {
fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
ent.Stack)
} else {
os.RemoveAll(o.(*pilosa.Holder).Path)
os.RemoveAll(o.(*pilosa.Holder).Path())
}
return err
}
func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path)
return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
}
return nil
}

1985
badger.go
View file

File diff suppressed because it is too large Load diff

1741
badger_test.go
View file

File diff suppressed because it is too large Load diff

2
blake3.go
View file

@ -73,7 +73,7 @@ func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash
// blake3sum16 might be slower because we allocate a new hasher every time, but
// it is more conenient for writing debug code. It returns
// a 16 byte hash as a hexidecimal string.
func blake3sum16(input []byte) string {
func Blake3sum16(input []byte) string {
hasher := blake3.New()
_, _ = hasher.Write(input)

2
blake3_test.go
View file

@ -37,7 +37,7 @@ func TestBlake3Hasher(t *testing.T) {
panic(fmt.Sprintf("expected hash:'%v' but observed hash '%v'", expected, observed))
}
obs2 := blake3sum16(input)
obs2 := Blake3sum16(input)
if obs2 != expected {
panic(fmt.Sprintf("expected hash:'%v' but observed hash from blake2sum16: '%v'", expected, obs2))
}

167
bluegreentx.go
View file

@ -38,26 +38,91 @@ type blueGreenTx struct {
a Tx
b Tx // b's output is returned
o Txo
as string
bs string
types []txtype
// roaring will not create as many Tx (they are
// psuedo Tx anyway), espcially when deleting
// files. Return the non-roaring Sn if
// possible, by referencing useSnA.
useSnA bool
idx *Index
checker blueGreenChecker
mu sync.Mutex
rollbackOrCommitDone bool
txf *TxFactory
}
func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
// blueGreenRegistry is used to force checking of (read) transactions
// before writes happen, if roaring is on one of the A/B branches.
// Because roaring won't have an MVCC view of the world. Writes to
// roaring will show up, while writes to the DB won't show up on
// readTx that have already started.
type blueGreenRegistry struct {
mu sync.Mutex
m map[int64]*blueGreenTx
}
func newBlueGreenReg() *blueGreenRegistry {
return &blueGreenRegistry{
m: make(map[int64]*blueGreenTx),
}
}
// add remembers the tx so we can check it
// should we see a write after creation
// but before rollback/commit.
func (b *blueGreenRegistry) add(c *blueGreenTx) {
b.mu.Lock()
defer b.mu.Unlock()
if c.useSnA {
b.m[c.a.Sn()] = c
} else {
b.m[c.b.Sn()] = c
}
}
func (b *blueGreenRegistry) finishedTx(tx *blueGreenTx) {
b.mu.Lock()
defer b.mu.Unlock()
sn := tx.Sn()
delete(b.m, sn)
//vv("blueGreenRegistry deleted _sn_ %v", sn)
}
func (b *blueGreenRegistry) Close() {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.m) > 0 {
panic(fmt.Sprintf("still have unchecked blueGreenTx: '%#v'", b.m))
//AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m)
}
}
func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx {
//func (reg *blueGreenRegistry) newBlueGreenTx(a, b Tx, idx *Index, o Txo, openTxSn []int64) *blueGreenTx {
as := a.Type()
bs := b.Type()
c := &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs}
c := &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs, txf: txf, types: txf.types}
if c.types[1] == roaringTxn {
c.useSnA = true
}
c.checker.c = c
c.o = o
if o.Write {
txf.blueGreenReg.add(c)
}
return c
}
var _ = newBlueGreenTx // keep linter happy
var _ Tx = (*blueGreenTx)(nil)
func (c *blueGreenTx) Type() string {
@ -74,6 +139,9 @@ func (c *blueGreenTx) Dump() {
c.a.Dump()
fmt.Printf("B(%v) Dump:\n", c.bs)
c.b.Dump()
fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx)
c.idx.Txf.dbPerShard.DumpAll()
}
func (c *blueGreenTx) Readonly() bool {
@ -105,6 +173,7 @@ func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, chan
// compareTxState is called for the first Commit or Rollback a blueGreenTx sees.
func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
//vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID())
aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0)
if aErr == nil || aIter != nil {
@ -137,12 +206,12 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
aKey, aValue := aIter.Value()
if !bIter.Next() {
AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, stack())
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack()))
}
bKey, bValue := bIter.Value()
if bKey != aKey {
//vv("6960 really ought to be present index='%v',field='%v';view='%v';shard='%v'; isIn=%v", index, field, view, shard, c.isIn(index, field, view, shard, 456130566))
AlwaysPrintf("problem in caller %v", Caller(2))
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack()))
@ -152,20 +221,43 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
//vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
}
//vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey)
}
// end checking everything in A, but does B have more?
if bIter.Next() {
AlwaysPrintf("bIter has more than it should. problem in caller %v", Caller(2))
AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn())
c.Dump()
bKey, _ := bIter.Value()
//vv("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack())
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack()))
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), stack()))
}
//vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID())
}
func (c *blueGreenTx) checkDatabase() {
if !c.o.Write {
// We only need to check the we are A/B consistent after every write.
// Then reads can only see that consitent state, and don't need
// to be checked themselves. Sketch of proof by induction:
// Starting with zero data, if we have agreement in both A/B
// database state after each write, then
// because there is only ever a single
// writer (for LMDB/RBF), we should always have the same
// data state between A and B as long as every prior
// A/B check of the serialized writes suceeded.
//
// This avoids a key problem we discovered when A/B checking reads.
// The MVCC of the transactional engines means that reads that
// start before a write commit will look very different
// when comparing to roaring's non-transactional state.
return
}
c.checker.mu.Lock()
defer c.checker.mu.Unlock()
if c.checker.checkDone {
return // idemopotent. checkDatabase can be called twice. Only the first does the checks.
}
c.checker.checkDone = true
// seen() returns nil on 2nd or any further call,
// so only the first Commit() or Rollback() does this.
@ -188,7 +280,9 @@ func (c *blueGreenTx) Rollback() {
}
c.rollbackOrCommitDone = true
c.checkDatabase()
if c.o.Write {
c.checkDatabase()
}
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
@ -200,21 +294,22 @@ func (c *blueGreenTx) Rollback() {
//vv("blueGreenTx.Rollback() about to call (%v) b.Rollback()", c.bs)
c.b.Rollback()
//vv("blueGreenTx.Rollback() done. bgtx p=%p", c)
c.txf.blueGreenReg.finishedTx(c)
}
func (c *blueGreenTx) Commit() error {
c.mu.Lock()
defer c.mu.Unlock()
// for rbf 6930 debug stuff:
//in := c.isIn("i", "x", "standard", 0, 456130566)
//fmt.Printf("blueGreenTx.Commit() called. bgtx p=%p; in rbf=%v\n", c, in[0])
if c.rollbackOrCommitDone {
return nil
}
//vv("blueGreenTx.Commit() called. bgtx p=%p", c)
c.rollbackOrCommitDone = true
c.checkDatabase()
if c.o.Write {
c.checkDatabase()
}
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
@ -225,14 +320,8 @@ func (c *blueGreenTx) Commit() error {
_ = errA
errB := c.b.Commit()
/*
tx2, err := c.idx.Txf.rbfDB.NewRBFTx(false, "", nil)
panicOn(err)
inRbf, err := tx2.Contains("i", "x", "standard", 0, 456130566)
panicOn(err)
fmt.Printf("AFTER commits happened, blueGreenTx.Commit() called. bgtx p=%p; in rbf=%v\n", c, inRbf)
*/
compareErrors(errA, errB)
c.txf.blueGreenReg.finishedTx(c)
return errB
}
@ -376,13 +465,6 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool,
c.checker.see(index, field, view, shard)
//vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard)
defer func() {
// rbf 6960 debug code:
/*
in := c.isIn("i", "x", "standard", 0, 456130566)
if in[0] || in[1] {
vv("first time 6960 present isIn=%v; bgtx p=%p stack=\n%v", in, c, stack())
}
*/
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
panic(r)
@ -662,9 +744,6 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
}
compareErrors(errA, errB)
//vv("end of blue-green OffsetRange, dump:")
//c.Dump()
return b, errB
}
@ -703,6 +782,25 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
}
}
func (c *blueGreenTx) Group() *TxGroup {
return c.b.Group()
}
func (c *blueGreenTx) Options() Txo {
return c.b.Options()
}
// Sn retreives the serial number of the Tx.
func (c *blueGreenTx) Sn() int64 {
asn := c.a.Sn()
bsn := c.b.Sn()
if c.useSnA {
return asn
}
return bsn
}
func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// doesn't change state, so we don't really need see() call here. And we don't have a single shard for it.
//c.checker.see(index, field, view, shard) // don't have shard.
@ -803,15 +901,18 @@ type blueGreenChecker struct {
// lock mu when using visited.
// otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey
mu sync.Mutex
checkDone bool
}
// see would mark a thing as seen.
func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
// keep this next Printf. Useful to see the sequence of Tx operations.
//fmt.Printf("blueGreenTx.%v on index='%v'\n", Caller(1), index)
//fmt.Printf("blueGreenTx.%v on index='%v' shard=%v\n", Caller(1), index, shard)
// is ckey 6960 present in i/x/standard/0 ?
////vv("6960 present index='%v',field='%v';view='%v';shard='%v'; isIn=%v", index, field, view, shard, b.c.isIn(index, field, view, shard, 456130566))
if !b.c.o.Write {
return
}
b.mu.Lock()
defer b.mu.Unlock()

17
catcher.go
View file

@ -26,10 +26,10 @@ import (
// of the executor_test swallows up
// the location of a panic.
type catcherTx struct {
b *BadgerTx
b Tx
}
func newCatcherTx(b *BadgerTx) *catcherTx {
func newCatcherTx(b Tx) *catcherTx {
return &catcherTx{b: b}
}
@ -299,3 +299,16 @@ func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (
}()
return c.b.SliceOfShards(index, field, view, optionalViewPath)
}
func (c *catcherTx) Group() *TxGroup {
return c.b.Group()
}
func (c *catcherTx) Options() Txo {
return c.b.Options()
}
// Sn retreives the serial number of the Tx.
func (c *catcherTx) Sn() int64 {
return c.b.Sn()
}

2
cluster.go
View file

@ -236,7 +236,7 @@ type cluster struct { // nolint: maligned
abortAntiEntropyCh chan struct{}
muAntiEntropy sync.Mutex
translationSyncer translationSyncer
translationSyncer TranslationSyncer
mu sync.RWMutex
jobs map[int64]*resizeJob

37
cluster_internal_test.go
View file

@ -96,8 +96,7 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
if err != nil {
panic(err)
}
h := NewHolder(DefaultPartitionN)
h.Path = path
h := NewHolder(path, nil)
index, err := h.CreateIndex(name, IndexOptions{})
testhook.Cleanup(tb, func() {
h.Close()
@ -164,30 +163,50 @@ func TestFragSources(t *testing.T) {
idx := newIndexWithTempPath(t, "i")
defer idx.Close()
// Obtain transaction.
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
defer tx.Rollback()
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
// Obtain transaction.
var shard uint64
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
_, err = field.SetBit(tx, 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(tx, 1, ShardWidth+1, nil)
panicOn(tx.Commit())
shard = 1
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(tx, 1, ShardWidth*2+1, nil)
panicOn(tx.Commit())
shard = 2
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(tx, 1, ShardWidth*3+1, nil)
panicOn(tx.Commit())
shard = 3
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
if err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
tests := []struct {
from *cluster

2
cmd/badloader/vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

3
cmd/convert/main.go
View file

@ -28,8 +28,7 @@ func main() {
log.Fatal("USAGE convert srcPath destPath")
}
holder := pilosa.NewHolder(256)
holder.Path = os.Args[1]
holder := pilosa.NewHolder(os.Args[1], nil)
err := holder.Open()
if err != nil {

2
cmd/demo-lmdb/vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

13
cmd/pilosa-chk/chk.go
View file

@ -37,11 +37,13 @@ func main() {
var showOpsLog bool
var showBits bool
var showFrags bool
var dirChecksum bool
home := os.Getenv("HOME")
flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
flag.BoolVar(&showFrags, "v", false, "show the checksum hash for each fragment in each index. Warning: long output")
flag.BoolVar(&showOpsLog, "ops", false, "show the ops log for each fragment. Warning: very long output. Implies -v")
flag.BoolVar(&showBits, "bits", false, "show the hot bits for each fragment. Warning: very, very long output. Implies -v")
flag.BoolVar(&dirChecksum, "dirsum", false, "compute a directory hash")
flag.Parse()
if showBits {
@ -51,10 +53,15 @@ func main() {
showFrags = true
}
fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
if dirChecksum {
fmt.Printf("path '%v' has dirhash %v\n", dir, pilosa.HashOfDir(dir))
return
}
fmt.Printf(" the blake-3 hash includes the value of each mapping and the field or partitionID.\n")
holder := pilosa.NewHolder(256)
holder.Path = dir
holder := pilosa.NewHolder(dir, nil)
holder.OpenTranslateStore = boltdb.OpenTranslateStore
err := holder.Open()
@ -79,7 +86,7 @@ func main() {
final.Sort()
hasher := blake3.New()
fmt.Printf("\nsummary of %v:\n", dir)
fmt.Printf("\nsummary of col/row translations%v:\n", dir)
for _, sum := range final.Sums {
//fmt.Printf("index: %v partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
_, _ = hasher.Write([]byte(sum.Checksum))

47
cmd/lmdb-keydump/keydump.go → cmd/pilosa-keydump/keydump.go
View file

@ -36,28 +36,28 @@ import (
"fmt"
"os"
"runtime"
"sort"
"github.com/glycerine/lmdb-go/lmdb"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
// keydump simply prints all the keys in the database path specified
// pilosa-keydump is a diagnostic tool that simply prints all the
// database keys in the database directory path specified
// as the first argument on the command line.
func main() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
if len(os.Args) < 2 {
fmt.Fprintf(os.Stderr, "must supply path to database as only arg\n")
fmt.Fprintf(os.Stderr, "must supply path to database directory as only arg\n")
os.Exit(1)
}
path := os.Args[1]
if !FileExists(path) {
fmt.Fprintf(os.Stderr, "path '%v' does not exist.\n", path)
if !DirExists(path) {
fmt.Fprintf(os.Stderr, "directory path '%v' does not exist.\n", path)
os.Exit(1)
}
@ -72,7 +72,7 @@ func main() {
panicOn(err)
//var myflags uint = NoReadahead | NoSubdir
var myflags uint = lmdb.NoSubdir
var myflags uint = 0 //lmdb.NoSubdir
err = env.Open(path, myflags, 0664)
panicOn(err)
@ -87,10 +87,9 @@ func main() {
dbnames := []string{}
shardSize := make(map[string]int)
var dbiRoot lmdb.DBI
var dbi lmdb.DBI
env.UseSphynxReader()
err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) {
//txn.RawRead = true
@ -157,14 +156,19 @@ database '%v':
}
}
vs := ""
if len(v) < 100 {
vs = fmt.Sprintf("%x", v) + " "
}
fmt.Printf("%04v %v len value; key: '%v' len %v -> %v\n", i, len(v), string(k), len(k), vs)
ckey := txkey.KeyExtractContainerKey(k)
pre := txkey.PrefixFromKey(k)
shardSize[string(pre)] += len(v)
n := len(v)
hash := pilosa.Blake3sum16(v[0:(n - 1)])
ct := pilosa.ToContainer(v[n-1], v[0:(n-1)])
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
srbm := pilosa.BitmapAsString(rbm)
fmt.Printf("%04v %v -> %v (%v hot)\n", i, txkey.ToString(k), hash, ct.N())
fmt.Printf(" .......%v\n", srbm)
}
return
})
@ -172,15 +176,4 @@ database '%v':
} // for dbnames
fmt.Printf("=================== done.\n")
var lines []*pilosa.LineSorter
for k, v := range shardSize {
lines = append(lines, &pilosa.LineSorter{Line: k, Tot: float64(v)})
}
sort.Sort(pilosa.SortByTot(lines))
fd, err := os.Create("shardsize")
panicOn(err)
defer fd.Close()
for _, ln := range lines {
fmt.Fprintf(fd, "%v %v\n", int(ln.Tot), ln.Line)
}
}

2
cmd/lmdb-keydump/vprint.go → cmd/pilosa-keydump/vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

2
cmd/slurp/vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

3
ctl/inspect.go
View file

@ -291,8 +291,7 @@ func findPartitionPath(path string, partitionN int) (int, error) {
}
func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
holder := pilosa.NewHolder(pilosa.DefaultPartitionN)
holder.Path = path
holder := pilosa.NewHolder(path, nil)
holder.Opts.Inspect = true
holder.Opts.ReadOnly = true
err := holder.Open()

432
dbshard.go
View file

@ -16,176 +16,422 @@ package pilosa
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
)
var _ = sort.Sort
// types to support a database file per shard
type DBnode struct {
Holder map[string]*DBholder
type DBHolder struct {
Index map[string]*DBIndex
}
type DBholder struct {
Index map[string]*DBindex
func NewDBHolder() *DBHolder {
return &DBHolder{
Index: make(map[string]*DBIndex),
}
}
type DBindex struct {
Field map[string]*DBfield
}
type DBfield struct {
View map[string]*DBview
}
type DBview struct {
Shard map[uint64]*DBshard
type DBIndex struct {
Shard map[uint64]*DBShard
}
type DBWrapper interface {
DeleteDBPath(path string) error
NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error)
DeleteDBPath(dbs *DBShard) error
Close() error
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
DeleteField(index, field, fieldPath string) error
OpenListString() string
OpenSnList() (sns []int64)
}
type DBRegistry interface {
OpenDBWrapper(path string) (DBWrapper, error)
OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error)
}
type DBshard struct {
Path string
ID DBID
Open bool
type DBShard struct {
Path string
Index string
Shard uint64
Open bool
// With RWMutex, the
// writer who calls Lock() automatically gets priority over
// any reader who arrives later, even if the lock is held
// by a reader to start with.
RWMut sync.RWMutex
mut sync.RWMutex
W DBWrapper
ParentDBview *DBview
types []txtype
W []DBWrapper
ParentDBIndex *DBIndex
idx *Index
per *DBPerShard
useOpenList int
}
type DBID struct {
Node string
Holder string
Index string
Field string
View string
Shard uint64
func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) {
for _, w := range dbs.W {
err = w.DeleteFragment(index, field, view, shard, frag)
if err != nil {
return err
}
}
return
}
func (id *DBID) Path() (s string) {
s = id.Node + sep + id.Holder + sep + id.Index +
sep + id.Field + sep + id.View + sep + fmt.Sprintf("%04v", id.Shard)
func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
for _, w := range dbs.W {
err = w.DeleteField(index, field, fieldPath)
if err != nil {
return err
}
}
return
}
func (dbs *DBShard) Close() (err error) {
for _, w := range dbs.W {
err = w.Close()
if err != nil {
return err
}
}
return
}
func (dbs *DBShard) String() string {
return dbs.Path
}
func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
dbs.mut.Lock()
defer dbs.mut.Unlock()
var txns []Tx
for _, w := range dbs.W {
tx, err = w.NewTx(write, initialIndexName, o)
if err != nil {
return nil, err
}
txns = append(txns, tx)
}
if len(txns) == 1 {
return
}
// blue green
return dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil
}
func (dbs *DBShard) DeleteDBPath() (err error) {
for _, w := range dbs.W {
err = w.DeleteDBPath(dbs)
if err != nil {
return err
}
}
return
}
type DBPerShard struct {
Mu sync.Mutex
Dir string
Node map[string]*DBnode
Dir string // holder dir
dbh *DBHolder
// just flat, not buried within the Node heirarchy.
// Easily see how many we have.
Flatmap map[*DBshard]struct{}
Flatmap map[*DBShard]struct{}
types []txtype
txf *TxFactory
// which of our types is not-roaring, since
// roaring doesn't keep a list of open Tx sn.
// or default to the 2nd.
useOpenList int
}
func NewDBPerShard(dir string) (d *DBPerShard) {
d = &DBPerShard{
Dir: dir,
Node: make(map[string]*DBnode),
Flatmap: make(map[*DBshard]struct{}),
func (per *DBPerShard) ListOpenString() (r string) {
for v := range per.Flatmap {
r += v.Path + " -> " + v.W[per.useOpenList].OpenListString() + "\n"
}
return
}
func (per *DBPerShard) GetDBshard(registry DBRegistry, id DBID) (dbs *DBshard, err error) {
func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string) (d *DBPerShard) {
useOpenList := 0
if len(types) == 2 {
// blue-green, avoid the empty roaring Tx open list.
// Prefer B's open list if neither is roaring.
if types[0] == roaringTxn || types[1] != roaringTxn {
useOpenList = 1
}
}
d = &DBPerShard{
types: types,
Dir: holderDir,
dbh: NewDBHolder(),
Flatmap: make(map[*DBShard]struct{}),
txf: txf,
useOpenList: useOpenList,
}
return
}
func (per *DBPerShard) DeleteIndex(index string) (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
dbn, ok := per.Node[id.Node]
dbi, ok := per.dbh.Index[index]
if !ok {
dbn = &DBnode{
Holder: make(map[string]*DBholder),
}
per.Node[id.Node] = dbn
// since we lazily make indexes upon use by a Tx now, we won't
// have an index for server/ TestQuerySQLUnary/test-20 to delete.
// Don't freak out. Just return nil.
return nil
}
dbh, ok := dbn.Holder[id.Holder]
if !ok {
dbh = &DBholder{
Index: make(map[string]*DBindex),
}
dbn.Holder[id.Holder] = dbh
for _, dbs := range dbi.Shard {
err := dbs.Close()
panicOn(err)
panicOn(os.RemoveAll(dbs.Path))
}
dbi, ok := dbh.Index[id.Index]
if !ok {
dbi = &DBindex{
Field: make(map[string]*DBfield),
return
}
func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
per.Mu.Lock()
defer func() {
if fieldPath != "" {
panicOn(os.RemoveAll(fieldPath))
}
dbh.Index[id.Index] = dbi
per.Mu.Unlock()
}()
dbi, ok := per.dbh.Index[index]
if !ok {
// TestIndex_Existence_Delete in index_internal_test.go
// will call us without having ever created a Tx or DB,
// so we can't complain here.
return nil
}
dbf, ok := dbi.Field[id.Field]
if !ok {
dbf = &DBfield{
View: make(map[string]*DBview),
for _, dbs := range dbi.Shard {
for _, w := range dbs.W {
err := w.DeleteField(index, field, fieldPath)
panicOn(err)
}
dbi.Field[id.Field] = dbf
}
dbv, ok := dbf.View[id.View]
if !ok {
dbv = &DBview{
Shard: make(map[uint64]*DBshard),
return
}
func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
dbs, err := per.GetDBShard(index, shard, nil)
panicOn(err)
return dbs.DeleteFragment(index, field, view, shard, frag)
}
func (dbs *DBShard) DumpAll() {
for i, ty := range dbs.types {
_ = i
tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx})
panicOn(err)
defer tx.Rollback()
tx.Dump()
switch ty {
case roaringTxn:
case rbfTxn:
case lmdbTxn:
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
dbf.View[id.View] = dbv
}
dbs, ok = dbv.Shard[id.Shard]
if !ok {
dbs = &DBshard{
ParentDBview: dbv,
ID: id,
Path: per.Dir + sep + id.Path(),
}
func (per *DBPerShard) DumpAll() {
per.Mu.Lock()
defer per.Mu.Unlock()
found1 := false
for _, dbi := range per.dbh.Index {
for _, dbs := range dbi.Shard {
if dbs.Open {
found1 = true
dbs.DumpAll()
}
}
dbv.Shard[id.Shard] = dbs
}
if !found1 {
AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.Dir)
}
}
func (per *DBPerShard) Path(index string, shard uint64) string {
return per.Dir + sep + index + sep + fmt.Sprintf("%04v", shard)
}
func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
dbi, ok := per.dbh.Index[index]
if !ok {
dbi = &DBIndex{
Shard: make(map[uint64]*DBShard),
}
per.dbh.Index[index] = dbi
}
dbs, ok = dbi.Shard[shard]
if !ok {
dbs = &DBShard{
types: per.types,
ParentDBIndex: dbi,
Index: index,
Shard: shard,
Path: per.Path(index, shard),
idx: idx,
per: per,
useOpenList: per.useOpenList,
}
dbi.Shard[shard] = dbs
}
if !dbs.Open {
dbs.W, err = registry.OpenDBWrapper(dbs.Path)
if dbs.W != nil {
per.Flatmap[dbs] = struct{}{}
var registry DBRegistry
for _, ty := range dbs.types {
switch ty {
case roaringTxn:
registry = globalRoaringReg
case rbfTxn:
registry = globalRbfDBReg
case lmdbTxn:
registry = globalLMDBReg
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
w, err := registry.OpenDBWrapper(dbs.Path, DetectMemAccessPastTx)
panicOn(err)
dbs.Open = true
if w != nil && len(dbs.W) == 0 {
per.Flatmap[dbs] = struct{}{}
}
dbs.W = append(dbs.W, w)
}
}
return
}
func (per *DBPerShard) Del(dbs *DBshard) (err error) {
func (per *DBPerShard) Del(dbs *DBShard) (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
err = dbs.W.Close()
err = dbs.Close()
if err != nil {
return
}
panicOn(dbs.W.DeleteDBPath(dbs.Path))
panicOn(dbs.DeleteDBPath())
delete(per.Flatmap, dbs)
// delete from the heirarchy
delete(dbs.ParentDBview.Shard, dbs.ID.Shard)
return
delete(dbs.ParentDBIndex.Shard, dbs.Shard)
return nil
}
func (per *DBPerShard) Close() (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
for _, dbn := range per.Node {
for _, dbh := range dbn.Holder {
for _, dbi := range dbh.Index {
for _, dbf := range dbi.Field {
for _, dbv := range dbf.View {
for _, dbs := range dbv.Shard {
err = dbs.W.Close()
panicOn(err)
}
}
}
}
for _, dbi := range per.dbh.Index {
for _, dbs := range dbi.Shard {
err = dbs.Close()
panicOn(err)
}
}
return
}
// requiredSuffix should be "-badgerdb" for badger, etc.
func DBPerShardGetShardsForIndex(idx *Index, roaringViewPath string) (sliceOfShards []uint64, err error) {
// follow the blueGreen convention of returning the answer for 'B' or
// the last wrapper type.
types := idx.holder.txf.Types()
ty := types[len(types)-1]
if ty == roaringTxn {
rx := &RoaringTx{
Index: idx,
}
return rx.SliceOfShards("", "", "", roaringViewPath)
}
requiredSuffix := ty.FileSuffix()
path := idx.Path()
ignoreEmpty := false
includeRoot := true
dbf, err := listDirUnderDir(path, includeRoot, requiredSuffix, ignoreEmpty)
panicOn(err)
for _, nm := range dbf {
base := filepath.Base(nm)
splt := strings.Split(base, requiredSuffix)
if len(splt) != 2 {
panic(fmt.Sprintf("should have 2 parts: nm='%v', base(nm)='%v'; requiredSuffix='%v'", nm, base, requiredSuffix))
}
prefix := splt[0]
// Parse filename into integer.
shard, err := strconv.ParseUint(prefix, 10, 64)
if err != nil {
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
if !dirExists(root) {
return nil, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
n := len(root) + 1
if includeRoot {
n = 0
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if len(path) < n {
// ignore
} else {
if info == nil {
// re-opening an RBF database hit this, racing with a directory rename.
// Don't freak out.
return nil
}
if !info.IsDir() {
// ignore files
} else {
if ignoreEmpty && info.Size() == 0 {
return nil
}
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
files = append(files, path[n:])
}
}
}
return nil
})
return
}

179
dbshard_internal_test.go Normal file
View file

@ -0,0 +1,179 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
)
// Shard per db evaluation
func TestShardPerDB_SetBit(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
_ = idx
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(tx, 120, 1); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(tx, 120, 6); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(tx, 121, 0); err != nil {
t.Fatal(err)
}
// should have two containers set in the fragment.
// Verify counts on rows.
if n := f.mustRow(tx, 120).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// commit the change, and verify it is still there
panicOn(tx.Commit())
// Close and reopen the fragment & verify the data.
err := f.Reopen() // roaring data not being flushed? red on roaring
if err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if n := f.mustRow(tx, 120).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
// test that we find all shards
func Test_DBPerShard_GetShardsForIndex(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestDBPerShardGetShardsForIndex")
panicOn(err)
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
for _, src := range []string{"lmdb", "roaring", "rbf"} {
makeSampleRoaringDir(tmpdir, src)
os.Setenv("PILOSA_TXSRC", src)
// must make Holder AFTER setting src.
holder := NewHolder(tmpdir, nil)
idx, err := NewIndex(holder, tmpdir, "rick")
panicOn(err)
estd := "rick/_exists/views/standard"
std := "rick/f/views/standard"
sos, err := DBPerShardGetShardsForIndex(idx, tmpdir+sep+std)
panicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if !inSlice(sos, shard) {
panic(fmt.Sprintf("missing shard=%v from sos='%#v'", shard, sos))
}
}
if src == "roaring" {
// check estd too
sos, err = DBPerShardGetShardsForIndex(idx, tmpdir+sep+estd)
panicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if !inSlice(sos, shard) {
panic(fmt.Sprintf("missing shard=%v from sos='%#v'", shard, sos))
}
}
}
}
}
func inSlice(sos []uint64, shard uint64) bool {
for i := range sos {
if shard == sos[i] {
return true
}
}
return false
}
// data for Test_DBPerShard_GetShardsForIndex
//
var sampleRoaringDirList = map[string]string{"roaring": `
rick/f/views/standard/fragments/215.cache
rick/f/views/standard/fragments/221.cache
rick/f/views/standard/fragments/223.cache
rick/f/views/standard/fragments/93.cache
rick/f/views/standard/fragments/217.cache
rick/f/views/standard/fragments/219.cache
rick/f/views/standard/fragments/217
rick/f/views/standard/fragments/219
rick/f/views/standard/fragments/215
rick/f/views/standard/fragments/221
rick/f/views/standard/fragments/223
rick/f/views/standard/fragments/93
rick/_exists/views/standard/fragments/221
rick/_exists/views/standard/fragments/215
rick/_exists/views/standard/fragments/217
rick/_exists/views/standard/fragments/93
rick/_exists/views/standard/fragments/219
rick/_exists/views/standard/fragments/223
`,
"lmdb": `
rick/0219-lmdb/data.mdb
rick/0219-lmdb/lock.mdb
rick/0093-lmdb/data.mdb
rick/0093-lmdb/lock.mdb
rick/0223-lmdb/data.mdb
rick/0223-lmdb/lock.mdb
rick/0215-lmdb/data.mdb
rick/0215-lmdb/lock.mdb
rick/0217-lmdb/data.mdb
rick/0217-lmdb/lock.mdb
rick/0221-lmdb/data.mdb
rick/0221-lmdb/lock.mdb
`,
"rbf": `
rick/0223-rbfdb/wal/0000000000000001.wal
rick/0223-rbfdb/data
rick/0093-rbfdb/wal/0000000000000001.wal
rick/0093-rbfdb/data
rick/0217-rbfdb/wal/0000000000000001.wal
rick/0217-rbfdb/data
rick/0215-rbfdb/wal/0000000000000001.wal
rick/0215-rbfdb/data
rick/0221-rbfdb/wal/0000000000000001.wal
rick/0221-rbfdb/data
rick/0219-rbfdb/wal/0000000000000001.wal
rick/0219-rbfdb/data
`,
}
func makeSampleRoaringDir(root, txsrc string) {
fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
for _, fn := range fns {
if fn == "" {
continue
}
path := root + sep + filepath.Dir(fn)
panicOn(os.MkdirAll(path, 0755))
fd, err := os.Create(root + sep + fn)
panicOn(err)
fd.Close()
}
}

196
dbshard_test.go Normal file
View file

@ -0,0 +1,196 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
)
var sep = string(os.PathSeparator)
func skipForNonLMDB(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src != "lmdb" {
t.Skip("skip if not lmdb")
}
}
var _ = skipForNonLMDB // happy linter
func skipForNonBadger(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src != "badger" {
t.Skip("skip if not badger")
}
}
// Can't write it all to one shard like we do (did).
func Test_DBPerShard_multiple_shards_used(t *testing.T) {
skipForNonBadger(t)
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := c.GetHolder(0)
index := "i"
hldr.SetBit(index, "general", 10, 0)
hldr.SetBit(index, "general", 10, ShardWidth+1)
hldr.SetBit(index, "general", 10, ShardWidth+2)
hldr.SetBit(index, "general", 11, 2)
hldr.SetBit(index, "general", 11, ShardWidth+2)
//tx_suffix := "-lmdb"
tx_suffix := "-badgerdb"
root := hldr.Path() + sep + index
shards := []string{"0000", "0001", "0002"}
pathShard := []string{}
// check that 3 different shard databases/files were made
for i := 0; i < 2; i++ {
path := root + sep + shards[i] + tx_suffix
pathShard = append(pathShard, path)
if !DirExists(pathShard[i]) {
panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i]))
}
sz, err := DiskUse(pathShard[i], "")
panicOn(err)
if sz < 100 {
panic(fmt.Sprintf("shard %v was too small", i))
}
}
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: `Union(Row(general=10), Row(general=11))`}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
func DiskUse(root string, requiredSuffix string) (tot int, err error) {
if !DirExists(root) {
return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip directories.
} else {
sz := info.Size()
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
tot += int(sz)
}
}
return nil
})
return
}
func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
m0 := c.GetNode(0)
t.Run("RowIDColumnKey", func(t *testing.T) {
ctx := context.Background()
indexName := "rick"
fieldName := "f"
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
if index.CreatedAt() == 0 {
t.Fatal("index createdAt is empty")
}
field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if field.CreatedAt() == 0 {
t.Fatal("field createdAt is empty")
}
rowID := uint64(1)
timestamp := int64(0)
// Generate some keyed records.
rowIDs := []uint64{}
timestamps := []int64{}
for i := 1; i <= 10; i++ {
rowIDs = append(rowIDs, rowID)
timestamps = append(timestamps, timestamp)
}
// Keys are sharded so ordering is not guaranteed.
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
Field: fieldName,
FieldCreatedAt: field.CreatedAt(),
Shard: 0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
qcx := m0.API.Txf().NewQcx()
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
//select {}
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
}
})
}

516
executor.go
View file

File diff suppressed because it is too large Load diff

14
executor_internal_test.go
View file

@ -27,7 +27,8 @@ import (
)
func TestExecutor_TranslateGroupByCall(t *testing.T) {
holder := NewHolder(DefaultPartitionN)
path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
holder := NewHolder(path, nil)
defer holder.Close()
cluster := NewTestCluster(t, 1)
@ -36,7 +37,6 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
Holder: holder,
Cluster: cluster,
}
e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
err := e.Holder.Open()
if err != nil {
t.Fatalf("opening holder: %v", err)
@ -137,14 +137,14 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
}
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
holder := NewHolder(DefaultPartitionN)
path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
holder := NewHolder(path, nil)
defer holder.Close()
e := &executor{
Holder: holder,
Cluster: NewTestCluster(t, 1),
}
e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
if err := e.Holder.Open(); err != nil {
t.Fatalf("opening holder: %v", err)
}
@ -154,10 +154,8 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
t.Fatalf("creating index: %v", err)
}
tx, err := holder.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
shard := uint64(0)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
fb, errb := idx.CreateField("b", OptFieldTypeBool())

49
executor_test.go
View file

@ -664,7 +664,7 @@ func TestExecutor_Execute_Set(t *testing.T) {
}
})
t.Run("ErrInvalidRowValueType", func(t *testing.T) { // // failing under badger_roaring
t.Run("ErrInvalidRowValueType", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
@ -934,10 +934,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Obtain transaction.
tx, err := hldr.BeginTx(!writable, index.Index)
if err != nil {
t.Fatal(err)
}
idx := index.Index
shard := uint64(0)
tx := idx.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idx, Shard: shard})
defer tx.Rollback()
f := hldr.Field("i", "f")
@ -2488,7 +2487,6 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns())
}
})
@ -2769,7 +2767,6 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) {
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns())
}
})
@ -3456,6 +3453,8 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatal(err)
}
//index.Dump("after Set 3x")
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) {
@ -3473,6 +3472,11 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatal(err)
}
hldr2 := c.GetHolder(0)
index2 := hldr2.Index("i")
_ = index2
//index2.Dump("after reopen")
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
@ -3777,9 +3781,19 @@ func TestExecutor_Execute_All(t *testing.T) {
req.RowIDs[bitCount-1] = 10
req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2)
if err := c.GetNode(0).API.Import(context.Background(), req); err != nil {
m0 := c.GetNode(0)
qcx := m0.API.Txf().NewQcx()
if err := m0.API.Import(context.Background(), qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
i0, err := m0.API.Index(context.Background(), "i")
panicOn(err)
if i0 == nil {
panic("nil index i0?")
}
tests := []struct {
qry string
@ -3802,7 +3816,7 @@ func TestExecutor_Execute_All(t *testing.T) {
{qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)},
}
for i, test := range tests {
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil {
if res, err := m0.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil {
t.Fatal(err)
} else if cnt := res.Results[0].(*pilosa.Row).Count(); cnt != test.expCnt {
t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt)
@ -3851,9 +3865,11 @@ func TestExecutor_Execute_All(t *testing.T) {
req.ColumnKeys[i] = fmt.Sprintf("c%d", i)
}
if err := c.GetNode(0).API.Import(context.Background(), req); err != nil {
qcx := c.GetNode(0).API.Txf().NewQcx()
if err := c.GetNode(0).API.Import(context.Background(), qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
tests := []struct {
qry string
@ -4378,10 +4394,13 @@ func benchmarkExistence(nn bool, b *testing.B) {
}
b.ResetTimer()
nodeAPI := c.GetNode(0).API
for i := 0; i < b.N; i++ {
if err := c.GetNode(0).API.Import(context.Background(), req); err != nil {
qcx := nodeAPI.Txf().NewQcx()
if err := nodeAPI.Import(context.Background(), qcx, req); err != nil {
b.Fatal(err)
}
panicOn(qcx.Finish())
}
}
@ -4880,7 +4899,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "vv", pilosa.OptFieldTypeInt(0, 1000))
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "nv", pilosa.OptFieldTypeInt(-1000, 1000))
if err := c.GetNode(0).API.Import(context.Background(), &pilosa.ImportRequest{
if err := c.GetNode(0).API.Import(context.Background(), nil, &pilosa.ImportRequest{
Index: "istring",
Field: "generals",
Shard: 0,
@ -4892,7 +4911,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4
if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{
if err := c.GetNode(0).API.ImportValue(context.Background(), nil, &pilosa.ImportValueRequest{
Index: "istring",
Field: "v",
Shard: 0,
@ -4902,7 +4921,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
t.Fatalf("importing: %v", err)
}
if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{
if err := c.GetNode(0).API.ImportValue(context.Background(), nil, &pilosa.ImportValueRequest{
Index: "istring",
Field: "vv",
Shard: 0,
@ -4912,7 +4931,7 @@ func TestExecutor_GroupByStrings(t *testing.T) {
t.Fatalf("importing: %v", err)
}
if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{
if err := c.GetNode(0).API.ImportValue(context.Background(), nil, &pilosa.ImportValueRequest{
Index: "istring",
Field: "nv",
Shard: 0,

56
field.go
View file

@ -418,6 +418,7 @@ func (f *Field) AvailableShards() *roaring.Bitmap {
b := f.remoteAvailableShards.Clone()
for _, view := range f.viewMap {
//b.Union(view.availableShards())
b.UnionInPlace(view.availableShards())
}
return b
@ -615,6 +616,7 @@ func (f *Field) Open() error {
func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte) {
path := filepath.Join(fieldPath, ".available.shards")
// Create a temporary file to save to.
tempPath := path + tempExt
err := ioutil.WriteFile(tempPath, availableShardBytes, 0666)
@ -627,7 +629,6 @@ func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte)
if err := os.Rename(tempPath, path); err != nil {
log.Printf("rename snapshot: %s", err)
}
}
func nonBlockingWriteAvailableShards(fieldPath string, availableShardBytes []byte, done chan bool) {
if len(availableShardBytes) == 0 {
@ -759,7 +760,7 @@ fileLoop:
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
if f.idx.Txf.TxType() == RoaringTxn {
if f.holder.txf.TxType() == RoaringTxn {
// Automatically upgrade BSI v1 fragments if they exist & reopen view.
if bsig := f.bsiGroup(f.name); bsig != nil {
if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil {
@ -1111,6 +1112,7 @@ func (f *Field) RowTime(tx Tx, rowID uint64, time time.Time, quantum string) (*R
if view == nil {
return nil, errors.Errorf("view with quantum %v not found.", quantum)
}
return view.row(tx, rowID)
}
@ -1523,7 +1525,15 @@ func (f *Field) MaxForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
return ValCount{}, nil
}
max, cnt, err := fragment.max(tx, filter, bsig.BitDepth)
var localTx Tx
if NilInside(tx) {
localTx = f.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard})
defer localTx.Rollback()
} else {
localTx = tx
}
max, cnt, err := fragment.max(localTx, filter, bsig.BitDepth)
if err != nil {
return ValCount{}, errors.Wrap(err, "calling fragment.max")
}
@ -1559,7 +1569,15 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
return ValCount{}, nil
}
min, cnt, err := fragment.min(tx, filter, bsig.BitDepth)
var localTx Tx
if NilInside(tx) {
localTx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard})
defer localTx.Rollback()
} else {
localTx = tx
}
min, cnt, err := fragment.min(localTx, filter, bsig.BitDepth)
if err != nil {
return ValCount{}, errors.Wrap(err, "calling fragment.min")
}
@ -1577,7 +1595,7 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
}
// Range performs a conditional operation on Field.
func (f *Field) Range(tx Tx, name string, op pql.Token, predicate int64) (*Row, error) {
func (f *Field) Range(qcx *Qcx, name string, op pql.Token, predicate int64) (*Row, error) {
// Retrieve and validate bsiGroup.
bsig := f.bsiGroup(name)
if bsig == nil {
@ -1597,11 +1615,11 @@ func (f *Field) Range(tx Tx, name string, op pql.Token, predicate int64) (*Row,
return NewRow(), nil
}
return view.rangeOp(tx, op, bsig.BitDepth, baseValue)
return view.rangeOp(qcx, op, bsig.BitDepth, baseValue)
}
// Import bulk imports data.
func (f *Field) Import(tx Tx, rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) error {
func (f *Field) Import(qcx *Qcx, rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) (err0 error) {
// Set up import options.
options := &ImportOptions{}
@ -1673,15 +1691,19 @@ func (f *Field) Import(tx Tx, rowIDs, columnIDs []uint64, timestamps []*time.Tim
return errors.Wrap(err, "creating fragment")
}
if err := frag.bulkImport(tx, data.RowIDs, data.ColumnIDs, options); err != nil {
return err
}
}
tx, finisher := qcx.GetTx(Txo{Write: true, Index: frag.idx, Fragment: frag, Shard: frag.shard})
err1 := frag.bulkImport(tx, data.RowIDs, data.ColumnIDs, options)
if err1 != nil {
finisher(&err1)
return err1
}
finisher(nil)
}
return nil
}
func (f *Field) importFloatValue(tx Tx, columnIDs []uint64, values []float64, options *ImportOptions) error {
func (f *Field) importFloatValue(qcx *Qcx, columnIDs []uint64, values []float64, options *ImportOptions) error {
// convert values to int64 values based on scale
ivalues := make([]int64, len(values))
bsig := f.bsiGroup(f.name)
@ -1693,11 +1715,11 @@ func (f *Field) importFloatValue(tx Tx, columnIDs []uint64, values []float64, op
ivalues[i] = int64(fval * mult)
}
// then call importValue
return f.importValue(tx, columnIDs, ivalues, options)
return f.importValue(qcx, columnIDs, ivalues, options)
}
// importValue bulk imports range-encoded value data.
func (f *Field) importValue(tx Tx, columnIDs []uint64, values []int64, options *ImportOptions) error {
func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, options *ImportOptions) (err0 error) {
viewName := viewBSIGroupPrefix + f.name
// Get the bsiGroup so we know bitDepth.
bsig := f.bsiGroup(f.name)
@ -1780,6 +1802,12 @@ func (f *Field) importValue(tx Tx, columnIDs []uint64, values []int64, options *
baseValues[i] = value - bsig.Base
}
// now we know which shard we discovered.
tx, finisher := qcx.GetTx(Txo{Write: writable, Index: f.idx, Shard: frag.shard})
// by deferring, even though we are in loop, we get en-mass commit at once if they all succeed,
// or en-mass rollback if any fail.
defer finisher(&err0)
if err := frag.importValue(tx, data.ColumnIDs, baseValues, requiredDepth, options.Clear); err != nil {
return err
}

85
field_internal_test.go
View file

@ -207,8 +207,7 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
if err != nil {
t.Fatal(err)
}
h := NewHolder(DefaultPartitionN)
h.Path = path
h := NewHolder(path, nil)
idx, err := h.CreateIndex("i", IndexOptions{})
if err != nil {
panic(err)
@ -319,7 +318,7 @@ func TestField_RowTime(t *testing.T) {
defer f.Close()
// Obtain transaction.
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
defer tx.Rollback()
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
@ -335,7 +334,7 @@ func TestField_RowTime(t *testing.T) {
panicOn(tx.Commit())
// obtain 2nd transaction to read it back.
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
defer tx.Rollback()
if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
@ -575,6 +574,9 @@ func TestBSIGroup_importValue(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
qcx := f.idx.Txf.NewQcx()
defer qcx.Abort()
options := &ImportOptions{}
for i, tt := range []struct {
columnIDs []uint64
@ -601,25 +603,16 @@ func TestBSIGroup_importValue(t *testing.T) {
[]uint64{100},
},
} {
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
// can't do this, we are in a loop, not a function:
// defer tx.Rollback()
if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
// no, same reason as above: defer tx.Rollback()
if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
panicOn(qcx.Finish())
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
tx.Rollback()
panicOn(qcx.Finish())
} // loop
}
@ -627,6 +620,9 @@ func TestIntField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
qcx := f.idx.Txf.NewQcx()
defer qcx.Abort()
options := &ImportOptions{}
for i, test := range []struct {
name string
@ -677,18 +673,16 @@ func TestIntField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil {
if err := f.importValue(qcx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(qcx.Finish())
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
shard := uint64(0)
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
// Rollback below manually, because we are in a loop.
maxvc, err := f.MaxForShard(tx, 0, nil)
maxvc, err := f.MaxForShard(tx, shard, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)
}
@ -696,19 +690,21 @@ func TestIntField_MinMaxForShard(t *testing.T) {
t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
}
minvc, err := f.MinForShard(tx, 0, nil)
minvc, err := f.MinForShard(tx, shard, nil)
if err != nil {
t.Fatalf("getting min for shard: %v", err)
}
if minvc != test.expMin {
t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
}
tx.Rollback()
})
}
}
// Ensure we get errors when they are expected.
func TestDecimalField_MinMaxBoundaries(t *testing.T) {
th := newTestHolder(t)
for i, test := range []struct {
scale int64
min pql.Decimal
@ -771,7 +767,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
},
} {
t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
_, err := NewField(NewHolder(DefaultPartitionN), "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
_, err := NewField(th, "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
if err != nil && test.expErr {
if !strings.Contains(err.Error(), "is not supported") {
t.Fatal(err)
@ -789,6 +785,9 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeDecimal(3))
defer f.Close()
qcx := f.idx.Txf.NewQcx()
defer qcx.Abort()
options := &ImportOptions{}
for i, test := range []struct {
name string
@ -839,18 +838,15 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil {
if err := f.importFloatValue(qcx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
shard := uint64(0)
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
defer tx.Rollback()
maxvc, err := f.MaxForShard(tx, 0, nil)
maxvc, err := f.MaxForShard(tx, shard, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)
}
@ -858,7 +854,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
}
minvc, err := f.MinForShard(tx, 0, nil)
minvc, err := f.MinForShard(tx, shard, nil)
if err != nil {
t.Fatalf("getting min for shard: %v", err)
}
@ -873,6 +869,9 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
qcx := f.idx.Txf.NewQcx()
defer qcx.Abort()
options := &ImportOptions{}
for i, tt := range []struct {
columnIDs []uint64
@ -899,25 +898,17 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
[]uint64{100},
},
} {
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
// can't do this, we are in a loop, not a function:
// defer tx.Rollback()
if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(qcx.Finish())
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
// no, same reason as above: defer tx.Rollback()
if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
tx.Rollback()
panicOn(qcx.Finish())
} // loop
// the test: can we re-open a BSI fragment under badger/rbf.

57
field_test.go
View file

@ -25,8 +25,6 @@ import (
"github.com/pilosa/pilosa/v2/testhook"
)
var panicOn = pilosa.PanicOn
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -38,9 +36,8 @@ func TestField_SetValue(t *testing.T) {
t.Fatal(err)
}
idxPilosa := f.Field.GetIndex()
tx := idxPilosa.NewTx(pilosa.Txo{Write: writable, Index: idxPilosa, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -74,9 +71,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -110,9 +107,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value.
if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
@ -128,9 +125,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value.
if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
@ -146,9 +143,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value.
if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
@ -162,7 +159,7 @@ func TestField_NameRestriction(t *testing.T) {
if err != nil {
panic(err)
}
field, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", ".meta", pilosa.OptFieldTypeDefault())
field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault())
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
@ -195,13 +192,13 @@ func TestField_NameValidation(t *testing.T) {
panic(err)
}
for _, name := range validFieldNames {
_, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
_, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}
@ -217,9 +214,9 @@ func TestField_AvailableShards(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetBit will lazily instantiate Tx.
var tx pilosa.Tx
// Set values on shards 0 & 2, and verify.
if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
@ -229,7 +226,6 @@ func TestField_AvailableShards(t *testing.T) {
} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
t.Fatal(diff)
}
panicOn(tx.Commit())
// Set remote shards and verify.
if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
@ -260,9 +256,8 @@ func TestField_ClearValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
var tx pilosa.Tx
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -270,9 +265,6 @@ func TestField_ClearValue(t *testing.T) {
} else if !changed {
t.Fatal("expected change")
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
// Read value.
if value, exists, err := f.Value(tx, 100); err != nil {
@ -282,17 +274,12 @@ func TestField_ClearValue(t *testing.T) {
} else if !exists {
t.Fatal("expected value to exist")
}
tx.Rollback()
tx = idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
if changed, err := f.ClearValue(tx, 100); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal(err)
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Read value.
if _, exists, err := f.Value(tx, 100); err != nil {

56
fragment.go
View file

@ -250,7 +250,7 @@ func (f *fragment) Open() error {
f.checksums = make(map[int][]byte)
// Read last bit to determine max row.
tx := f.idx.Txf.NewTx(Txo{Write: false, Index: f.idx, Fragment: f})
tx := f.idx.Txf.NewTx(Txo{Write: false, Index: f.idx, Fragment: f, Shard: f.shard}) // first index 'i' shard 0
defer tx.Rollback()
return f.calculateMaxRowID(tx)
}(); err != nil {
@ -485,7 +485,7 @@ func (f *fragment) openCache() error {
return nil
}
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Read in all rows by ID.
@ -624,7 +624,7 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
doSetFunc := func() error {
// handle mutux field type
if f.mutexVector != nil {
if err := f.handleMutex(tx, rowID, columnID); err != nil {
@ -633,7 +633,13 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro
}
changed, err = f.unprotectedSetBit(tx, rowID, columnID)
return err
})
}
// avoid crashing when f.gen is nil
if f.gen != nil {
err = f.gen.Transaction(wp, doSetFunc)
} else {
err = doSetFunc()
}
return changed, err
}
@ -992,9 +998,22 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
}
// TODO get rid of this and use positionsForValue to generate a single write op, and set that with importPositions.
func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
tx := txOrig
if NilInside(tx) {
tx = f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}()
}
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
@ -1976,7 +1995,7 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) {
if idx == nil {
panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'; f.holder.indexes='%#v'\n", f.index, f.holder.indexes))
}
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// no Commit below, b/c is read-only.
@ -2061,7 +2080,7 @@ func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64, err error) {
defer f.mu.Unlock()
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx})
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Shard: f.shard})
defer tx.Rollback()
// readonly, so no Commit()
@ -2268,7 +2287,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() error { // segfault
doFunc := func() error {
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
@ -2302,12 +2321,6 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
start := rowID * ShardWidth
end := (rowID + 1) * ShardWidth
// avoid a 2nd r/w iterator if possible,
// aiming to having fewer write/read Tx conflicts.
badgerTx, isBadger := tx.(*BadgerTx)
if isBadger {
badgerTx.frag = f
}
n, err := tx.CountRange(f.index, f.field, f.view, f.shard, start, end)
if err != nil {
return errors.Wrap(err, "CountRange")
@ -2323,7 +2336,14 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
f.cache.Recalculate()
}
return nil
})
}
var err error
if f.gen != nil {
err = f.gen.Transaction(wp, doFunc)
} else {
err = doFunc()
}
if err != nil {
// we got an error. it's possible that the error indicates that something went wrong.
mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to)
@ -2758,7 +2778,7 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) {
// used in shipping the slices across the network for a resize.
func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx})
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard})
defer tx.Rollback()
file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path)
if err != nil {
@ -2832,7 +2852,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
switch hdr.Name {
case "data":
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if err := f.fillFragmentFromArchive(tx, tr); err != nil {
return 0, errors.Wrap(err, "reading storage")
@ -3612,7 +3632,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
}
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: f.shard})
defer tx.Rollback()
// Merge blocks together.

217
fragment_internal_test.go
View file

@ -77,11 +77,11 @@ func TestFragment_SetBit(t *testing.T) {
panicOn(tx.Commit())
// Close and reopen the fragment & verify the data.
err := f.Reopen() // roaring data not being flushed? red on roaring
err := f.Reopen()
if err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if n := f.mustRow(tx, 120).Count(); n != 2 {
@ -114,7 +114,7 @@ func TestFragment_ClearBit(t *testing.T) {
// In that spirit, we will check that the Tx Commit is visible afterwards.
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Close and reopen the fragment & verify the data.
@ -199,7 +199,7 @@ func TestFragment_ClearRow(t *testing.T) {
t.Fatalf("unexpected count: %d", n)
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Close and reopen the fragment & verify the data.
@ -246,7 +246,7 @@ func TestFragment_SetRow(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Verify data on row.
@ -259,7 +259,7 @@ func TestFragment_SetRow(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Close and reopen the fragment & verify the data.
@ -304,7 +304,7 @@ func TestFragment_SetValue(t *testing.T) {
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Read value.
@ -356,7 +356,7 @@ func TestFragment_SetValue(t *testing.T) {
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if value, exists, err := f.value(tx, 100, 16); err != nil {
@ -401,7 +401,7 @@ func TestFragment_SetValue(t *testing.T) {
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if value, exists, err := f.value(tx, 100, 16); err != nil {
@ -478,7 +478,7 @@ func TestFragment_SetValue(t *testing.T) {
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Ensure values are set.
@ -525,7 +525,7 @@ func TestFragment_Sum(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
t.Run("NoFilter", func(t *testing.T) {
@ -549,7 +549,7 @@ func TestFragment_Sum(t *testing.T) {
})
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// verify that clearValue clears values
@ -558,7 +558,7 @@ func TestFragment_Sum(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
t.Run("ClearValue", func(t *testing.T) {
@ -599,7 +599,7 @@ func TestFragment_MinMax(t *testing.T) {
panicOn(tx.Commit())
// the new tx is shared by Min/Max below.
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
t.Run("Min", func(t *testing.T) {
@ -1197,7 +1197,7 @@ func TestFragment_Snapshot(t *testing.T) {
t.Fatal(err)
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Snapshot bitmap and verify data.
@ -1290,7 +1290,7 @@ func TestFragment_Top_Filter(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Retrieve top rows.
@ -1467,7 +1467,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
defer f.Clean(t)
// Obtain transaction.
tx := index.Txf.NewTx(Txo{Write: writable, Index: index})
tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Set bits on various rows.
@ -1546,7 +1546,7 @@ func TestFragment_Blocks(t *testing.T) {
}
prev = blocks
tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
// Set bit on different row.
if _, err := f.setBit(tx, 20, 0); err != nil {
t.Fatal(err)
@ -1561,7 +1561,7 @@ func TestFragment_Blocks(t *testing.T) {
prev = blocks
// Set bit on different column.
tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if _, err := f.setBit(tx, 20, 100); err != nil {
t.Fatal(err)
@ -1658,7 +1658,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
// Obtain transaction.
tx := index.Txf.NewTx(Txo{Write: writable, Index: index})
tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Set bits on the fragment.
@ -1669,7 +1669,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
panicOn(tx.Commit())
tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f})
tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Verify correct cache type and size.
@ -1769,9 +1769,11 @@ func BenchmarkFragment_Blocks(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
}
th := newTestHolder(b)
// Open the fragment specified by the path.
f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
// Open the fragment specified by the path. Note that newFragment
// is overriding the usual holder-to-fragment path logic...
f := newFragment(th, *FragmentPath, "i", "f", viewStandard, 0, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@ -1806,7 +1808,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Snapshot to disk before benchmarking.
@ -2147,7 +2149,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2159,7 +2161,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Clear import.
@ -2169,7 +2171,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2185,13 +2187,27 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
func TestFragment_ConcurrentImport(t *testing.T) {
t.Run("bulkImportStandard", func(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
shard := uint64(0)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "")
_ = idx
defer f.Clean(t)
// note: write Tx must be used on the same goroutine that created them.
// So we close out the "default" Tx created by mustOpenFragment, and
// have the goroutines below each make their own. One should get the
// write lock first, and thus they should get serialized.
tx.Rollback()
eg := errgroup.Group{}
eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) })
eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) })
eg.Go(func() error {
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
defer func() { panicOn(tx.Commit()) }()
return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{})
})
eg.Go(func() error {
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
defer func() { panicOn(tx.Commit()) }()
return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{})
})
err := eg.Wait()
if err != nil {
t.Fatalf("importing data to fragment: %v", err)
@ -2412,7 +2428,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2424,7 +2440,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Clear import.
@ -2434,7 +2450,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2662,7 +2678,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2674,7 +2690,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Clear import.
@ -2684,7 +2700,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Check for expected results.
@ -2705,7 +2721,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
b.ReportAllocs()
// Open the fragment specified by the path.
f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
f := newFragment(newTestHolder(b), *FragmentPath, "i", "f", viewStandard, 0, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@ -2747,7 +2763,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
i++
}
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if err := f.bulkImport(tx, rows, cols, options); err != nil {
@ -3120,31 +3136,30 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
origF.Close()
fi.Close()
h := NewHolder(DefaultPartitionN)
h.Path = fi.Name()
h := NewHolder(fi.Name(), nil)
idx, err := h.CreateIndex("i", IndexOptions{})
panicOn(err)
nf := newFragment(h, fi.Name(), "i", "f", viewStandard, 0, 0)
err = nf.Open()
f := newFragment(h, fi.Name(), "i", "f", viewStandard, 0, 0)
err = f.Open()
if err != nil {
b.Fatalf("opening fragment: %v", err)
}
// Obtain transaction.
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
copy(rows, rowsOrig)
copy(cols, colsOrig)
b.StartTimer()
err = nf.bulkImport(tx, rows, cols, opts)
err = f.bulkImport(tx, rows, cols, opts)
b.StopTimer()
if err != nil {
b.Fatalf("bulkImport: %v", err)
}
panicOn(tx.Commit())
nf.Clean(b)
f.Clean(b)
}
}
@ -3152,6 +3167,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
b.StopTimer()
initBigFrag(b)
updata := getUpdataRoaring(10000000, 11000, 0)
th := newTestHolder(b)
for i := 0; i < b.N; i++ {
origF, err := os.Open(bigFrag)
if err != nil {
@ -3171,24 +3187,23 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
// want to do this, but no path argument.
//nf, idx, tx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte)
th := newTestHolder()
idx := fragTestMustOpenIndex(filepath.Dir(fi.Name()), "i", th, IndexOptions{})
idx := fragTestMustOpenIndex("i", th, IndexOptions{})
if th.NeedsSnapshot() {
th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
}
// XXX TODO: newFragment is using the wrong path here, we should fix that someday.
nf := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0)
defer nf.Clean(b)
f := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0)
defer f.Clean(b)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
err = nf.Open()
err = f.Open()
if err != nil {
b.Fatalf("opening fragment: %v", err)
}
b.StartTimer()
err = nf.importRoaringT(tx, updata, false)
err = f.importRoaringT(tx, updata, false)
b.StopTimer()
if err != nil {
b.Fatalf("bulkImport: %v", err)
@ -3414,15 +3429,15 @@ func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64)
return mustOpenFragmentFlags(tb, index, field, view, shard, "", 1)
}
func newTestHolder() *Holder {
h := NewHolder(DefaultPartitionN)
func newTestHolder(tb testing.TB) *Holder {
path, _ := testhook.TempDirInDir(tb, *TempDir, "holder-dir")
h := NewHolder(path, nil)
//h.SnapshotQueue = newSnapshotQueue(1, 1, nil)
return h
}
// fragTestMustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func fragTestMustOpenIndex(holderDir, index string, holder *Holder, opt IndexOptions) *Index {
holder.Path = holderDir
func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Index {
holder.mu.Lock()
idx, err := holder.createIndex(index, opt)
holder.mu.Unlock()
@ -3439,19 +3454,15 @@ func fragTestMustOpenIndex(holderDir, index string, holder *Holder, opt IndexOpt
// mustOpenFragment returns a new instance of Fragment with a temporary path.
func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint64, cacheType string, flags byte) (*fragment, *Index, Tx) {
holderDir, err := testhook.TempDirInDir(tb, *TempDir, "holder-dir")
panicOn(err)
if cacheType == "" {
cacheType = DefaultCacheType
}
th := newTestHolder()
th := newTestHolder(tb)
testhook.Cleanup(tb, func() {
th.Close()
})
idx := fragTestMustOpenIndex(holderDir, index, th, IndexOptions{})
idx := fragTestMustOpenIndex(index, th, IndexOptions{})
if th.NeedsSnapshot() {
th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
}
@ -3461,7 +3472,7 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
fragPath := fragDir + fmt.Sprintf("%v", shard)
f := newFragment(th, fragPath, index, field, view, shard, flags)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
testhook.Cleanup(tb, func() {
tx.Rollback()
panicOn(idx.Txf.CloseIndex(idx))
@ -3571,7 +3582,7 @@ func TestFragment_RowsIteration(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
ids, err := f.rows(context.Background(), tx, 0)
@ -3999,7 +4010,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 3, 0)
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
iter, err := f.rowIterator(tx, false)
@ -4047,7 +4058,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 7, 0)
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
iter, err := f.rowIterator(tx, false)
@ -4095,7 +4106,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 3, 0)
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
iter, err := f.rowIterator(tx, true)
@ -4132,7 +4143,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 7, 0)
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
iter, err := f.rowIterator(tx, true)
@ -4351,8 +4362,8 @@ func sliceEq(x, y []uint64) bool {
}
func TestFragmentBSIUnsigned(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone)
_ = idx
shard := uint64(0)
f, idx, tx := mustOpenFragment(t, "i", "f", "v", shard, CacheTypeNone)
defer f.Clean(t)
// Number of bits to test.
@ -4367,6 +4378,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
t.Fatalf("no change when setting col %d to %d", uint64(i), int64(i))
}
}
panicOn(tx.Commit()) // t.Run beolow on different goro and so need their own Tx anyway.
// Generate a list of columns.
cols := make([]uint64, 1<<k)
@ -4379,6 +4391,10 @@ func TestFragmentBSIUnsigned(t *testing.T) {
minCheck, maxCheck := -3, 1<<(k+1)
t.Run("<", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
row, err := f.rangeLT(tx, k, int64(i), false)
if err != nil {
@ -4399,6 +4415,10 @@ func TestFragmentBSIUnsigned(t *testing.T) {
}
})
t.Run("<=", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
row, err := f.rangeLT(tx, k, int64(i), true)
if err != nil {
@ -4419,6 +4439,10 @@ func TestFragmentBSIUnsigned(t *testing.T) {
}
})
t.Run(">", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
row, err := f.rangeGT(tx, k, int64(i), false)
if err != nil {
@ -4439,6 +4463,9 @@ func TestFragmentBSIUnsigned(t *testing.T) {
}
})
t.Run(">=", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
row, err := f.rangeGT(tx, k, int64(i), true)
if err != nil {
@ -4459,6 +4486,10 @@ func TestFragmentBSIUnsigned(t *testing.T) {
}
})
t.Run("Range", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
for j := i; j < maxCheck; j++ {
row, err := f.rangeBetween(tx, k, int64(i), int64(j))
@ -4491,6 +4522,10 @@ func TestFragmentBSIUnsigned(t *testing.T) {
}
})
t.Run("==", func(t *testing.T) {
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
defer tx.Rollback()
for i := minCheck; i < maxCheck; i++ {
row, err := f.rangeEQ(tx, k, int64(i))
if err != nil {
@ -4528,7 +4563,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) {
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Generate a list of columns.
@ -4690,6 +4725,10 @@ func TestFragmentBSISigned(t *testing.T) {
}
}
panicOn(tx.Commit())
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Generate a list of columns.
cols := make([]uint64, (maxVal-minVal)+1)
for i := range cols {
@ -4901,7 +4940,7 @@ func TestImportClearRestart(t *testing.T) {
panicOn(tx.Commit())
err = f.Open()
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if err != nil {
t.Fatalf("reopening fragment: %v", err)
@ -4915,7 +4954,7 @@ func TestImportClearRestart(t *testing.T) {
check(t, tx, f, exp)
h := NewHolder(DefaultPartitionN)
h := newTestHolder(t)
idx2, err := h.CreateIndex("i", IndexOptions{})
_ = idx2
panicOn(err)
@ -4931,7 +4970,7 @@ func TestImportClearRestart(t *testing.T) {
panicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation.
tx2 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2})
tx2 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard})
defer tx2.Rollback()
err = f.closeStorage()
@ -4968,8 +5007,7 @@ func TestImportClearRestart(t *testing.T) {
panicOn(tx2.Commit())
h3 := NewHolder(DefaultPartitionN)
h3.Path = filepath.Dir(f2.path)
h3 := NewHolder(filepath.Dir(f2.path), nil)
idx3, err := h3.CreateIndex("i", IndexOptions{})
_ = idx3
panicOn(err)
@ -4978,7 +5016,7 @@ func TestImportClearRestart(t *testing.T) {
f3.MaxOpN = maxOpN
f3.CacheType = f.CacheType
tx3 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3})
tx3 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard})
defer tx3.Rollback()
err = f2.closeStorage()
@ -5052,7 +5090,7 @@ func TestImportValueConcurrent(t *testing.T) {
for i := 0; i < 4; i++ {
i := i
eg.Go(func() error {
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
for j := uint64(0); j < 10; j++ {
err := f.importValue(tx, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0)
@ -5100,7 +5138,7 @@ func TestImportMultipleValues(t *testing.T) {
// probably too slow, would hit disk alot:
//panicOn(tx.Commit())
//tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
//tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true})
//defer tx.Rollback()
for i := range test.checkCols {
@ -5182,25 +5220,40 @@ func TestImportValueRowCache(t *testing.T) {
}
}
// part of copy-on-write patch: test for races
// do we see races/corruption around concurrent read/write.
// especially on writes to the row cache.
func TestFragmentConcurrentReadWrite(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
_ = idx
defer f.Clean(t)
// actual transaction backends, there won't be any
// data, and in particular, the blue-green tests will
// note this and fire a false-positive.
notBlueGreenTest(t)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean(t)
tx.Rollback()
// Obtain transaction, but don't start another b/c the
// two goroutines below need the same view.
eg := &errgroup.Group{}
eg.Go(func() error {
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
for i := uint64(0); i < 1000; i++ {
_, err := f.setBit(tx, i%4, i)
if err != nil {
return errors.Wrap(err, "setting bit")
}
}
panicOn(tx.Commit())
return nil
})
// need read-only Tx so as not to block on the writer finishing above.
tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
acc := uint64(0)
for i := uint64(0); i < 100; i++ {
r := f.mustRow(tx, i%4)

4
gid.go
View file

@ -49,6 +49,10 @@ var littleBuf = sync.Pool{
var _ = curGID // happy linter
func curGID() uint64 {
if true {
return 0 // avoid doing too much work during production profiling.
}
bp := littleBuf.Get().(*[]byte)
defer littleBuf.Put(bp)
b := *bp

24
go.mod
View file

@ -11,19 +11,18 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/glycerine/lmdb-go v1.9.27
github.com/glycerine/lmdb-go v1.9.32
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.2.0
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.3.3
github.com/google/go-cmp v0.2.0
github.com/google/go-cmp v0.4.0
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/lib/pq v1.8.0
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.0.0
github.com/prometheus/client_model v0.1.0
github.com/prometheus/prom2json v1.3.0
@ -32,18 +31,21 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil v2.18.12+incompatible
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.3.2
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.4.0
github.com/uber-go/atomic v1.4.0 // indirect
github.com/uber/jaeger-client-go v2.16.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/zeebo/blake3 v0.0.4
go.uber.org/atomic v1.4.0 // indirect
golang.org/x/mod v0.3.0
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/text v0.3.2 // indirect
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/text v0.3.3 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
modernc.org/mathutil v1.0.0
modernc.org/strutil v1.0.0
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible

102
go.sum
View file

@ -5,8 +5,6 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@ -33,32 +31,29 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
github.com/glycerine/lmdb-go v1.9.27 h1:k20zfiumwC/E1g/MYIzZ2GkhOH0hicDfEXa0KUjWLjY=
github.com/glycerine/lmdb-go v1.9.27/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
github.com/glycerine/lmdb-go v1.9.32 h1:thLnzCykFcmn2rACYnwpR4ovYauLNKaAuk+xj7YMbS0=
github.com/glycerine/lmdb-go v1.9.32/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@ -67,10 +62,11 @@ github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -80,18 +76,24 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI=
github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
@ -110,10 +112,13 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -136,6 +141,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
@ -147,9 +153,12 @@ github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVT
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@ -159,19 +168,24 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@ -180,8 +194,10 @@ github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAri
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
@ -190,14 +206,16 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@ -205,13 +223,15 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
@ -220,15 +240,18 @@ github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@ -241,12 +264,16 @@ golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -254,24 +281,27 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -280,6 +310,10 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
@ -287,6 +321,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
@ -294,8 +329,11 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

152
holder.go
View file

@ -50,6 +50,12 @@ const (
existenceFieldName = "_exists"
)
func init() {
// For performance tuning, leave these readily available:
// CPUProfileForDur(time.Minute, "server.cpu.pprof")
// MemProfileForDur(2*time.Minute, "server.mem.pprof")
}
// Holder represents a container for indexes.
type Holder struct {
mu sync.RWMutex
@ -75,7 +81,7 @@ type Holder struct {
Stats stats.StatsClient
// Data directory path.
Path string
path string
// The interval at which the cached row ids are persisted to disk.
cacheFlushInterval time.Duration
@ -93,7 +99,7 @@ type Holder struct {
// transactionManager
transactionManager *TransactionManager
translationSyncer translationSyncer
translationSyncer TranslationSyncer
// Queue of fields (having a foreign index) which have
// opened before their foreign index has opened.
@ -108,8 +114,12 @@ type Holder struct {
Opts HolderOpts
Auditor testhook.Auditor
txf *TxFactory
}
// HolderOpts holds information about the holder which other things might want
// to look up later while using the holder.
type HolderOpts struct {
// ReadOnly indicates that this holder's contents should not produce
// disk writes under any circumstances. It must be set before Open
@ -166,38 +176,89 @@ func (lc *lockedChan) Recv() {
lc.mu.RUnlock()
}
// NewHolder returns a new instance of Holder.
func NewHolder(partitionN int) *Holder {
// HolderConfig holds configuration details that need to be set up at
// initial holder creation. NewHolder takes a *HolderConfig, which can be
// nil. Use DefaultHolderConfig to get a default-valued HolderConfig you
// can then alter.
type HolderConfig struct {
PartitionN int
OpenTranslateStore OpenTranslateStoreFunc
OpenTranslateReader OpenTranslateReaderFunc
OpenTransactionStore OpenTransactionStoreFunc
TranslationSyncer TranslationSyncer
CacheFlushInterval time.Duration
StatsClient stats.StatsClient
NewAttrStore func(string) AttrStore
Logger logger.Logger
Txsrc string
}
func DefaultHolderConfig() *HolderConfig {
return &HolderConfig{
PartitionN: DefaultPartitionN,
OpenTranslateStore: OpenInMemTranslateStore,
OpenTranslateReader: nil,
OpenTransactionStore: OpenInMemTransactionStore,
TranslationSyncer: NopTranslationSyncer,
CacheFlushInterval: defaultCacheFlushInterval,
StatsClient: stats.NopStatsClient,
NewAttrStore: newNopAttrStore,
Logger: logger.NopLogger,
Txsrc: DefaultTxsrc,
}
}
// NewHolder returns a new instance of Holder for the given path.
func NewHolder(path string, cfg *HolderConfig) *Holder {
if cfg == nil {
cfg = DefaultHolderConfig()
// still want the PILOSA_TXSRC to override, for tests use.
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc != "" {
_ = MustTxsrcToTxtype(txsrc)
// INVAR: have valid txsrc.
cfg.Txsrc = txsrc
}
}
h := &Holder{
partitionN: partitionN,
indexes: make(map[string]*Index),
closing: make(chan struct{}),
indexes: make(map[string]*Index),
closing: make(chan struct{}),
opened: lockedChan{ch: make(chan struct{})},
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
NewAttrStore: newNopAttrStore,
cacheFlushInterval: defaultCacheFlushInterval,
OpenTranslateStore: OpenInMemTranslateStore,
OpenTransactionStore: OpenInMemTransactionStore,
translationSyncer: NopTranslationSyncer,
Logger: logger.NopLogger,
partitionN: cfg.PartitionN,
Stats: cfg.StatsClient,
NewAttrStore: cfg.NewAttrStore,
cacheFlushInterval: cfg.CacheFlushInterval,
OpenTranslateStore: cfg.OpenTranslateStore,
OpenTranslateReader: cfg.OpenTranslateReader,
OpenTransactionStore: cfg.OpenTransactionStore,
translationSyncer: cfg.TranslationSyncer,
Logger: cfg.Logger,
Opts: HolderOpts{Txsrc: cfg.Txsrc},
SnapshotQueue: defaultSnapshotQueue,
Auditor: NewAuditor(),
path: path,
}
txf, err := NewTxFactory(cfg.Txsrc, path, h)
panicOn(err)
h.txf = txf
_ = testhook.Created(h.Auditor, h, nil)
return h
}
// Path() returns the path directory the holder was created with.
func (h *Holder) Path() string {
return h.path
}
type HolderInfo struct {
FragmentInfo map[string]FragmentInfo
FragmentNames []string
@ -482,8 +543,8 @@ func (h *Holder) Open() error {
h.setFileLimit()
h.Logger.Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
h.Logger.Printf("open holder path: %s", h.path)
if err := os.MkdirAll(h.path, 0777); err != nil {
return errors.Wrap(err, "creating directory")
}
@ -494,7 +555,7 @@ func (h *Holder) Open() error {
return ErrCannotOpenV1TranslateFile
}
tstore, err := h.OpenTransactionStore(h.Path)
tstore, err := h.OpenTransactionStore(h.path)
if err != nil {
return errors.Wrap(err, "opening transaction store")
}
@ -502,7 +563,7 @@ func (h *Holder) Open() error {
h.transactionManager.Log = h.Logger
// Open path to read all index directories.
f, err := os.Open(h.Path)
f, err := os.Open(h.path)
if err != nil {
return errors.Wrap(err, "opening directory")
}
@ -618,6 +679,7 @@ func (h *Holder) Close() error {
if globalUseStatTx {
fmt.Printf("%v\n", globalCallStats.report())
}
h.txf.blueGreenReg.Close()
h.Stats.Close()
@ -648,11 +710,6 @@ func (h *Holder) Close() error {
return nil
}
// Begin starts a transaction on the holder.
func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) {
return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil
}
func (h *Holder) NeedsSnapshot() bool {
h.mu.RLock()
defer h.mu.RUnlock()
@ -674,13 +731,13 @@ func (h *Holder) HasData() (bool, error) {
return true, nil
}
// Open path to read all index directories.
if _, err := os.Stat(h.Path); os.IsNotExist(err) {
if _, err := os.Stat(h.path); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, errors.Wrap(err, "statting data dir")
}
f, err := os.Open(h.Path)
f, err := os.Open(h.path)
if err != nil {
return false, errors.Wrap(err, "opening data dir")
}
@ -702,7 +759,7 @@ func (h *Holder) HasData() (bool, error) {
// hasV1TranslateKeysFile returns true if a v1 translation data file exists on disk.
func (h *Holder) hasV1TranslateKeysFile() (bool, error) {
if _, err := os.Stat(filepath.Join(h.Path, ".keys")); os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(h.path, ".keys")); os.IsNotExist(err) {
return true, nil
} else if err != nil {
return false, err
@ -842,7 +899,7 @@ func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string {
return filepath.Join(h.Path, name)
return filepath.Join(h.path, name)
}
// HolderPathFromIndexPath is
@ -1119,9 +1176,9 @@ func (h *Holder) setFileLimit() {
}
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, ".id")
idPath := path.Join(h.path, ".id")
h.Logger.Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
if err := os.MkdirAll(h.path, 0777); err != nil {
return "", errors.Wrap(err, "creating directory")
}
@ -1148,7 +1205,7 @@ func (h *Holder) logStartup() error {
}
logLine := fmt.Sprintf("%s\t%s\n", time, Version)
f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
f, err := os.OpenFile(h.path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return errors.Wrap(err, "opening startup log")
}
@ -1425,12 +1482,12 @@ func (s *holderSyncer) resetTranslationSync() error {
// a future iteration on this may be to make it more generic so
// it can act as an internal message bus where one of the messages
// being published is "translationSyncReset".
type translationSyncer interface {
type TranslationSyncer interface {
Reset() error
}
// NopTranslationSyncer represents a translationSyncer that doesn't do anything.
var NopTranslationSyncer translationSyncer = &nopTranslationSyncer{}
var NopTranslationSyncer TranslationSyncer = &nopTranslationSyncer{}
type nopTranslationSyncer struct{}
@ -1865,3 +1922,24 @@ func (h *Holder) addIndexFromField(idx *Index) {
func (h *Holder) unprotectedAddIndexFromField(idx *Index) {
h.indexes[idx.Name()] = idx
}
func (h *Holder) DumpAllShards() {
h.mu.RLock()
defer h.mu.RUnlock()
for index, idx := range h.indexes {
fmt.Printf("dump of index '%v'\n", index)
idx.Txf.dbPerShard.DumpAll()
}
}
func (h *Holder) Txf() *TxFactory {
h.mu.Lock()
defer h.mu.Unlock()
return h.txf
}
// Begin starts a transaction on the holder. The index and shard
// must be specified.
func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) {
return idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
}

9
holder_internal_test.go
View file

@ -78,8 +78,7 @@ func makeHolder(tb testing.TB) (*Holder, string, error) {
if err != nil {
return nil, "", err
}
h := NewHolder(DefaultPartitionN)
h.Path = path
h := NewHolder(path, nil)
return h, path, nil
}
@ -90,10 +89,8 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
t.Fatalf("creating index: %v", err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
shard := columnID / ShardWidth
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())

42
holder_test.go
View file

@ -108,11 +108,11 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil {
} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0000); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0755)
_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0755)
}()
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
@ -133,7 +133,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil {
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", ".meta"), 2); err != nil {
t.Fatal(err)
}
@ -155,7 +155,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil {
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", ".data"), 2); err != nil {
t.Fatal(err)
}
@ -178,10 +178,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
var shard uint64
tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
@ -192,11 +191,11 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0644)
_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0644)
}()
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
@ -214,7 +213,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
tx, err := h.BeginTx(writable, idx)
var shard uint64
tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
if err != nil {
t.Fatal(err)
}
@ -228,7 +228,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
t.Fatal(err)
}
@ -246,10 +246,8 @@ func TestHolder_Open(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
var shard uint64
tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
@ -260,7 +258,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
t.Fatal(err)
}
@ -390,10 +388,12 @@ func TestHolder_HasData(t *testing.T) {
})
t.Run("Peek at missing directory", func(t *testing.T) {
h := test.NewHolder(t)
// Ensure that hasData is false when dir doesn't exist.
h.Path = "bad-path"
// Note that we are intentionally not using test.NewHolder,
// because we want to create a Holder object with an invalid path,
// rather than creating a valid holder with a temporary path.
h := pilosa.NewHolder("bad-path", nil)
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
@ -748,7 +748,7 @@ func TestHolderSyncer_IntField(t *testing.T) {
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
// expects exists==true, a==1
t.Errorf("unexpected value(node%d/0): a:%d, exists: %v", i, a, exists) // failing TestHolderSyncer_IntField under Badger, unexpected value(node1/0): a:0, exists: true
t.Errorf("unexpected value(node%d/0): a:%d, exists: %v", i, a, exists)
}
if a, exists := hldr.Value("i", "f", 2); exists {
t.Errorf("unexpected value(node%d/1): a:%d, exists: %v", i, a, exists)

13
http/client_test.go
View file

@ -1391,3 +1391,16 @@ func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pi
AttrVals: attrVals,
}
}
// verify that serverInfo has TxSrc
func TestClient_ServerInfoHasTxSrc(t *testing.T) {
//srcs := []string{"roaring", "rbf", "lmdb"}
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
si := cmd.API.Info()
if si.TxSrc == "" {
panic("should have gotten a TxSrc back")
}
pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid
}

29
http/handler.go
View file

@ -2173,7 +2173,14 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re
return
}
if err := h.api.ImportAtomicRecord(r.Context(), req, opt); err != nil {
qcx := h.api.Txf().NewQcx()
err = h.api.ImportAtomicRecord(r.Context(), qcx, req, opt)
if err == nil {
err = qcx.Finish()
} else {
qcx.Abort()
}
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -2243,7 +2250,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.api.ImportValue(r.Context(), req, opts...); err != nil {
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
if err := h.api.ImportValue(r.Context(), qcx, req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -2252,6 +2262,11 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
return
}
err := qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("error in qcx.Finish(): '%v'", err.Error()), http.StatusInternalServerError)
return
}
} else {
// Field type: set, time, mutex
// Marshal into request object.
@ -2261,7 +2276,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.api.Import(r.Context(), req, opts...); err != nil {
qcx := h.api.Txf().NewQcx()
defer qcx.Abort()
if err := h.api.Import(r.Context(), qcx, req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -2270,6 +2288,11 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
return
}
err := qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("error in qcx.Finish() on set,time,mutex: '%v'", err.Error()), http.StatusInternalServerError)
return
}
}
// Write response.

49
index.go
View file

@ -65,12 +65,11 @@ type Index struct {
// Per-partition translation stores
translateStores map[int]TranslateStore
translationSyncer translationSyncer
translationSyncer TranslationSyncer
// Instantiates new translation stores
OpenTranslateStore OpenTranslateStoreFunc
// txf chooses the transaction and storage strategy
Txf *TxFactory
}
@ -82,36 +81,11 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
// the defaults, because we may be under a simple "go test" run where
// not all that command line machinery has been spun up.
// needed for the tests:
txsrc := os.Getenv("PILOSA_TXSRC")
// Warning: won't work for the tests to say:
// txsrc := holder.Opts.Txsrc // WILL BREAK TESTS
// For *most* of the tests and in a production pilosa server run, we expect that
// if holder.opts.Txsrc is set, it will be the exact same as PILOSA_TXSRC.
// Unfortunately there are some tests where that won't hold.
// So if the env var PILOSA_TXSRC *is* set, we always give it precedence.
// This lets `PILOSA_TXSRC=rbf go test -v -run "one_of_my_RBF_tests"` succeed.
if txsrc == "" {
// nothing in the env for PILOSA_TXSRC; therefore not running under a "make topt.rbf" for example.
if holder.Opts.Txsrc != "" {
txsrc = holder.Opts.Txsrc
} else {
txsrc = DefaultTxsrc
}
}
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
txf, err := NewTxFactory(txsrc, holder.Path, name)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
idx := &Index{
path: path,
name: name,
@ -131,9 +105,9 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
OpenTranslateStore: OpenInMemTranslateStore,
Txf: txf,
// the Txf should be shared across all holder.
Txf: holder.txf,
}
idx.Txf.idx = idx
return idx, nil
}
@ -338,9 +312,9 @@ fileLoop:
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
i.holder.Logger.Debugf("add field to index.fields: %s", fi.Name())
mu.Lock()
i.mu.Lock()
i.fields[fld.Name()] = fld
mu.Unlock()
i.mu.Unlock()
return nil
})
}
@ -460,6 +434,7 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
b := roaring.NewBitmap()
for _, f := range i.fields {
//b.Union(f.AvailableShards())
b.UnionInPlace(f.AvailableShards())
}
@ -727,11 +702,9 @@ func FormatQualifiedIndexName(index string) string {
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Mostly for debugging.
func (idx *Index) Dump(label string) {
fileline := FileLine(2)
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx})
defer tx.Rollback()
fmt.Printf("\n%v Index.Dump('%v') for index '%v':\n", fileline, label, idx.name)
tx.Dump()
//fileline := FileLine(2)
fmt.Printf("\nDump: %v\n\n", label)
idx.Txf.dbPerShard.DumpAll()
}
func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []uint64, err error) {
@ -818,7 +791,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar
}
sum.Field = fld.name
sum.Index = i.Name()
sum.Checksum = blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
if verbose {
fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
@ -840,7 +813,7 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar
sum.PartitionID = partitionID
sum.Index = i.Name()
sum.Checksum = blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
sum.Checksum = Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
if verbose {
fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID)
}

3
index_internal_test.go
View file

@ -26,8 +26,7 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
if err != nil {
panic(err)
}
h := NewHolder(1)
h.Path = path
h := NewHolder(path, nil)
index, err := h.CreateIndex("i", opt)
testhook.Cleanup(tb, func() {
h.Close()

2
index_test.go
View file

@ -242,7 +242,7 @@ func TestIndex_InvalidName(t *testing.T) {
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "ABC")
index, err := pilosa.NewIndex(pilosa.NewHolder(path, nil), path, "ABC")
if err == nil {
t.Fatalf("should have gotten an error on index name with caps")
}

405
internal/public.pb.go
View file

@ -2065,6 +2065,53 @@ func (m *AtomicRecord) GetIr() []*ImportRequest {
return nil
}
type AtomicImportResponse struct {
Error string `protobuf:"bytes,1,opt,name=Error,proto3" json:"Error,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *AtomicImportResponse) Reset() { *m = AtomicImportResponse{} }
func (m *AtomicImportResponse) String() string { return proto.CompactTextString(m) }
func (*AtomicImportResponse) ProtoMessage() {}
func (*AtomicImportResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{28}
}
func (m *AtomicImportResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *AtomicImportResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_AtomicImportResponse.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *AtomicImportResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_AtomicImportResponse.Merge(m, src)
}
func (m *AtomicImportResponse) XXX_Size() int {
return m.Size()
}
func (m *AtomicImportResponse) XXX_DiscardUnknown() {
xxx_messageInfo_AtomicImportResponse.DiscardUnknown(m)
}
var xxx_messageInfo_AtomicImportResponse proto.InternalMessageInfo
func (m *AtomicImportResponse) GetError() string {
if m != nil {
return m.Error
}
return ""
}
type TranslateKeysRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -2078,7 +2125,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} }
func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) }
func (*TranslateKeysRequest) ProtoMessage() {}
func (*TranslateKeysRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{28}
return fileDescriptor_413a91106d7bcce8, []int{29}
}
func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2139,7 +2186,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} }
func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) }
func (*TranslateKeysResponse) ProtoMessage() {}
func (*TranslateKeysResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{29}
return fileDescriptor_413a91106d7bcce8, []int{30}
}
func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2188,7 +2235,7 @@ func (m *TranslateIDsRequest) Reset() { *m = TranslateIDsRequest{} }
func (m *TranslateIDsRequest) String() string { return proto.CompactTextString(m) }
func (*TranslateIDsRequest) ProtoMessage() {}
func (*TranslateIDsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{30}
return fileDescriptor_413a91106d7bcce8, []int{31}
}
func (m *TranslateIDsRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2249,7 +2296,7 @@ func (m *TranslateIDsResponse) Reset() { *m = TranslateIDsResponse{} }
func (m *TranslateIDsResponse) String() string { return proto.CompactTextString(m) }
func (*TranslateIDsResponse) ProtoMessage() {}
func (*TranslateIDsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{31}
return fileDescriptor_413a91106d7bcce8, []int{32}
}
func (m *TranslateIDsResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2297,7 +2344,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi
func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) }
func (*ImportRoaringRequestView) ProtoMessage() {}
func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{32}
return fileDescriptor_413a91106d7bcce8, []int{33}
}
func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2356,7 +2403,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} }
func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRoaringRequest) ProtoMessage() {}
func (*ImportRoaringRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{33}
return fileDescriptor_413a91106d7bcce8, []int{34}
}
func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2443,7 +2490,7 @@ func (m *ImportColumnAttrsRequest) Reset() { *m = ImportColumnAttrsReque
func (m *ImportColumnAttrsRequest) String() string { return proto.CompactTextString(m) }
func (*ImportColumnAttrsRequest) ProtoMessage() {}
func (*ImportColumnAttrsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_413a91106d7bcce8, []int{34}
return fileDescriptor_413a91106d7bcce8, []int{35}
}
func (m *ImportColumnAttrsRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2543,6 +2590,7 @@ func init() {
proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest")
proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest")
proto.RegisterType((*AtomicRecord)(nil), "internal.AtomicRecord")
proto.RegisterType((*AtomicImportResponse)(nil), "internal.AtomicImportResponse")
proto.RegisterType((*TranslateKeysRequest)(nil), "internal.TranslateKeysRequest")
proto.RegisterType((*TranslateKeysResponse)(nil), "internal.TranslateKeysResponse")
proto.RegisterType((*TranslateIDsRequest)(nil), "internal.TranslateIDsRequest")
@ -2555,109 +2603,110 @@ func init() {
func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) }
var fileDescriptor_413a91106d7bcce8 = []byte{
// 1620 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xdb, 0x46,
0x16, 0x37, 0x45, 0xca, 0x92, 0x9e, 0x64, 0xc7, 0x99, 0x38, 0x59, 0x22, 0xeb, 0x38, 0x02, 0xe1,
0xdd, 0x68, 0xf7, 0xe0, 0xc0, 0xd9, 0x24, 0xc8, 0x65, 0x77, 0x63, 0x47, 0xce, 0x9a, 0xc8, 0xda,
0x9b, 0x1d, 0x19, 0xde, 0xdb, 0x02, 0xb4, 0x34, 0x75, 0x88, 0x52, 0xa2, 0x4a, 0x51, 0x91, 0x7d,
0x29, 0xd0, 0xcf, 0x90, 0x4b, 0x3f, 0x42, 0x3f, 0x47, 0x2f, 0xed, 0xb1, 0xc7, 0x02, 0xbd, 0x14,
0x69, 0xbf, 0x45, 0x2e, 0xc5, 0x7b, 0xc3, 0xd1, 0x0c, 0x29, 0xda, 0x31, 0x82, 0xde, 0xe6, 0xfd,
0x99, 0x37, 0xf3, 0x7e, 0xef, 0xc7, 0x37, 0x4f, 0x82, 0xd6, 0x78, 0x7a, 0x1a, 0x85, 0xfd, 0xed,
0x71, 0x12, 0xa7, 0x31, 0xab, 0x87, 0xa3, 0x54, 0x24, 0xa3, 0x20, 0xf2, 0x26, 0x60, 0xf3, 0x78,
0xc6, 0x5c, 0xa8, 0xbd, 0x88, 0xa3, 0xe9, 0x70, 0x34, 0x71, 0xad, 0xb6, 0xdd, 0x71, 0xb8, 0x12,
0x19, 0x03, 0xe7, 0x95, 0xb8, 0x98, 0xb8, 0x76, 0xdb, 0xee, 0x34, 0x38, 0xad, 0xd9, 0x16, 0x54,
0x77, 0xd3, 0x34, 0x99, 0xb8, 0x95, 0xb6, 0xdd, 0x69, 0x3e, 0x5a, 0xdd, 0x56, 0xe1, 0xb6, 0x51,
0xcd, 0xa5, 0x11, 0x63, 0xf2, 0x38, 0x48, 0xc2, 0xd1, 0x99, 0xeb, 0xb4, 0xad, 0x4e, 0x8b, 0x2b,
0xd1, 0x3b, 0x84, 0x46, 0x2f, 0x3c, 0x1b, 0x89, 0x01, 0x1e, 0x7d, 0x1f, 0xec, 0xd7, 0x31, 0x1e,
0x6b, 0x75, 0x9a, 0x8f, 0x56, 0x74, 0x28, 0x1e, 0xcf, 0x38, 0x5a, 0xd0, 0xe1, 0x48, 0x9c, 0xb9,
0x95, 0x52, 0x87, 0x23, 0x71, 0xe6, 0x3d, 0x83, 0x55, 0x1e, 0xcf, 0xfc, 0x81, 0x18, 0xa5, 0xe1,
0x67, 0xa1, 0x48, 0xe8, 0xd2, 0x3c, 0x9e, 0xa9, 0x5c, 0x68, 0x3d, 0x4f, 0xa4, 0xa2, 0x13, 0xf1,
0xee, 0xc2, 0xb2, 0xdf, 0xfd, 0x77, 0x38, 0x49, 0xd9, 0x1a, 0xd8, 0x7e, 0x57, 0x6d, 0xc0, 0xa5,
0xe7, 0xc3, 0xcd, 0xfd, 0xf3, 0x34, 0x09, 0xfa, 0xa9, 0x18, 0xf8, 0x5d, 0x09, 0x07, 0x5b, 0x85,
0x8a, 0xdf, 0xa5, 0xbb, 0x3a, 0xbc, 0xe2, 0x77, 0xd9, 0x16, 0x38, 0x27, 0x41, 0xa4, 0x80, 0x58,
0xd3, 0x97, 0x93, 0x61, 0x39, 0x59, 0xbd, 0xd3, 0x5c, 0xa8, 0xc3, 0x20, 0x4d, 0xc2, 0x73, 0x76,
0x07, 0x96, 0x5f, 0x86, 0x22, 0x1a, 0xc8, 0x43, 0x1b, 0x3c, 0x93, 0xd8, 0x13, 0x5d, 0x0a, 0x19,
0xf5, 0x8f, 0x3a, 0xea, 0xc2, 0x85, 0xe6, 0x75, 0xf2, 0xee, 0x41, 0xed, 0x95, 0xb8, 0xa0, 0x5c,
0x54, 0xa6, 0x96, 0x91, 0xe9, 0x4f, 0x16, 0xdc, 0x9a, 0xef, 0x3e, 0x0e, 0x4e, 0x23, 0x71, 0x12,
0x44, 0x53, 0xc1, 0xb6, 0x54, 0xde, 0x56, 0xd9, 0xfd, 0x0f, 0x96, 0x08, 0x0b, 0xf6, 0x60, 0x8e,
0x1d, 0xba, 0xdd, 0xd4, 0x6e, 0xd9, 0x91, 0x07, 0x4b, 0x19, 0x33, 0x36, 0xa0, 0xbe, 0xd7, 0xf3,
0x29, 0xb4, 0x6b, 0xb7, 0xad, 0x8e, 0x7d, 0xb0, 0xc4, 0xe7, 0x1a, 0x76, 0x17, 0x6a, 0x87, 0xd3,
0x54, 0x9c, 0xfb, 0x5d, 0x62, 0x84, 0x73, 0xb0, 0xc4, 0x95, 0x02, 0x77, 0xd2, 0xf2, 0x95, 0xb8,
0x70, 0xab, 0x6d, 0xab, 0xd3, 0xc0, 0x9d, 0x4a, 0xc3, 0xd6, 0xc1, 0xd9, 0x8b, 0xe3, 0xc8, 0x5d,
0x6e, 0x5b, 0x9d, 0x3a, 0x9e, 0x86, 0xd2, 0x5e, 0x0d, 0xaa, 0x14, 0xd8, 0xfb, 0x12, 0xd6, 0xf3,
0xc9, 0x65, 0xe5, 0x62, 0x60, 0x63, 0x3c, 0x2b, 0x8b, 0x87, 0x02, 0x5b, 0xa3, 0x12, 0x56, 0xb2,
0xf3, 0xb1, 0x88, 0x4f, 0x60, 0x99, 0xc2, 0x48, 0x92, 0x37, 0x1f, 0xdd, 0x2b, 0x01, 0x5c, 0x43,
0xc6, 0x33, 0xe7, 0xbd, 0x06, 0x21, 0xfe, 0x9f, 0xc4, 0xef, 0x7a, 0x7f, 0x2f, 0x82, 0x4b, 0xb5,
0xc4, 0x42, 0x1c, 0x05, 0x43, 0x21, 0xcf, 0xe7, 0xb4, 0x46, 0xdd, 0xf1, 0xc5, 0x58, 0xd0, 0x05,
0x1a, 0x9c, 0xd6, 0xde, 0x57, 0x16, 0xac, 0xe6, 0xf7, 0xe3, 0x9d, 0x0c, 0x76, 0x5c, 0x71, 0x27,
0xf2, 0x9a, 0x93, 0xe7, 0x59, 0x91, 0x3c, 0x9b, 0x97, 0xed, 0x2b, 0xf2, 0xe7, 0x1f, 0xe0, 0xbc,
0x0e, 0xc2, 0x64, 0x81, 0xe1, 0x6b, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x5a, 0x54, 0x5f, 0xc4,
0xd3, 0x51, 0x2a, 0x31, 0xe4, 0x52, 0xf0, 0xf6, 0xa1, 0x81, 0xfb, 0x65, 0xe2, 0x9e, 0x0c, 0x96,
0xd1, 0xca, 0xe8, 0x0f, 0xa8, 0xe5, 0xf2, 0xa0, 0x75, 0xa8, 0x92, 0x73, 0x86, 0x84, 0x14, 0xbc,
0x03, 0x00, 0xb4, 0x4e, 0x64, 0x9c, 0x2d, 0xa8, 0x92, 0x94, 0x81, 0x50, 0x0c, 0x24, 0x8d, 0x97,
0x44, 0xba, 0x07, 0x55, 0x7f, 0x94, 0x3e, 0x7d, 0x8c, 0x66, 0x49, 0x48, 0xbc, 0x8d, 0xcd, 0x33,
0xca, 0x4c, 0xa1, 0x2e, 0xa1, 0x8b, 0x67, 0x3a, 0x80, 0x65, 0x04, 0x40, 0x2d, 0xb6, 0x95, 0xae,
0xca, 0x93, 0x04, 0xfc, 0x6c, 0x79, 0x3c, 0xd3, 0x90, 0x64, 0x12, 0xfb, 0x93, 0x3a, 0xc5, 0xa1,
0x9c, 0x6f, 0x18, 0x9f, 0x12, 0xde, 0x42, 0x1d, 0xfb, 0x7f, 0x80, 0x7f, 0x25, 0xf1, 0x74, 0x4c,
0xa0, 0xb1, 0x0e, 0x54, 0x49, 0xca, 0xf2, 0x63, 0x7a, 0x93, 0xba, 0x1b, 0x97, 0x0e, 0xe5, 0xa0,
0x63, 0x71, 0x7a, 0xd3, 0xa1, 0xfc, 0xd2, 0x38, 0x2e, 0x91, 0x4a, 0xf5, 0x93, 0x20, 0x9a, 0x9b,
0x4f, 0x82, 0x28, 0xcb, 0x1b, 0x97, 0xf9, 0x30, 0xb6, 0x0a, 0x73, 0x17, 0xea, 0x2f, 0xa3, 0x38,
0x48, 0xd1, 0x19, 0x63, 0x59, 0x7c, 0x2e, 0xb3, 0x1d, 0x80, 0xae, 0xe8, 0x87, 0xc3, 0x20, 0x42,
0xab, 0x53, 0x6c, 0x00, 0x99, 0x8d, 0x1b, 0x4e, 0xde, 0x13, 0xa8, 0x65, 0x52, 0x39, 0xf6, 0xa8,
0xed, 0xf5, 0x83, 0x48, 0xa8, 0x5b, 0x90, 0xe0, 0xfd, 0x0f, 0x56, 0x24, 0x19, 0xf1, 0xf9, 0xe8,
0x89, 0xf4, 0x1a, 0x54, 0xbc, 0xd6, 0x43, 0xe4, 0x7d, 0x63, 0x81, 0x83, 0x2b, 0x15, 0xc0, 0xd2,
0x01, 0xcc, 0xaf, 0xd1, 0x91, 0x5f, 0x23, 0x6b, 0x43, 0xb3, 0x97, 0xe2, 0x3b, 0xa5, 0xdb, 0x58,
0x83, 0x9b, 0x2a, 0xc4, 0xcb, 0x1f, 0xa5, 0xba, 0xdc, 0x36, 0x9f, 0xcb, 0x6c, 0x03, 0x1a, 0xd8,
0x9b, 0xa4, 0x11, 0x1b, 0x59, 0x9d, 0x6b, 0x05, 0xdb, 0x04, 0x50, 0xc8, 0x4e, 0x05, 0x75, 0x33,
0x8b, 0x1b, 0x1a, 0xef, 0x21, 0xd4, 0xf0, 0xa6, 0x87, 0xc1, 0x58, 0xe7, 0x66, 0x5d, 0x95, 0xdb,
0x07, 0x0b, 0x5a, 0xff, 0x9d, 0x8a, 0xe4, 0x82, 0x8b, 0x2f, 0xa6, 0x62, 0x92, 0x22, 0xb6, 0x24,
0x2b, 0x2e, 0x93, 0x80, 0xac, 0xed, 0xbd, 0x09, 0x92, 0x81, 0x44, 0xca, 0xe1, 0x99, 0x84, 0xb9,
0x6a, 0xcc, 0x27, 0x94, 0x6b, 0x9d, 0x9b, 0x2a, 0xe2, 0xbb, 0x18, 0xc6, 0xa9, 0x4a, 0x26, 0x93,
0x58, 0x07, 0x6e, 0xec, 0x9f, 0xf7, 0xa3, 0xe9, 0x40, 0xf0, 0x78, 0x26, 0x77, 0x53, 0x73, 0xe6,
0x45, 0x35, 0xfb, 0x33, 0x36, 0x37, 0x52, 0xa9, 0xd6, 0x54, 0x23, 0xc7, 0x82, 0x96, 0xed, 0x40,
0x6b, 0x7f, 0x78, 0x2a, 0x06, 0x03, 0x31, 0xe8, 0x06, 0x69, 0xe0, 0xd6, 0x29, 0xef, 0xc2, 0x83,
0x9f, 0x73, 0xf1, 0xde, 0x59, 0xb0, 0x92, 0x65, 0x3f, 0x19, 0xc7, 0xa3, 0x89, 0xc0, 0x12, 0xef,
0x27, 0x89, 0x2a, 0xf1, 0x7e, 0x92, 0xb0, 0x87, 0x50, 0xe3, 0x62, 0x32, 0x8d, 0x52, 0xc5, 0x92,
0xdb, 0x3a, 0xa2, 0xda, 0x3b, 0x8d, 0x52, 0xae, 0xbc, 0xd8, 0x3f, 0x61, 0x35, 0xc7, 0x43, 0xf5,
0x2c, 0xfc, 0x41, 0xef, 0xcb, 0xd9, 0x79, 0xc1, 0xdd, 0xfb, 0xe0, 0x40, 0xd3, 0x88, 0x3c, 0x27,
0x19, 0xe2, 0xb3, 0x92, 0x91, 0xec, 0x3e, 0xcd, 0x5d, 0x97, 0x4c, 0x3d, 0xd8, 0x93, 0x5a, 0x60,
0x1d, 0x65, 0xb4, 0xb4, 0x8e, 0x74, 0x23, 0xb4, 0xaf, 0x6a, 0x84, 0x38, 0xc5, 0xbd, 0x09, 0x46,
0x67, 0x62, 0x40, 0xb4, 0xac, 0x73, 0x25, 0xb2, 0x6d, 0xdd, 0x15, 0xa8, 0x8e, 0xb9, 0x5e, 0xa3,
0x2c, 0x5c, 0x77, 0x0e, 0xd9, 0xe5, 0x70, 0x32, 0xa8, 0x49, 0xbe, 0x48, 0x89, 0x3d, 0x85, 0xa6,
0x6e, 0x5f, 0x93, 0xac, 0x44, 0xeb, 0x3a, 0x94, 0x36, 0x72, 0xd3, 0x91, 0x3d, 0x2f, 0x8e, 0x68,
0x6e, 0x83, 0x6e, 0xe1, 0xe6, 0x32, 0x37, 0xec, 0xbc, 0x38, 0xd2, 0xed, 0x18, 0x33, 0xa3, 0x0b,
0xb4, 0xf9, 0x96, 0xde, 0x3c, 0x37, 0x71, 0x63, 0xb2, 0x7c, 0x6c, 0xbe, 0x25, 0x6e, 0x93, 0xf6,
0xac, 0xe7, 0x91, 0x93, 0x36, 0x6e, 0xbe, 0x39, 0x3b, 0xc6, 0x43, 0xe6, 0xb6, 0x8a, 0x07, 0xcd,
0x4d, 0xdc, 0x78, 0xee, 0xfc, 0x92, 0xf9, 0xce, 0x5d, 0xa1, 0xad, 0xe5, 0xc3, 0x9b, 0x74, 0xe1,
0x25, 0x53, 0xe1, 0xf3, 0xe2, 0x24, 0xe0, 0xae, 0x16, 0x81, 0xca, 0xdb, 0x79, 0xc1, 0xdf, 0xfb,
0xae, 0x02, 0x2b, 0xfe, 0x70, 0x1c, 0x27, 0xa9, 0xd1, 0x12, 0xfc, 0xd1, 0x40, 0x9c, 0xab, 0x96,
0x40, 0x42, 0xf9, 0xab, 0x49, 0xad, 0x19, 0x5b, 0x03, 0xb5, 0x02, 0x87, 0x4b, 0xc1, 0xa0, 0x83,
0x93, 0xa3, 0xc3, 0x06, 0x34, 0x24, 0xf7, 0xd1, 0x54, 0x25, 0x93, 0x56, 0xc8, 0x1f, 0x00, 0x33,
0x1a, 0x1c, 0x6b, 0x34, 0x8a, 0x2a, 0x11, 0xdb, 0xa0, 0x74, 0x23, 0x63, 0x9d, 0x8c, 0x86, 0x06,
0xed, 0xc7, 0xe1, 0x50, 0x4c, 0xd2, 0x60, 0x38, 0xc6, 0xbe, 0x62, 0x77, 0x6c, 0x6e, 0x68, 0xb0,
0xa5, 0x50, 0x12, 0x2f, 0x12, 0x11, 0xa4, 0x62, 0xb0, 0x9b, 0x12, 0x9d, 0x6c, 0x5e, 0xd0, 0xa2,
0x1f, 0xa5, 0xa5, 0xfd, 0x40, 0xfa, 0xe5, 0xb5, 0xf4, 0x2c, 0x46, 0x22, 0x48, 0x88, 0x24, 0x75,
0x2e, 0x05, 0xef, 0xc7, 0x0a, 0x30, 0x89, 0xa4, 0x1c, 0xfc, 0x7e, 0x37, 0x38, 0xaf, 0x86, 0x2d,
0x0f, 0x4e, 0x6d, 0x01, 0x9c, 0x3b, 0xf3, 0x71, 0x55, 0x02, 0x93, 0x49, 0xd8, 0xcb, 0xf5, 0x4b,
0x22, 0x51, 0xb5, 0xb8, 0xa9, 0x62, 0x1e, 0xb4, 0x8c, 0x67, 0x0c, 0xbf, 0x41, 0x8c, 0x9d, 0xd3,
0x95, 0x40, 0x0b, 0xd7, 0x84, 0xb6, 0x79, 0x35, 0xb4, 0x2d, 0x13, 0xda, 0x77, 0x16, 0xb4, 0x76,
0xd3, 0x78, 0x18, 0xf6, 0xb9, 0xe8, 0xc7, 0xc9, 0xe0, 0x72, 0x50, 0x25, 0x7c, 0x15, 0x13, 0xbe,
0x6d, 0xb0, 0xfd, 0xb7, 0x49, 0xd6, 0x0a, 0x37, 0x8c, 0x41, 0x6b, 0xa1, 0x56, 0x1c, 0x1d, 0xd9,
0x03, 0xa8, 0xf8, 0x09, 0x31, 0x37, 0xd7, 0xc4, 0x73, 0x1f, 0x09, 0xaf, 0xf8, 0x89, 0x77, 0x02,
0xeb, 0xc7, 0x49, 0x30, 0x9a, 0x44, 0x41, 0x2a, 0x10, 0xea, 0x4f, 0xa9, 0x78, 0xc9, 0xef, 0x65,
0xef, 0x2f, 0x70, 0xbb, 0x10, 0x57, 0xbf, 0x56, 0x48, 0x01, 0x5b, 0xff, 0xea, 0xec, 0xc1, 0xad,
0xb9, 0xab, 0xdf, 0xfd, 0xa4, 0x1b, 0x2c, 0x06, 0xfd, 0xab, 0x91, 0x17, 0x05, 0xcd, 0x8e, 0x2f,
0xbb, 0xeb, 0x1e, 0xb8, 0x19, 0x30, 0xf2, 0xc7, 0x7a, 0x76, 0x83, 0x93, 0x50, 0xcc, 0x2e, 0xfb,
0x3d, 0x43, 0xaf, 0x75, 0x85, 0x7e, 0xe2, 0xd3, 0xda, 0xfb, 0xd5, 0x82, 0xf5, 0xb2, 0x20, 0x9a,
0x0c, 0x96, 0x41, 0x06, 0xf6, 0x0c, 0xaa, 0x6f, 0x43, 0x31, 0x53, 0xef, 0xb3, 0xb7, 0x50, 0xa2,
0x85, 0x9b, 0x70, 0xb9, 0x01, 0x3f, 0x85, 0xdd, 0x7e, 0x1a, 0xc6, 0x23, 0x35, 0x8c, 0x4b, 0x09,
0xcf, 0xd9, 0x8b, 0xe2, 0xfe, 0xe7, 0xf2, 0x67, 0x26, 0x97, 0x42, 0x09, 0xb5, 0xab, 0xd7, 0xa4,
0xf6, 0x72, 0x19, 0xb5, 0xbd, 0x6f, 0x2d, 0x85, 0x95, 0x31, 0x30, 0x7d, 0xb4, 0x62, 0x9a, 0xd0,
0xb6, 0x22, 0xb4, 0x2b, 0xa7, 0x3e, 0x3d, 0xdc, 0x2a, 0x11, 0x27, 0x4d, 0x5c, 0xd2, 0x7f, 0x0c,
0x0e, 0x55, 0x69, 0x2e, 0x7f, 0xa4, 0x8b, 0x2c, 0x26, 0xbb, 0x5c, 0x96, 0xec, 0xde, 0xda, 0xf7,
0xef, 0x37, 0xad, 0x1f, 0xde, 0x6f, 0x5a, 0x3f, 0xbf, 0xdf, 0xb4, 0xbe, 0xfe, 0x65, 0x73, 0xe9,
0x74, 0x99, 0xfe, 0x23, 0xfa, 0xdb, 0x6f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x58, 0x5b, 0x7b, 0xe3,
0x33, 0x12, 0x00, 0x00,
// 1639 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xdb, 0xca,
0x11, 0x37, 0x45, 0xca, 0x92, 0x46, 0xb2, 0xe3, 0x6c, 0x94, 0x94, 0x48, 0x1d, 0x47, 0x20, 0xdc,
0x46, 0x2d, 0x0a, 0x07, 0x4e, 0x93, 0x20, 0x97, 0xb6, 0xb1, 0x23, 0xa7, 0x26, 0x52, 0xbb, 0xe9,
0xca, 0x70, 0x6f, 0x05, 0x68, 0x69, 0xeb, 0x10, 0xa5, 0x44, 0x95, 0xa2, 0x22, 0xfb, 0x52, 0xa0,
0x9f, 0x21, 0x97, 0x7e, 0x84, 0x7e, 0x8e, 0x5e, 0xfa, 0x8e, 0xef, 0xf8, 0x80, 0x77, 0x79, 0xc8,
0x7b, 0xdf, 0x22, 0x97, 0x87, 0x99, 0xe5, 0x6a, 0x97, 0x14, 0xed, 0x18, 0xc1, 0xbb, 0xed, 0xfc,
0xd9, 0xd9, 0x99, 0xdf, 0xcc, 0xce, 0x0e, 0x09, 0xad, 0xc9, 0xec, 0x2c, 0x0a, 0x07, 0x3b, 0x93,
0x24, 0x4e, 0x63, 0x56, 0x0f, 0xc7, 0xa9, 0x48, 0xc6, 0x41, 0xe4, 0x4d, 0xc1, 0xe6, 0xf1, 0x9c,
0xb9, 0x50, 0x7b, 0x15, 0x47, 0xb3, 0xd1, 0x78, 0xea, 0x5a, 0x1d, 0xbb, 0xeb, 0x70, 0x45, 0x32,
0x06, 0xce, 0x1b, 0x71, 0x39, 0x75, 0xed, 0x8e, 0xdd, 0x6d, 0x70, 0x5a, 0xb3, 0x6d, 0xa8, 0xee,
0xa5, 0x69, 0x32, 0x75, 0x2b, 0x1d, 0xbb, 0xdb, 0x7c, 0xb2, 0xbe, 0xa3, 0xcc, 0xed, 0x20, 0x9b,
0x4b, 0x21, 0xda, 0xe4, 0x71, 0x90, 0x84, 0xe3, 0x73, 0xd7, 0xe9, 0x58, 0xdd, 0x16, 0x57, 0xa4,
0x77, 0x04, 0x8d, 0x7e, 0x78, 0x3e, 0x16, 0x43, 0x3c, 0xfa, 0x21, 0xd8, 0x6f, 0x63, 0x3c, 0xd6,
0xea, 0x36, 0x9f, 0xac, 0x69, 0x53, 0x3c, 0x9e, 0x73, 0x94, 0xa0, 0xc2, 0xb1, 0x38, 0x77, 0x2b,
0xa5, 0x0a, 0xc7, 0xe2, 0xdc, 0x7b, 0x01, 0xeb, 0x3c, 0x9e, 0xfb, 0x43, 0x31, 0x4e, 0xc3, 0xbf,
0x87, 0x22, 0x21, 0xa7, 0x79, 0x3c, 0x57, 0xb1, 0xd0, 0x7a, 0x11, 0x48, 0x45, 0x07, 0xe2, 0xdd,
0x87, 0x55, 0xbf, 0xf7, 0xa7, 0x70, 0x9a, 0xb2, 0x0d, 0xb0, 0xfd, 0x9e, 0xda, 0x80, 0x4b, 0xcf,
0x87, 0xdb, 0x07, 0x17, 0x69, 0x12, 0x0c, 0x52, 0x31, 0xf4, 0x7b, 0x12, 0x0e, 0xb6, 0x0e, 0x15,
0xbf, 0x47, 0xbe, 0x3a, 0xbc, 0xe2, 0xf7, 0xd8, 0x36, 0x38, 0xa7, 0x41, 0xa4, 0x80, 0xd8, 0xd0,
0xce, 0x49, 0xb3, 0x9c, 0xa4, 0xde, 0x59, 0xce, 0xd4, 0x51, 0x90, 0x26, 0xe1, 0x05, 0xbb, 0x07,
0xab, 0xaf, 0x43, 0x11, 0x0d, 0xe5, 0xa1, 0x0d, 0x9e, 0x51, 0xec, 0x99, 0x4e, 0x85, 0xb4, 0xfa,
0x73, 0x6d, 0x75, 0xc9, 0xa1, 0x45, 0x9e, 0xbc, 0x07, 0x50, 0x7b, 0x23, 0x2e, 0x29, 0x16, 0x15,
0xa9, 0x65, 0x44, 0xfa, 0xad, 0x05, 0x77, 0x16, 0xbb, 0x4f, 0x82, 0xb3, 0x48, 0x9c, 0x06, 0xd1,
0x4c, 0xb0, 0x6d, 0x15, 0xb7, 0x55, 0xe6, 0xff, 0xe1, 0x0a, 0x61, 0xc1, 0x1e, 0x2d, 0xb0, 0x43,
0xb5, 0xdb, 0x5a, 0x2d, 0x3b, 0xf2, 0x70, 0x25, 0xab, 0x8c, 0x4d, 0xa8, 0xef, 0xf7, 0x7d, 0x32,
0xed, 0xda, 0x1d, 0xab, 0x6b, 0x1f, 0xae, 0xf0, 0x05, 0x87, 0xdd, 0x87, 0xda, 0xd1, 0x2c, 0x15,
0x17, 0x7e, 0x8f, 0x2a, 0xc2, 0x39, 0x5c, 0xe1, 0x8a, 0x81, 0x3b, 0x69, 0xf9, 0x46, 0x5c, 0xba,
0xd5, 0x8e, 0xd5, 0x6d, 0xe0, 0x4e, 0xc5, 0x61, 0x6d, 0x70, 0xf6, 0xe3, 0x38, 0x72, 0x57, 0x3b,
0x56, 0xb7, 0x8e, 0xa7, 0x21, 0xb5, 0x5f, 0x83, 0x2a, 0x19, 0xf6, 0xfe, 0x05, 0xed, 0x7c, 0x70,
0x59, 0xba, 0x18, 0xd8, 0x68, 0xcf, 0xca, 0xec, 0x21, 0xc1, 0x36, 0x28, 0x85, 0x95, 0xec, 0x7c,
0x4c, 0xe2, 0x33, 0x58, 0x25, 0x33, 0xb2, 0xc8, 0x9b, 0x4f, 0x1e, 0x94, 0x00, 0xae, 0x21, 0xe3,
0x99, 0xf2, 0x7e, 0x83, 0x10, 0xff, 0x73, 0xe2, 0xf7, 0xbc, 0xdf, 0x15, 0xc1, 0xa5, 0x5c, 0x62,
0x22, 0x8e, 0x83, 0x91, 0x90, 0xe7, 0x73, 0x5a, 0x23, 0xef, 0xe4, 0x72, 0x22, 0xc8, 0x81, 0x06,
0xa7, 0xb5, 0xf7, 0x6f, 0x0b, 0xd6, 0xf3, 0xfb, 0xd1, 0x27, 0xa3, 0x3a, 0xae, 0xf1, 0x89, 0xb4,
0x16, 0xc5, 0xf3, 0xa2, 0x58, 0x3c, 0x5b, 0x57, 0xed, 0x2b, 0xd6, 0xcf, 0xef, 0xc1, 0x79, 0x1b,
0x84, 0xc9, 0x52, 0x85, 0x6f, 0x48, 0x08, 0x6d, 0x72, 0xd7, 0x96, 0xb9, 0xa8, 0xbe, 0x8a, 0x67,
0xe3, 0x54, 0x62, 0xc8, 0x25, 0xe1, 0x1d, 0x40, 0x03, 0xf7, 0xcb, 0xc0, 0x3d, 0x69, 0x2c, 0x2b,
0x2b, 0xa3, 0x3f, 0x20, 0x97, 0xcb, 0x83, 0xda, 0x50, 0x25, 0xe5, 0x0c, 0x09, 0x49, 0x78, 0x87,
0x00, 0x28, 0x9d, 0x4a, 0x3b, 0xdb, 0x50, 0x25, 0x2a, 0x03, 0xa1, 0x68, 0x48, 0x0a, 0xaf, 0xb0,
0xf4, 0x00, 0xaa, 0xfe, 0x38, 0x7d, 0xfe, 0x14, 0xc5, 0xb2, 0x20, 0xd1, 0x1b, 0x9b, 0x67, 0x25,
0x33, 0x83, 0xba, 0x84, 0x2e, 0x9e, 0x6b, 0x03, 0x96, 0x61, 0x00, 0xb9, 0xd8, 0x56, 0x7a, 0x2a,
0x4e, 0x22, 0xf0, 0xda, 0xf2, 0x78, 0xae, 0x21, 0xc9, 0x28, 0xf6, 0x0b, 0x75, 0x8a, 0x43, 0x31,
0xdf, 0x32, 0xae, 0x12, 0x7a, 0xa1, 0x8e, 0xfd, 0x1b, 0xc0, 0x1f, 0x93, 0x78, 0x36, 0x21, 0xd0,
0x58, 0x17, 0xaa, 0x44, 0x65, 0xf1, 0x31, 0xbd, 0x49, 0xf9, 0xc6, 0xa5, 0x42, 0x39, 0xe8, 0x98,
0x9c, 0xfe, 0x6c, 0x24, 0x6f, 0x1a, 0xc7, 0x25, 0x96, 0x52, 0xfd, 0x34, 0x88, 0x16, 0xe2, 0xd3,
0x20, 0xca, 0xe2, 0xc6, 0x65, 0xde, 0x8c, 0xad, 0xcc, 0xdc, 0x87, 0xfa, 0xeb, 0x28, 0x0e, 0x52,
0x54, 0x46, 0x5b, 0x16, 0x5f, 0xd0, 0x6c, 0x17, 0xa0, 0x27, 0x06, 0xe1, 0x28, 0x88, 0x50, 0xea,
0x14, 0x1b, 0x40, 0x26, 0xe3, 0x86, 0x92, 0xf7, 0x0c, 0x6a, 0x19, 0x55, 0x8e, 0x3d, 0x72, 0xfb,
0x83, 0x20, 0x12, 0xca, 0x0b, 0x22, 0xbc, 0xbf, 0xc2, 0x9a, 0x2c, 0x46, 0x7c, 0x3e, 0xfa, 0x22,
0xbd, 0x41, 0x29, 0xde, 0xe8, 0x21, 0xf2, 0xfe, 0x6b, 0x81, 0x83, 0x2b, 0x65, 0xc0, 0xd2, 0x06,
0xcc, 0xdb, 0xe8, 0xc8, 0xdb, 0xc8, 0x3a, 0xd0, 0xec, 0xa7, 0xf8, 0x4e, 0xe9, 0x36, 0xd6, 0xe0,
0x26, 0x0b, 0xf1, 0xf2, 0xc7, 0xa9, 0x4e, 0xb7, 0xcd, 0x17, 0x34, 0xdb, 0x84, 0x06, 0xf6, 0x26,
0x29, 0xc4, 0x46, 0x56, 0xe7, 0x9a, 0xc1, 0xb6, 0x00, 0x14, 0xb2, 0x33, 0x41, 0xdd, 0xcc, 0xe2,
0x06, 0xc7, 0x7b, 0x0c, 0x35, 0xf4, 0xf4, 0x28, 0x98, 0xe8, 0xd8, 0xac, 0xeb, 0x62, 0xfb, 0x64,
0x41, 0xeb, 0x2f, 0x33, 0x91, 0x5c, 0x72, 0xf1, 0xcf, 0x99, 0x98, 0xa6, 0x88, 0x2d, 0xd1, 0xaa,
0x96, 0x89, 0xc0, 0xaa, 0xed, 0xbf, 0x0b, 0x92, 0xa1, 0x44, 0xca, 0xe1, 0x19, 0x85, 0xb1, 0x6a,
0xcc, 0xa7, 0x14, 0x6b, 0x9d, 0x9b, 0x2c, 0xaa, 0x77, 0x31, 0x8a, 0x53, 0x15, 0x4c, 0x46, 0xb1,
0x2e, 0xdc, 0x3a, 0xb8, 0x18, 0x44, 0xb3, 0xa1, 0xe0, 0xf1, 0x5c, 0xee, 0xa6, 0xe6, 0xcc, 0x8b,
0x6c, 0xf6, 0x4b, 0x6c, 0x6e, 0xc4, 0x52, 0xad, 0xa9, 0x46, 0x8a, 0x05, 0x2e, 0xdb, 0x85, 0xd6,
0xc1, 0xe8, 0x4c, 0x0c, 0x87, 0x62, 0xd8, 0x0b, 0xd2, 0xc0, 0xad, 0x53, 0xdc, 0x85, 0x07, 0x3f,
0xa7, 0xe2, 0x7d, 0xb0, 0x60, 0x2d, 0x8b, 0x7e, 0x3a, 0x89, 0xc7, 0x53, 0x81, 0x29, 0x3e, 0x48,
0x12, 0x95, 0xe2, 0x83, 0x24, 0x61, 0x8f, 0xa1, 0xc6, 0xc5, 0x74, 0x16, 0xa5, 0xaa, 0x4a, 0xee,
0x6a, 0x8b, 0x6a, 0xef, 0x2c, 0x4a, 0xb9, 0xd2, 0x62, 0x7f, 0x80, 0xf5, 0x5c, 0x1d, 0xaa, 0x67,
0xe1, 0x67, 0x7a, 0x5f, 0x4e, 0xce, 0x0b, 0xea, 0xde, 0x27, 0x07, 0x9a, 0x86, 0xe5, 0x45, 0x91,
0x21, 0x3e, 0x6b, 0x59, 0x91, 0x3d, 0xa4, 0xb9, 0xeb, 0x8a, 0xa9, 0x07, 0x7b, 0x52, 0x0b, 0xac,
0xe3, 0xac, 0x2c, 0xad, 0x63, 0xdd, 0x08, 0xed, 0xeb, 0x1a, 0x21, 0x4e, 0x71, 0xef, 0x82, 0xf1,
0xb9, 0x18, 0x52, 0x59, 0xd6, 0xb9, 0x22, 0xd9, 0x8e, 0xee, 0x0a, 0x94, 0xc7, 0x5c, 0xaf, 0x51,
0x12, 0xae, 0x3b, 0x87, 0xec, 0x72, 0x38, 0x19, 0xd4, 0x64, 0xbd, 0x48, 0x8a, 0x3d, 0x87, 0xa6,
0x6e, 0x5f, 0xd3, 0x2c, 0x45, 0x6d, 0x6d, 0x4a, 0x0b, 0xb9, 0xa9, 0xc8, 0x5e, 0x16, 0x47, 0x34,
0xb7, 0x41, 0x5e, 0xb8, 0xb9, 0xc8, 0x0d, 0x39, 0x2f, 0x8e, 0x74, 0xbb, 0xc6, 0xcc, 0xe8, 0x02,
0x6d, 0xbe, 0xa3, 0x37, 0x2f, 0x44, 0xdc, 0x98, 0x2c, 0x9f, 0x9a, 0x6f, 0x89, 0xdb, 0xa4, 0x3d,
0xed, 0x3c, 0x72, 0x52, 0xc6, 0xcd, 0x37, 0x67, 0xd7, 0x78, 0xc8, 0xdc, 0x56, 0xf1, 0xa0, 0x85,
0x88, 0x1b, 0xcf, 0x9d, 0x5f, 0x32, 0xdf, 0xb9, 0x6b, 0xb4, 0xb5, 0x7c, 0x78, 0x93, 0x2a, 0xbc,
0x64, 0x2a, 0x7c, 0x59, 0x9c, 0x04, 0xdc, 0xf5, 0x22, 0x50, 0x79, 0x39, 0x2f, 0xe8, 0x7b, 0xff,
0xaf, 0xc0, 0x9a, 0x3f, 0x9a, 0xc4, 0x49, 0x6a, 0xb4, 0x04, 0x7f, 0x3c, 0x14, 0x17, 0xaa, 0x25,
0x10, 0x51, 0xfe, 0x6a, 0x52, 0x6b, 0xc6, 0xd6, 0x40, 0xad, 0xc0, 0xe1, 0x92, 0x30, 0xca, 0xc1,
0xc9, 0x95, 0xc3, 0x26, 0x34, 0x64, 0xed, 0xa3, 0xa8, 0x4a, 0x22, 0xcd, 0x90, 0x1f, 0x00, 0x73,
0x1a, 0x1c, 0x6b, 0x34, 0x8a, 0x2a, 0x12, 0xdb, 0xa0, 0x54, 0x23, 0x61, 0x9d, 0x84, 0x06, 0x07,
0xe5, 0x27, 0xe1, 0x48, 0x4c, 0xd3, 0x60, 0x34, 0xc1, 0xbe, 0x62, 0x77, 0x6d, 0x6e, 0x70, 0xb0,
0xa5, 0x50, 0x10, 0xaf, 0x12, 0x11, 0xa4, 0x62, 0xb8, 0x97, 0x52, 0x39, 0xd9, 0xbc, 0xc0, 0x45,
0x3d, 0x0a, 0x4b, 0xeb, 0x81, 0xd4, 0xcb, 0x73, 0xe9, 0x59, 0x8c, 0x44, 0x90, 0x50, 0x91, 0xd4,
0xb9, 0x24, 0xbc, 0x6f, 0x2a, 0xc0, 0x24, 0x92, 0x72, 0xf0, 0xfb, 0xc9, 0xe0, 0xbc, 0x1e, 0xb6,
0x3c, 0x38, 0xb5, 0x25, 0x70, 0xee, 0x2d, 0xc6, 0x55, 0x09, 0x4c, 0x46, 0x61, 0x2f, 0xd7, 0x2f,
0x89, 0x44, 0xd5, 0xe2, 0x26, 0x8b, 0x79, 0xd0, 0x32, 0x9e, 0x31, 0xbc, 0x83, 0x68, 0x3b, 0xc7,
0x2b, 0x81, 0x16, 0x6e, 0x08, 0x6d, 0xf3, 0x7a, 0x68, 0x5b, 0x26, 0xb4, 0x1f, 0x2c, 0x68, 0xed,
0xa5, 0xf1, 0x28, 0x1c, 0x70, 0x31, 0x88, 0x93, 0xe1, 0xd5, 0xa0, 0x4a, 0xf8, 0x2a, 0x26, 0x7c,
0x3b, 0x60, 0xfb, 0xef, 0x93, 0xac, 0x15, 0x6e, 0x1a, 0x83, 0xd6, 0x52, 0xae, 0x38, 0x2a, 0xb2,
0x47, 0x50, 0xf1, 0x13, 0xaa, 0xdc, 0x5c, 0x13, 0xcf, 0x5d, 0x12, 0x5e, 0xf1, 0x13, 0xef, 0x37,
0xd0, 0x96, 0x4e, 0x29, 0x51, 0xf6, 0xa8, 0xb4, 0xa1, 0x7a, 0x90, 0x24, 0xb1, 0x7a, 0x56, 0x24,
0xe1, 0x9d, 0x42, 0xfb, 0x24, 0x09, 0xc6, 0xd3, 0x28, 0x48, 0x05, 0x26, 0xe6, 0x4b, 0xea, 0xa3,
0xe4, 0xeb, 0xda, 0xfb, 0x15, 0xdc, 0x2d, 0xd8, 0xd5, 0x6f, 0x1b, 0x16, 0x8c, 0xad, 0xbf, 0x51,
0xfb, 0x70, 0x67, 0xa1, 0xea, 0xf7, 0xbe, 0xc8, 0x83, 0x65, 0xa3, 0xbf, 0x36, 0xe2, 0x22, 0xa3,
0xd9, 0xf1, 0x65, 0xbe, 0xee, 0x83, 0x9b, 0x61, 0x25, 0x3f, 0xed, 0x33, 0x0f, 0x4e, 0x43, 0x31,
0xbf, 0xea, 0xeb, 0x87, 0xde, 0xf6, 0x0a, 0xfd, 0x10, 0xa0, 0xb5, 0xf7, 0x83, 0x05, 0xed, 0x32,
0x23, 0xba, 0x74, 0x2c, 0xa3, 0x74, 0xd8, 0x0b, 0xa8, 0xbe, 0x0f, 0xc5, 0x5c, 0xbd, 0xe6, 0xde,
0x52, 0x42, 0x97, 0x3c, 0xe1, 0x72, 0x03, 0x5e, 0x9c, 0xbd, 0x41, 0x1a, 0xc6, 0x63, 0x35, 0xba,
0x4b, 0x0a, 0xcf, 0xd9, 0x8f, 0xe2, 0xc1, 0x3f, 0xe4, 0x47, 0x29, 0x97, 0x44, 0xc9, 0x45, 0xa8,
0xde, 0xf0, 0x22, 0xac, 0x96, 0x5d, 0x04, 0xef, 0x7f, 0x96, 0xc2, 0xca, 0x18, 0xaf, 0x3e, 0x9b,
0x31, 0x5d, 0xfe, 0xb6, 0x2a, 0x7f, 0x57, 0xce, 0x88, 0x7a, 0x14, 0x56, 0x24, 0xce, 0xa5, 0xb8,
0xa4, 0x3f, 0x12, 0x0e, 0x65, 0x69, 0x41, 0x7f, 0xa6, 0xe7, 0x2c, 0x07, 0xbb, 0x5a, 0x16, 0xec,
0xfe, 0xc6, 0x57, 0x1f, 0xb7, 0xac, 0xaf, 0x3f, 0x6e, 0x59, 0xdf, 0x7d, 0xdc, 0xb2, 0xfe, 0xf3,
0xfd, 0xd6, 0xca, 0xd9, 0x2a, 0xfd, 0x51, 0xfa, 0xed, 0x8f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa2,
0x18, 0x37, 0x1a, 0x61, 0x12, 0x00, 0x00,
}
func (m *Row) Marshal() (dAtA []byte, err error) {
@ -4548,6 +4597,40 @@ func (m *AtomicRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *AtomicImportResponse) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *AtomicImportResponse) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *AtomicImportResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.Error) > 0 {
i -= len(m.Error)
copy(dAtA[i:], m.Error)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Error)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@ -5809,6 +5892,22 @@ func (m *AtomicRecord) Size() (n int) {
return n
}
func (m *AtomicImportResponse) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Error)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *TranslateKeysRequest) Size() (n int) {
if m == nil {
return 0
@ -11038,6 +11137,92 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: AtomicImportResponse: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: AtomicImportResponse: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthPublic
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Error = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0

4
internal/public.proto
View file

@ -199,6 +199,10 @@ message AtomicRecord {
repeated ImportRequest Ir = 4;
}
message AtomicImportResponse {
string Error = 1;
}
message TranslateKeysRequest {
string Index = 1;
string Field = 2;

5
license.exceptions
View file

@ -10,10 +10,11 @@
./logger/filewriter.go
./logger/filewriter_test.go
./vprint.go
./vprint_test.go
./rbf/vprint.go
./cmd/slurp/vprint.go
./cmd/badloader/vprint.go
./cmd/demo-lmdb/vprint.go
./gid.go
./cmd/lmdb-keydump/vprint.go
./cmd/lmdb-keydump/keydump.go
./cmd/pilosa-keydump/vprint.go
./cmd/pilosa-keydump/keydump.go

315
lmdb.go
View file

@ -37,6 +37,19 @@ import (
"github.com/pkg/errors"
)
// Linux builds note:
//
// This setting of -DMDB_USE_SYSV_SEM=1 is required in the cgo build
// under Linux to avoid a random deadlock occurance deep inside
// the LMDB C code.
//
// Futher documentation here: https://github.com/bmatsuo/lmdb-go/issues/94
//
// The effect of the -D define above is to that the LMDB C code then
// uses SysV semaphores and avoids POSIX thread-local semaphores. SysV semaphores
// are stored system-wide. POSIX semaphores are kept on thread-local storage,
// which is apparently problematic.
// lmdbRegistrar facilitates shutdown
// of all the lmdb databases started under
// tests. Its needed because most tests don't cleanup
@ -57,6 +70,8 @@ type lmdbRegistrar struct {
var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar()
var globalNextTxSnLMDB int64
func newLMDBTestRegistrar() *lmdbRegistrar {
return &lmdbRegistrar{
@ -88,7 +103,6 @@ func DumpAllLMDB() {
globalLMDBReg.mu.Lock()
defer globalLMDBReg.mu.Unlock()
for w := range globalLMDBReg.mp {
_ = w
AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil))
}
}
@ -110,7 +124,7 @@ func lmdbPath(path string) string {
// if one does not exist for its bpath. Otherwise it returns
// the existing instance. This insures only one lmdbDB
// per bpath in this pilosa node.
func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper, error) {
path := lmdbPath(path0)
r.mu.Lock()
@ -123,20 +137,23 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
// otherwise, make a new lmdb and store it in globalLMDBReg
runtime.LockOSThread()
defer runtime.UnlockOSThread()
const MaxReaders = 256 // default is 126
const MaxReaders = 126 // default is 126
env, err := lmdb.NewEnvMaxReaders(MaxReaders)
panicOn(err)
err = env.SetMaxDBs(1)
panicOn(err)
//err = env.SetMapSize(256 << 30) // 256GB
err = env.SetMapSize(16 << 30) // 16GB
err = env.SetMapSize(4 << 30) // 4GB
panicOn(err)
panicOn(os.MkdirAll(filepath.Dir(path), 0755))
flags := uint(lmdb.NoReadahead | lmdb.NoSubdir)
//flags := uint(lmdb.NoReadahead | lmdb.NoSubdir)
//flags := uint(lmdb.NoSubdir) // no difference without the No.Readahead on ./query.
flags := uint(0)
// unsafe, but get upper bound on performance.
// WriteMap = C.MDB_WRITEMAP // Use a writable memory map.
@ -163,6 +180,9 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
lmdb.NoSync | // Don't fsync after commit.
lmdb.MapAsync // Flush asynchronously when using the WriteMap flag.
if !DirExists(path) {
panicOn(os.MkdirAll(path, 0755))
}
err = env.Open(path, flags, 0644)
if err != nil {
AlwaysPrintf("error env.Open(path='%v'): '%v'; on gid = '%v'", path, err, curGID())
@ -191,18 +211,53 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
panicOn(err)
w = &LMDBWrapper{
name: name,
env: env,
reg: r,
path: path,
dbi: dbi,
name: name,
env: env,
reg: r,
path: path,
dbi: dbi,
doAllocZero: doAllocZero,
openTx: make(map[*LMDBTx]bool),
}
r.unprotectedRegister(w)
w.startStack = stack()
return w, nil
}
func (w *LMDBWrapper) OpenListString() (r string) {
list := w.listopen()
if len(list) == 0 {
return "<no open LMDBTx>"
}
for i, ltx := range list {
if ltx.o.Write {
r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o)
} else {
r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o)
}
}
return
}
func (w *LMDBWrapper) listopen() (slc []*LMDBTx) {
w.muDb.Lock()
for v := range w.openTx {
slc = append(slc, v)
}
w.muDb.Unlock()
return
}
func (w *LMDBWrapper) OpenSnList() (slc []int64) {
w.muDb.Lock()
for v := range w.openTx {
slc = append(slc, v.sn)
}
w.muDb.Unlock()
return
}
var ErrShutdown = fmt.Errorf("shutting down")
// DeleteIndex deletes all the containers associated with
@ -212,8 +267,8 @@ func (w *LMDBWrapper) DeleteIndex(indexName string) error {
// We use the apostrophie rune `'` to locate the end of the
// index name in the key prefix, so we cannot allow indexNames
// themselves to contain apostrophies.
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in LMDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
if strings.Contains(indexName, "/") {
return fmt.Errorf("error: bad indexName `%v` in LMDBWrapper.DeleteIndex() call: indexName cannot contain '/'.", indexName)
}
prefix := txkey.IndexOnlyPrefix(indexName)
return w.DeletePrefix(prefix)
@ -222,7 +277,7 @@ func (w *LMDBWrapper) DeleteIndex(indexName string) error {
// statically confirm that LMDBTx satisfies the Tx interface.
var _ Tx = (*LMDBTx)(nil)
// LMDBWrapper provides the NewLMDBTx() method.
// LMDBWrapper provides the NewTx() method.
// Execute lmdbJob's via LMDBWrapper.submit(); these must
// be done by the lmdb goroutine worker pool.
type LMDBWrapper struct {
@ -249,22 +304,28 @@ type LMDBWrapper struct {
// corrupted data.
doAllocZero bool
// stack() from our creation point, to track tests
// that haven't closed us.
startStack string
DeleteEmptyContainer bool
nextTxSn int64
openTx map[*LMDBTx]bool
}
func (w *LMDBWrapper) IsClosed() bool {
w.muDb.Lock()
defer w.muDb.Unlock()
return w.closed
// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are.
// Can't have more than one active write per database, so the
// 2nd one will block until the first finishes.
func (w *LMDBWrapper) NewTxWRITE() *lmdb.Txn {
lmdbTxn, err := w.env.BeginTxn(nil, 0)
panicOn(err)
return lmdbTxn
}
// NewLMDBTx produces LMDB based ACID transactions. If
// NewTxREAD lets us see in the callstack dumps where the READ tx are.
func (w *LMDBWrapper) NewTxREAD() *lmdb.Txn {
lmdbTxn, err := w.env.BeginTxn(nil, lmdb.Readonly)
panicOn(err)
return lmdbTxn
}
// NewTx produces LMDB based ACID or ACI transactions. If
// the transaction will modify data, then the write flag must be true.
// Read-only queries should set write to false, to allow more concurrency.
// Methods on a LMDBTx are thread-safe, and can be called from
@ -276,34 +337,43 @@ func (w *LMDBWrapper) IsClosed() bool {
// but when set is highly useful for debugging. It has no impact
// on transaction behavior.
//
func (w *LMDBWrapper) NewLMDBTx(write bool, initialIndexName string, frag *fragment) (tx *LMDBTx) {
//w.muDb.Lock()
//defer w.muDb.Unlock()
func (w *LMDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
rwflag := uint(0) // writable txn denotated by lack of the lmdb.Readonly flag.
if !write {
rwflag = lmdb.Readonly
}
sn := atomic.AddInt64(&globalNextTxSnLMDB, 1)
//vv("lmdb new tx _sn_ %v; stack \n%v", sn, stack())
runtime.LockOSThread()
sn := atomic.AddInt64(&w.nextTxSn, 1)
lmdbTxn, err := w.env.BeginTxn(nil, rwflag)
panicOn(err)
var lmdbTxn *lmdb.Txn
if write {
// see the WRITE tx on the callstack.
lmdbTxn = w.NewTxWRITE()
} else {
// see the READ tx on the callstack.
lmdbTxn = w.NewTxREAD()
}
lmdbTxn.RawRead = true
tx = &LMDBTx{
sn: sn,
write: write,
tx: lmdbTxn,
dbi: w.dbi,
Db: w,
frag: frag,
//initloc: stack(),
ltx := &LMDBTx{
sn: sn,
write: write,
tx: lmdbTxn,
dbi: w.dbi,
Db: w,
frag: o.Fragment,
doAllocZero: w.doAllocZero,
initialIndexName: initialIndexName,
DeleteEmptyContainer: w.DeleteEmptyContainer,
o: o,
gid: curGID(),
}
tx = ltx
w.muDb.Lock()
w.openTx[ltx] = true
w.muDb.Unlock()
return
}
@ -312,6 +382,13 @@ func (w *LMDBWrapper) Close() (err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
// complain if there are still Tx in flight, b/c otherwise we will see
// the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1'
if len(w.openTx) > 0 {
AlwaysPrintf("error: cannot close LMDBWrapper with Tx still in flight.")
return
}
w.reg.unregister(w)
w.closed = true
w.env.CloseDBI(w.dbi)
@ -321,6 +398,13 @@ func (w *LMDBWrapper) Close() (err error) {
return nil
}
func (w *LMDBWrapper) IsClosed() (closed bool) {
w.muDb.Lock()
closed = w.closed
w.muDb.Unlock()
return
}
// LMDBTx wraps a lmdb.Txn and provides the Tx interface
// method implementations.
// The methods on LMDBTx are thread-safe, and can be called
@ -348,6 +432,39 @@ type LMDBTx struct {
DeleteEmptyContainer bool
unlocked bool // runtime.UnlockOSThread has been done.
o Txo
// NewTx, write operations, Commit and/or Rollback must all take place on
// the same gid and it must the runtime.LockOSThreaded first. Verify
// that we are using the right goroutine in a debug build using the
// gid, stored here, used for NewTx().
gid uint64
}
// sanity check that database is open.
func (tx *LMDBTx) sanity() {
if tx.Db.IsClosed() {
panic("cannot operate on closed LMDB")
}
}
// debugOnlyGidcheck is expected to be sort of slow. So once we are sure
// of correctness, turn it off.
// func (tx *LMDBTx) debugOnlyGidcheck() {
//
// // only applies to write txn. Reads should be able to share the txn.
// if tx.o.Write {
// cur := curGID()
// if cur != tx.gid {
// panic(fmt.Sprintf("must use write LMDBTx from same gid that created it! creator gid=%v, but user gid now =%v", tx.gid, cur))
// }
// }
//
// }
func (tx *LMDBTx) Group() *TxGroup {
return tx.o.Group
}
func (tx *LMDBTx) Type() string {
@ -367,10 +484,18 @@ func (tx *LMDBTx) Pointer() string {
// Rollback rolls back the transaction.
func (tx *LMDBTx) Rollback() {
tx.sanity()
//vv("lmdb rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack())
tx.Db.muDb.Lock()
delete(tx.Db.openTx, tx)
tx.Db.muDb.Unlock()
tx.mu.Lock()
defer tx.mu.Unlock()
//pp("LMDBTx.Rollback p=%p, its: '%v' initloc: '%v',\n rollbackloc:'%v'", tx, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack())
//tx.debugOnlyGidcheck()
tx.tx.Abort() // must hold tx.mu mutex lock
if !tx.unlocked {
@ -383,11 +508,18 @@ func (tx *LMDBTx) Rollback() {
// Commits can handle up to 100k updates to fragments
// at once, but not more. This is a LMDBDB imposed limit.
func (tx *LMDBTx) Commit() error {
tx.sanity()
//vv("lmdb commit tx _sn_ %v; stack \n%v", tx.sn, stack())
tx.Db.muDb.Lock()
delete(tx.Db.openTx, tx)
tx.Db.muDb.Unlock()
tx.mu.Lock()
defer tx.mu.Unlock()
//pp("LMDBTx.Commit (write:%v) p=%p, stackID=%x openit: '%v' initloc: '%v', commitloc:\n%v", tx.write, tx, stackID, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack())
//tx.debugOnlyGidcheck()
err := tx.tx.Commit() // must hold tx.mu mutex lock
panicOn(err)
@ -395,7 +527,6 @@ func (tx *LMDBTx) Commit() error {
runtime.UnlockOSThread()
tx.unlocked = true
}
//pp("done committing LMDBTx sn=%v", tx.sn)
return err
}
@ -421,6 +552,8 @@ func (tx *LMDBTx) Container(index, field, view string, shard uint64, ckey uint64
bkey := txkey.Key(index, field, view, shard, ckey)
tx.mu.Lock()
//tx.debugOnlyGidcheck()
v, err := tx.tx.Get(tx.dbi, bkey)
tx.mu.Unlock()
@ -462,9 +595,10 @@ func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uin
panic(fmt.Sprintf("unknown container type: %v", ct))
}
tx.mu.Lock()
//tx.debugOnlyGidcheck()
err := tx.tx.Put(tx.dbi, bkey, append(by, ct), 0) // TODO: this might make a copy; can meta byte be stored elsewhere?
tx.mu.Unlock()
//panicOn(err) // mdb_put: invalid argument
// TODO(jea): need to handle?
// lmdb.TxnFull
@ -477,6 +611,8 @@ func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uin
func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error {
bkey := txkey.Key(index, field, view, shard, ckey)
tx.mu.Lock()
//tx.debugOnlyGidcheck()
err := tx.tx.Del(tx.dbi, bkey, nil)
tx.mu.Unlock()
if lmdb.IsNotFound(err) {
@ -579,6 +715,9 @@ func (tx *LMDBTx) Contains(index, field, view string, shard uint64, key uint64)
bkey := txkey.Key(index, field, view, shard, hi)
tx.mu.Lock()
var v []byte
//tx.debugOnlyGidcheck()
v, err = tx.tx.Get(tx.dbi, bkey)
tx.mu.Unlock()
if lmdb.IsNotFound(err) {
@ -671,7 +810,13 @@ type LMDBIterator struct {
// NewLMDBIterator creates an iterator on tx that will
// only return badgerKeys that start with prefix.
func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
tx.mu.Lock()
//tx.debugOnlyGidcheck()
cur, err := tx.tx.OpenCursor(tx.dbi)
tx.mu.Unlock()
panicOn(err)
bi = &LMDBIterator{
@ -686,7 +831,11 @@ func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
// Close tells the database and transaction that the user is done
// with the iterator.
func (bi *LMDBIterator) Close() {
bi.tx.mu.Lock()
//bi.tx.debugOnlyGidcheck()
bi.cur.Close()
bi.tx.mu.Unlock()
}
// Valid returns false if there are no more values in the iterator's range.
@ -696,9 +845,13 @@ func (bi *LMDBIterator) Valid() bool {
// Seek allows the iterator to start at needle instead of the global begining.
func (bi *LMDBIterator) Seek(needle []byte) (ok bool) {
bi.tx.mu.Lock()
defer bi.tx.mu.Unlock()
bi.seen++ // if ommited, red TestLMDB_ContainerIterator_empty_iteration_loop() in lmdb_test.go.
//bi.tx.debugOnlyGidcheck()
var k, v []byte
var err error
getflag := uint(lmdb.SetRange)
@ -783,6 +936,8 @@ func (bi *LMDBIterator) Next() (ok bool) {
skipEmpty:
var k, v []byte
var err error
bi.tx.mu.Lock()
if getflag == lmdb.SetRange && len(prefix) == 0 {
// don't do nil as key on setrange, will panic
// b/c keys in LMDB must be at least one byte long.
@ -793,6 +948,10 @@ skipEmpty:
} else {
k, v, err = bi.cur.Get(prefix, nil, getflag)
}
bi.tx.mu.Unlock()
//bi.tx.debugOnlyGidcheck()
if lmdb.IsNotFound(err) {
bi.lastKey = nil
bi.lastVal = nil
@ -934,6 +1093,8 @@ func (tx *LMDBTx) Max(index, field, view string, shard uint64) (uint64, error) {
prefix := txkey.Prefix(index, field, view, shard)
seekto := txkey.Prefix(index, field, view, shard+1)
//tx.debugOnlyGidcheck()
cur, err := tx.tx.OpenCursor(tx.dbi)
panicOn(err)
defer cur.Close()
@ -1109,14 +1270,6 @@ func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, e
needle := txkey.Key(index, field, view, shard, hi0)
prefix := txkey.Prefix(index, field, view, shard)
n2, pre2 := txkey.KeyAndPrefix(index, field, view, shard, hi0)
if string(n2) != string(needle) {
panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(n2), string(needle)))
}
if string(pre2) != string(prefix) {
panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(pre2), string(prefix)))
}
it := NewLMDBIterator(tx, prefix)
defer it.Close()
it.Seek(needle)
@ -1263,6 +1416,8 @@ func (tx *LMDBTx) ImportRoaringBits(index, field, view string, shard uint64, itr
func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
//tx.debugOnlyGidcheck()
if len(v) == 0 {
return nil
}
@ -1292,7 +1447,10 @@ func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
} else {
w = v
}
return ToContainer(typ, w)
}
func ToContainer(typ byte, w []byte) (r *roaring.Container) {
switch typ {
case roaring.ContainerArray:
c := roaring.NewContainerArray(toArray16(w))
@ -1312,9 +1470,9 @@ func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
// keys available in lmdb.
func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
if optionalUseThisTx == nil {
tx := w.NewLMDBTx(!writable, "<StringifiedLMDBKeys>", nil)
tx, _ := w.NewTx(!writable, "<StringifiedLMDBKeys>", Txo{})
defer tx.Rollback()
r = stringifiedLMDBKeysTx(tx)
r = stringifiedLMDBKeysTx(tx.(*LMDBTx))
return
}
@ -1331,6 +1489,8 @@ func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
// formatted bkey.
func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
//tx.debugOnlyGidcheck()
v, err := tx.tx.Get(tx.dbi, bkey)
if lmdb.IsNotFound(err) {
// some queries bkey may not be present! don't panic.
@ -1347,7 +1507,6 @@ func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
}
func (tx *LMDBTx) Dump() {
fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx))
}
@ -1370,7 +1529,7 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
any = true
bkey := it.lastKey
key := string(bkey)
key := txkey.ToString(bkey)
ckey := txkey.KeyExtractContainerKey(bkey)
hash := ""
srbm := ""
@ -1379,24 +1538,26 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
if n == 0 {
panic("should not have empty v here")
}
hash = blake3sum16(v[0:(n - 1)])
hash = Blake3sum16(v[0:(n - 1)])
ct := tx.toContainer(v[n-1], v[0:(n-1)])
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
srbm = bitmapAsString(rbm)
srbm = BitmapAsString(rbm)
r += fmt.Sprintf("%v -> %v (%v hot)\n", key, hash, tx.countBitsSet(bkey))
r += " ......." + srbm + "\n"
}
r += "]\n all-in-blake3:" + blake3sum16([]byte(r))
r += "]\n all-in-blake3:" + Blake3sum16([]byte(r))
if !any {
return ""
return "<empty lmdb database>"
}
return "lmdb-" + r
}
func (w *LMDBWrapper) DeleteDBPath(path string) (err error) {
func (w *LMDBWrapper) DeleteDBPath(dbs *DBShard) (err error) {
path := dbs.Path
err = os.RemoveAll(path)
if err != nil {
return errors.Wrap(err, "DeleteDBPath")
@ -1413,7 +1574,7 @@ func (w *LMDBWrapper) DeleteField(index, field, fieldPath string) error {
// under blue-green roaring_lmdb, the directory will not be found, b/c roaring will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := w.DeleteDBPath(fieldPath)
err := w.DeleteDBPath(&DBShard{Path: fieldPath})
if err != nil {
return errors.Wrap(err, "removing directory")
}
@ -1428,20 +1589,25 @@ func (w *LMDBWrapper) DeleteFragment(index, field, view string, shard uint64, fr
func (w *LMDBWrapper) DeletePrefix(prefix []byte) error {
tx := w.NewLMDBTx(writable, w.name, nil)
tx, _ := w.NewTx(writable, w.name, Txo{})
// NewLMDBTx will grab these, so don't lock until after it.
// NewTx will grab these, so don't lock until after it.
w.muDb.Lock()
defer w.muDb.Unlock()
bi := NewLMDBIterator(tx, prefix)
bi := NewLMDBIterator(tx.(*LMDBTx), prefix)
for bi.Next() {
err := bi.cur.Del(0)
panicOn(err)
if err != nil {
w.muDb.Unlock()
panic(err)
}
}
bi.Close()
// Commit will grab the w.muDb lock, so we must release it first.
w.muDb.Unlock()
err := tx.Commit()
panicOn(err)
@ -1461,3 +1627,12 @@ func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, f
}
return ioutil.NopCloser(&buf), sz, err
}
func (tx *LMDBTx) Options() Txo {
return tx.o
}
// Sn retreives the serial number of the Tx.
func (tx *LMDBTx) Sn() int64 {
return tx.sn
}

17
lmdb_other.go
View file

@ -59,6 +59,10 @@ func newLMDBTestRegistrar() *lmdbRegistrar {
}
}
func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper, error) {
panic("lmdb only available on 64-bit arch")
}
// register each lmdb created under tests, so we
// can clean them up. This is called by openLMDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
@ -154,6 +158,10 @@ func (tx *LMDBTx) UseRowCache() bool {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) Group() *TxGroup {
panic("lmdb only available on 64-bit arch")
}
// Pointer gives us a memory address for the underlying transaction for debugging.
// It is public because we use it in roaring to report invalid container memory access
// outside of a transaction.
@ -161,6 +169,11 @@ func (tx *LMDBTx) Pointer() string {
panic("lmdb only available on 64-bit arch")
}
// Sn retreives the serial number of the Tx.
func (tx *LMDBTx) Sn() int64 {
panic("lmdb only available on 64-bit arch")
}
// Rollback rolls back the transaction.
func (tx *LMDBTx) Rollback() {
panic("lmdb only available on 64-bit arch")
@ -421,3 +434,7 @@ func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, f
func (tx *LMDBTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) Options() Txo {
panic("lmdb only available on 64-bit arch")
}

152
lmdb_test.go
View file

@ -17,15 +17,12 @@
package pilosa
import (
"bytes"
"fmt"
"math"
"os"
"strconv"
"testing"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
// helpers, each runs their own new txn, and commits if a change/delete
@ -33,7 +30,7 @@ import (
func LMDBMustHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -46,7 +43,7 @@ func LMDBMustHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard
func LMDBMustNotHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -57,7 +54,7 @@ func LMDBMustNotHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, sha
}
func LMDBMustSetBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
// add a bit
changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@ -75,14 +72,14 @@ func LMDBMustSetBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard u
}
func LMDBMustDeleteBitvalueContainer(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
}
func LMDBMustDeleteBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
@ -92,18 +89,19 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) {
var err error
fn := lmdbPath(path)
panicOn(os.RemoveAll(fn))
w, err = globalLMDBReg.openLMDBWrapper(fn)
ww, err := globalLMDBReg.OpenDBWrapper(fn, DetectMemAccessPastTx)
panicOn(err)
w = ww.(*LMDBWrapper)
// verify it is empty
allkeys := w.StringifiedLMDBKeys(nil)
if allkeys != "" {
if allkeys != "<empty lmdb database>" {
panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys))
}
return w, func() {
w.Close()
panicOn(w.DeleteDBPath(fn))
panicOn(w.DeleteDBPath(&DBShard{Path: fn}))
}
}
@ -120,7 +118,7 @@ func TestLMDB_DeleteFragment(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
shard1 := uint64(1)
@ -155,7 +153,7 @@ func TestLMDB_DeleteFragment(t *testing.T) {
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ = dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
for _, s := range shards {
@ -210,13 +208,12 @@ func TestLMDB_Max_on_many_containers(t *testing.T) {
}
}
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
for _, shard := range shards {
max, err := tx.Max(index, field, view, uint64(shard))
panicOn(err)
//vv("highbits of max = %v from shard = %v", max, shard)
if max != uint64(shard) {
panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard))
}
@ -243,7 +240,7 @@ func TestLMDB_SetBitmap(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
bitvalue := uint64(0)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -264,7 +261,7 @@ func TestLMDB_SetBitmap(t *testing.T) {
// commited, so should be visible outside the txn
//
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
@ -284,7 +281,7 @@ func TestLMDB_OffsetRange(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
@ -318,13 +315,13 @@ func TestLMDB_OffsetRange(t *testing.T) {
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
// should see our 1M value
s2 := bitmapAsString(rbm2)
s2 := BitmapAsString(rbm2)
expect2 := "c(1048576, 1048577)"
if s2 != expect2 {
panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2))
@ -332,13 +329,13 @@ func TestLMDB_OffsetRange(t *testing.T) {
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewLMDBTx(!writable, index, nil)
tx3, _ := dbwrap.NewTx(!writable, index, Txo{})
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
//expect to see 3M == 3145728
s3 := bitmapAsString(rbm3)
s3 := BitmapAsString(rbm3)
expect3 := "c(3145728, 3145729)"
if s3 != expect3 {
@ -360,7 +357,7 @@ func TestLMDB_Count_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
@ -376,7 +373,7 @@ func TestLMDB_Count_dense_containers(t *testing.T) {
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
expected := 0
for i := uint64(0); i < (1<<16)+2; i += 2 {
@ -402,7 +399,7 @@ func TestLMDB_ContainerIterator_on_empty(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
@ -420,7 +417,7 @@ func TestLMDB_ContainerIterator_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
bitvalue := uint64(42)
@ -472,55 +469,12 @@ func TestLMDB_ContainerIterator_on_one_bit(t *testing.T) {
}
}
func TestLMDB_badgerKey_badgerPrefix(t *testing.T) {
// badgerPrefix() must agree with badgerKey(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := txkey.Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := txkey.Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("txkey.Prefix() output '%v'was not a prefix of txkey.Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("txkey.Prefix() output '%v'was 20 characters shorter than txkey.Key() '%v'", string(needle), string(prefix)))
}
// validate assumption that txkey.KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to txkey.KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
txkey.KeyExtractContainerKey(prefix) // should panic.
}()
}
func TestLMDB_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -575,7 +529,7 @@ func TestLMDB_ContainerIterator_empty_iteration_loop(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -625,7 +579,7 @@ func TestLMDB_ForEach_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
bitvalue := uint64(42)
@ -684,7 +638,7 @@ func TestLMDB_RemoveContainer_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
@ -693,7 +647,7 @@ func TestLMDB_RemoveContainer_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewLMDBTx(writable, index, nil)
tx, _ = dbwrap.NewTx(writable, index, Txo{})
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
@ -745,7 +699,7 @@ func TestLMDB_Remove_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
@ -756,7 +710,7 @@ func TestLMDB_Remove_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewLMDBTx(writable, index, nil)
tx, _ = dbwrap.NewTx(writable, index, Txo{})
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
@ -788,7 +742,7 @@ func TestLMDB_Min_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
@ -805,7 +759,7 @@ func TestLMDB_Min_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ = dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
@ -826,7 +780,7 @@ func TestLMDB_CountRange_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
if n != 0 {
@ -842,7 +796,7 @@ func TestLMDB_CountRange_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ = dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@ -870,7 +824,7 @@ func TestLMDB_CountRange_middle_container(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
// pick out just the middle container with the 1 bit set on it.
@ -895,7 +849,7 @@ func TestLMDB_CountRange_many_middle_container(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
// get them all
@ -926,7 +880,7 @@ func TestLMDB_UnionInPlace(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
if n != 2 {
@ -941,7 +895,7 @@ func TestLMDB_UnionInPlace(t *testing.T) {
}
mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
@ -966,7 +920,7 @@ func TestLMDB_RoaringBitmap(t *testing.T) {
putme := expected
LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
@ -990,9 +944,9 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
tx.(*LMDBTx).DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
//bitvalue := uint64(42)
@ -1059,10 +1013,10 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
if n != 0 {
panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n))
}
allkeys := stringifiedLMDBKeysTx(tx)
allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
// should have no keys
if allkeys != "" {
if allkeys != "<empty lmdb database>" {
panic("lmdb should have no keys now")
}
}
@ -1073,7 +1027,7 @@ func TestLMDB_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1123,7 +1077,7 @@ func TestLMDB_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1182,7 +1136,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
bitvalue := uint64(777)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
for _, v := range bits {
@ -1219,7 +1173,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index2, nil)
tx, _ = dbwrap.NewTx(!writable, index2, Txo{})
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1231,7 +1185,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
exists, err = tx.Contains(index, field, view, shard, v)
panicOn(err)
if exists {
allkeys := stringifiedLMDBKeysTx(tx)
allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))
}
}
@ -1244,7 +1198,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index, nil)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
bitvalue := uint64(777)
limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
//limit := uint64(101)
@ -1257,7 +1211,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
panicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
tx = dbwrap.NewLMDBTx(writable, index, nil)
tx, _ = dbwrap.NewTx(writable, index, Txo{})
}
}
@ -1274,7 +1228,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index2, nil)
tx, _ = dbwrap.NewTx(!writable, index2, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1286,7 +1240,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
exists, err = tx.Contains(index, field, view, shard, v<<16)
panicOn(err)
if exists {
allkeys := stringifiedLMDBKeysTx(tx)
allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))
}
}
@ -1303,7 +1257,7 @@ func TestLMDB_SliceOfShards(t *testing.T) {
for _, shard := range shards {
LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index, nil)
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
slc, err := tx.SliceOfShards(index, field, view, "")

9
main_test.go
View file

@ -15,11 +15,20 @@
package pilosa_test
import (
"fmt"
"net/http"
"testing"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/testhook"
_ "net/http/pprof"
)
func TestMain(m *testing.M) {
port := pilosa.GetAvailPort()
fmt.Printf("pilosa/ TestMain: online stack-traces: curl http://localhost:%v/debug/pprof/goroutine?debug=2\n", port)
go func() {
_ = http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", port), nil)
}()
testhook.RunTestsWithHooks(m)
}

2
mmap_test.go
View file

@ -36,7 +36,7 @@ func forceSnapshotsCheckMapping(t *testing.T) {
f.Logger = logger.NewLogfLogger(t)
defer f.Clean(t)
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
for i := 0; i < f.MaxOpN; i++ {

308
mtx.go
View file

@ -1,308 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
)
// MultiTx implements the transaction interface to combine multiple transactions.
type MultiTx struct {
mu sync.Mutex
writable bool
holder *Holder
index *Index
txs map[multiTxKey]Tx
}
// NewMultiTx returns a new instance of MultiTx for a Holder.
func NewMultiTx(writable bool, holder *Holder) *MultiTx {
return &MultiTx{
writable: writable,
holder: holder,
txs: make(map[multiTxKey]Tx),
}
}
// NewMultiTxWithIndex returns a new instance of MultiTx for a single index.
func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
return &MultiTx{
writable: writable,
index: index,
txs: make(map[multiTxKey]Tx),
}
}
var _ Tx = (*MultiTx)(nil)
type multiTxKey struct {
index string
shard uint64
write bool
}
func (mtx *MultiTx) Type() string {
return mtx.index.Txf.TxType()
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
tx.Rollback()
}
}
// Commit commits all underlying transactions.
func (mtx *MultiTx) Commit() (err error) {
for _, tx := range mtx.txs {
if e := tx.Commit(); e != nil && err == nil {
err = e
}
}
return err
}
func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.RoaringBitmap(index, field, view, shard)
}
func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.Container(index, field, view, shard, key)
}
func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.PutContainer(index, field, view, shard, key, c)
}
func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
}
return tx.Contains(index, field, view, shard, v)
}
func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, false, err
}
return tx.ContainerIterator(index, field, view, shard, key)
}
func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEach(index, field, view, shard, fn)
}
func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEachRange(index, field, view, shard, start, end, fn)
}
func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Count(index, field, view, shard)
}
func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Max(index, field, view, shard)
}
func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, false, err
}
return tx.Min(index, field, view, shard)
}
func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.UnionInPlace(index, field, view, shard, others...)
}
func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.CountRange(index, field, view, shard, start, end)
}
func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.OffsetRange(index, field, view, shard, offset, start, end)
}
// tx returns a transaction by index/shard. Reuses transaction if already open.
// Otherwise begins a new transaction.
func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
// If transaction doesn't exist, lookup the index.
idx := mtx.index
if mtx.holder != nil {
if idx = mtx.holder.Index(index); idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
}
// Begin tranaction & cache it.
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[mkey] = tx
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("txNoShard: no prior tx in MultiTx available, looking up index='%v'", index))
}

2
pilosa.go
View file

@ -170,7 +170,7 @@ func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) {
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// validateName ensures that the index or field name is a valid format.
// validateName ensures that the index or field or view name is a valid format.
func validateName(name string) error {
if !nameRegexp.Match([]byte(name)) {
return errors.Wrapf(ErrName, "'%s'", name)

36
pprof.go
View file

@ -15,11 +15,13 @@
package pilosa
import (
"fmt"
"os"
"runtime"
"runtime/pprof"
"time"
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"runtime/pprof"
)
func CPUProfileForDur(dur time.Duration, outpath string) {
@ -34,14 +36,40 @@ func CPUProfileForDur(dur time.Duration, outpath string) {
panicOn(err)
if dur == 0 {
dur = time.Hour
dur = time.Minute
}
vv("starting cpu profile for dur '%v', output to '%v'", dur, path)
AlwaysPrintf("starting cpu profile for dur '%v', output to '%v'", dur, path)
_ = pprof.StartCPUProfile(f)
go func() {
<-time.After(dur)
pprof.StopCPUProfile()
f.Close()
vv("stopping cpu profile after dur '%v', output: '%v'", dur, path)
AlwaysPrintf("stopping cpu profile after dur '%v', output: '%v'", dur, path)
}()
}
func MemProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc == "" {
txsrc = "roaring"
}
path := outpath + "." + txsrc
f, err := os.Create(path)
panicOn(err)
if dur == 0 {
dur = time.Minute
}
AlwaysPrintf("will write memory profile after dur '%v', output to '%v'", dur, path)
go func() {
<-time.After(dur)
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
panic(fmt.Sprintf("could not write memory profile: %v", err))
}
f.Close()
AlwaysPrintf("wrote memory profile after dur '%v', output: '%v'", dur, path)
}()
}

3
proto/pilosa.proto
View file

@ -1,6 +1,8 @@
syntax = "proto3";
package pilosa;
import "public.proto";
message VDS {
string name = 1;
}
@ -124,4 +126,5 @@ service Pilosa {
rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};
rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {};
rpc Inspect(InspectRequest) returns (stream RowResponse) {};
rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {};
}

85
rbf.go
View file

@ -24,6 +24,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
@ -38,10 +39,14 @@ type RbfDBWrapper struct {
reg *rbfDBRegistrar
muDb sync.Mutex
openTx map[*RBFTx]bool
// make Close() idempotent, avoiding panic on double Close()
closed bool
//DeleteEmptyContainer bool // needed for roaring compat?
doAllocZero bool
}
// rbfDBRegistrar also allows opening the same path twice to
@ -65,7 +70,7 @@ func newRbfDBRegistrar() *rbfDBRegistrar {
}
// register each rbf.DB created, so we dedup and can
// can clean them up. This is called by openRbfDB() while
// can clean them up. This is called by OpenDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
@ -92,15 +97,15 @@ func rbfPath(path string) string {
return path
}
// openRbfDB opens the database in the path directoy
// OpenDBWrapper opens the database in the path directoy
// without deleting any prior content. Any
// database directory will have the "-rbfdb" suffix.
//
// openRbfDB will check the registry and make a new instance only
// OpenDBWrapper will check the registry and make a new instance only
// if one does not exist for its path. Otherwise it returns
// the existing instance. This insures only one RbfDBWrapper
// per bpath in this pilosa node.
func (r *rbfDBRegistrar) openRbfDB(path0 string) (*RbfDBWrapper, error) {
func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper, error) {
path := rbfPath(path0)
r.mu.Lock()
defer r.mu.Unlock()
@ -112,12 +117,15 @@ func (r *rbfDBRegistrar) openRbfDB(path0 string) (*RbfDBWrapper, error) {
db := rbf.NewDB(path)
w = &RbfDBWrapper{
reg: r,
Path: path,
db: db,
reg: r,
Path: path,
db: db,
doAllocZero: doAllocZero,
openTx: make(map[*RBFTx]bool),
}
r.unprotectedRegister(w)
rbf.DoAllocZero = doAllocZero
err := db.Open()
if err != nil {
@ -132,6 +140,9 @@ type RBFTx struct {
initialIndex string
frag *fragment
tx *rbf.Tx
o Txo
sn int64 // serial number
Db *RbfDBWrapper
}
func (tx *RBFTx) DBPath() string {
@ -143,10 +154,18 @@ func (tx *RBFTx) Type() string {
}
func (tx *RBFTx) Rollback() {
tx.Db.muDb.Lock()
delete(tx.Db.openTx, tx)
tx.Db.muDb.Unlock()
tx.tx.Rollback()
}
func (tx *RBFTx) Commit() error {
tx.Db.muDb.Lock()
delete(tx.Db.openTx, tx)
tx.Db.muDb.Unlock()
return tx.tx.Commit()
}
@ -365,6 +384,18 @@ func (tx *RBFTx) Readonly() bool {
return !tx.tx.Writable()
}
func (tx *RBFTx) Group() *TxGroup {
return tx.o.Group
}
func (tx *RBFTx) Options() Txo {
return tx.o
}
func (tx *RBFTx) Sn() int64 {
return tx.sn
}
func (tx *RBFTx) UseRowCache() bool {
// since RFB returns memory mapped data, we can't use
// the rowCache without first making a copy.
@ -437,15 +468,32 @@ func (w *RbfDBWrapper) Close() error {
return w.db.Close()
}
func (w *RbfDBWrapper) NewRBFTx(write bool, initialIndex string, frag *fragment) (*RBFTx, error) {
var globalNextTxSnRBFTx int64
func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (Tx, error) {
tx, err := w.db.Begin(write)
if err != nil {
return nil, err
}
return &RBFTx{tx: tx, initialIndex: initialIndex, frag: frag}, nil
sn := atomic.AddInt64(&globalNextTxSnRBFTx, 1)
rtx := &RBFTx{
tx: tx,
initialIndex: initialIndex,
frag: o.Fragment,
o: o,
sn: sn,
Db: w,
}
w.muDb.Lock()
w.openTx[rtx] = true
w.muDb.Unlock()
return rtx, nil
}
func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
tx, err := w.db.Begin(true)
if err != nil {
return err
@ -458,3 +506,20 @@ func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, f
}
return tx.Commit()
}
func (w *RbfDBWrapper) DeleteDBPath(dbs *DBShard) error {
panic("TODO")
}
func (w *RbfDBWrapper) OpenListString() (r string) {
return "rbf OpenListString not implemented yet"
}
func (w *RbfDBWrapper) OpenSnList() (slc []int64) {
w.muDb.Lock()
for v := range w.openTx {
slc = append(slc, v.sn)
}
w.muDb.Unlock()
return
}

6
rbf/cursorx.go
View file

@ -28,6 +28,10 @@ import (
// directly, but only a copy.
const EnableRowCache = true
// makes a copy, BUT doesn't do the zero out for now TODO(jea) zero out actually to detect
// access past tx.
var DoAllocZero bool
//probably should just implement the container interface
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
@ -147,7 +151,7 @@ func toContainer(l leafCell, tx *Tx) *roaring.Container {
orig := l.Data
var cpMaybe []byte
if EnableRowCache {
if EnableRowCache || DoAllocZero {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = make([]byte, len(orig))

44
rbf/tx.go
View file

@ -24,8 +24,11 @@ import (
"github.com/benbjohnson/immutable"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
var _ = txkey.ToString
// Tx represents a transaction.
type Tx struct {
mu sync.RWMutex
@ -652,6 +655,16 @@ func (tx *Tx) putContainer(name string, key uint64, ct *roaring.Container) error
return c.putLeafCell(cell)
}
func (tx *Tx) putContainerWithCursor(cur *Cursor, key uint64, ct *roaring.Container) error {
if tx.DeleteEmptyContainer && ct.N() == 0 {
if exact, err := cur.Seek(key); err != nil || !exact {
return err
}
return cur.deleteLeafCell(key)
}
return cur.putLeafCell(ConvertToLeafArgs(key, ct))
}
// RemoveContainer removes a container from the bitmap by key.
func (tx *Tx) RemoveContainer(name string, key uint64) error {
tx.mu.Lock()
@ -1403,7 +1416,8 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string
rbm := &roaring.Bitmap{Containers: cts}
srbm := bitmapAsString(rbm)
bkey := rrName + fmt.Sprintf("%020d", ckey)
pre := txkey.PrefixToString([]byte(rrName))
bkey := pre + fmt.Sprintf("ckey@%020d", ckey)
s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
s += " ......." + srbm + "\n"
@ -1439,7 +1453,11 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
var currRow uint64
var oldC *roaring.Container
cur, err := tx.cursor(name)
if err != nil {
return changed, rowSet, err
}
for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() {
if rowSize != 0 {
currRow = itrKey / rowSize
@ -1450,10 +1468,12 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
}
// INVAR: nsynth > 0
oldC, err = tx.container(name, itrKey)
panicOn(err)
if err != nil {
return
// Find existing container, if any.
var oldC *roaring.Container
if exact, err := cur.Seek(itrKey); err != nil {
return changed, rowSet, err
} else if exact {
oldC = toContainer(cur.cell(), tx)
}
if oldC == nil || oldC.N() == 0 {
@ -1462,13 +1482,11 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
// changed of 0 and empty rowSet is perfect, no need to change the defaults.
continue
} else {
changed += nsynth
rowSet[currRow] += nsynth
err = tx.putContainer(name, itrKey, synthC)
if err != nil {
return
if err := tx.putContainerWithCursor(cur, itrKey, synthC); err != nil {
return changed, rowSet, err
}
continue
}
@ -1488,7 +1506,7 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
changes := int(existN - newC.N())
changed += changes
rowSet[currRow] -= changes
err = tx.putContainer(name, itrKey, newC)
err = tx.putContainerWithCursor(cur, itrKey, newC)
if err != nil {
return
}
@ -1506,7 +1524,7 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
// can nsynth be zero? No, because of the continue/invariant above where nsynth > 0
changed += nsynth
rowSet[currRow] += nsynth
err = tx.putContainer(name, itrKey, synthC)
err = tx.putContainerWithCursor(cur, itrKey, synthC)
if err != nil {
return
}
@ -1523,7 +1541,7 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
changed += changes
rowSet[currRow] += changes
err = tx.putContainer(name, itrKey, newC)
err = tx.putContainerWithCursor(cur, itrKey, newC)
if err != nil {
panicOn(err)
return

2
rbf/vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

73
rrtx.go
View file

@ -21,6 +21,7 @@ import (
"os"
"path/filepath"
"strconv"
"sync/atomic"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
@ -32,6 +33,8 @@ type RoaringTx struct {
Index *Index
Field *Field
fragment *fragment
o Txo
sn int64 // serial number
}
func (tx *RoaringTx) Type() string {
@ -50,7 +53,8 @@ func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string)
// SliceOfShards is based on view.openFragments()
file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
path := filepath.Join(optionalViewPath, "fragments")
file, err := os.Open(path)
if os.IsNotExist(err) {
return
} else if err != nil {
@ -70,7 +74,8 @@ func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string)
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
tx.Index.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//panic(fmt.Sprintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name()))
//tx.Index.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
@ -176,7 +181,6 @@ func (tx *RoaringTx) Add(index, field, view string, shard uint64, batched bool,
// This creates a problem because RoaringTx needs the op-log
// to know when to flush the fragment to disk.
count, err := b.AddN(a...) // AddN does oplog batches. needed to keep op-log up to date.
return count, err
}
@ -186,7 +190,6 @@ func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64
return 0, err
}
changed, err := b.Remove(a...) // green TestFragment_Bug_Q2DoubleDelete
panicOn(err)
if changed {
return 1, err
} else {
@ -211,6 +214,7 @@ func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64,
if err != nil {
return nil, false, err
}
//vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice())
citer, found = b.Containers.Iterator(key)
return citer, found, nil
}
@ -292,9 +296,18 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
tx.fragment.field != field ||
tx.fragment.view != view ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
// still insist that index and shard match, since that is the current scope of all Tx.
if tx.fragment.index != index ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. index='%v', field='%v'; view='%v'; shard='%v'; tx.fragment='%#v'", index, field, view, shard, tx.fragment))
}
// cannot use this fragment.
tx.fragment = nil
} else {
return tx.fragment, nil
}
return tx.fragment, nil
}
// If a field is attached, start from there.
@ -346,18 +359,46 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
return frag.storage, nil
}
var globalRoaringReg = &RoaringStore{}
func (r *RoaringStore) OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error) {
return r, nil
}
func (w *RoaringStore) DeleteDBPath(dbs *DBShard) error {
return os.RemoveAll(dbs.Path)
}
func (r *RoaringStore) Close() error {
return nil
}
func (w *RoaringStore) OpenListString() (r string) {
return "RoaringStore.OpenListString() not yet implemented"
}
func (w *RoaringStore) OpenSnList() (sns []int64) {
return nil
}
type RoaringStore struct{}
func NewRoaringStore() *RoaringStore {
return &RoaringStore{}
}
func (db *RoaringStore) Close() error {
return nil
var globalNextTxSnRoaring int64
func (db *RoaringStore) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
sn := atomic.AddInt64(&globalNextTxSnRoaring, 1)
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment, o: o, sn: sn}, nil
}
func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
// under blue-green badger_roaring, the directory will not be found, b/c badger will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
@ -372,6 +413,9 @@ func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
// The fragment should be closed before this.
func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
@ -402,3 +446,16 @@ func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64
r = file
return
}
func (tx *RoaringTx) Group() *TxGroup {
return tx.o.Group
}
func (tx *RoaringTx) Options() Txo {
return tx.o
}
// Sn retreives the serial number of the Tx.
func (tx *RoaringTx) Sn() int64 {
return tx.sn
}

23
server.go
View file

@ -78,8 +78,10 @@ type Server struct { // nolint: maligned
isCoordinator bool
syncer holderSyncer
translationSyncer translationSyncer
translationSyncer TranslationSyncer
resetTranslationSyncCh chan struct{}
// HolderConfig stashes server options that are really Holder options.
holderConfig *HolderConfig
defaultClient InternalClient
dataDir string
@ -98,6 +100,7 @@ type ServerOption func(s *Server) error
func OptServerLogger(l logger.Logger) ServerOption {
return func(s *Server) error {
s.logger = l
s.holderConfig.Logger = l
return nil
}
}
@ -125,7 +128,7 @@ func OptServerDataDir(dir string) ServerOption {
// attribute store.
func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption {
return func(s *Server) error {
s.holder.NewAttrStore = af
s.holderConfig.NewAttrStore = af
return nil
}
}
@ -213,7 +216,7 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
// used to specify the stats client.
func OptServerStatsClient(sc stats.StatsClient) ServerOption {
return func(s *Server) error {
s.holder.Stats = sc
s.holderConfig.StatsClient = sc
return nil
}
}
@ -307,7 +310,7 @@ func OptServerClusterHasher(h Hasher) ServerOption {
// used to specify the translation data store type.
func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption {
return func(s *Server) error {
s.holder.OpenTranslateStore = fn
s.holderConfig.OpenTranslateStore = fn
return nil
}
}
@ -316,7 +319,7 @@ func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption {
// used to specify the remote translation data reader.
func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
return func(s *Server) error {
s.holder.OpenTranslateReader = fn
s.holderConfig.OpenTranslateReader = fn
return nil
}
}
@ -327,7 +330,7 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
// being used for all Tx interface calls.
func OptServerTxsrc(txsrc string) ServerOption {
return func(s *Server) error {
s.holder.Opts.Txsrc = txsrc
s.holderConfig.Txsrc = txsrc
return nil
}
}
@ -339,7 +342,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s := &Server{
closing: make(chan struct{}),
cluster: cluster,
holder: NewHolder(cluster.partitionN),
diagnostics: newDiagnosticsCollector(defaultDiagnosticServer),
systemInfo: newNopSystemInfo(),
defaultClient: nopInternalClient{},
@ -360,10 +362,12 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.InternalClient = s.defaultClient
s.translationSyncer = newActiveTranslationSyncer(s.resetTranslationSyncCh)
s.holder.translationSyncer = s.translationSyncer
s.cluster.translationSyncer = s.translationSyncer
s.diagnostics.server = s
s.holderConfig = DefaultHolderConfig()
s.holderConfig.TranslationSyncer = s.translationSyncer
s.holderConfig.Logger = s.logger
for _, opt := range opts {
err := opt(s)
@ -383,8 +387,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
if err != nil {
return nil, err
}
s.holder.Path = path
s.holder.Logger = s.logger
s.holder = NewHolder(path, s.holderConfig)
s.holder.Stats.SetLogger(s.logger)
s.cluster.Path = path

4
server/grpc.go
View file

@ -377,8 +377,8 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
return errToStatusError(err)
}
// Obtain transaction.
tx := pilosa.NewMultiTxWithIndex(true, index)
// It is okay to pass a nil Tx to field.StringValue(). It will lazily create it.
var tx pilosa.Tx
var fields []*pilosa.Field
for _, field := range index.Fields() {

5
server/handler_test.go
View file

@ -175,7 +175,8 @@ func TestHandler_Endpoints(t *testing.T) {
})
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
tx0, err := holder.BeginTx(true, i0.Index)
const shard = 0
tx0, err := holder.BeginTx(true, i0.Index, shard)
if err != nil {
t.Fatal(err)
}
@ -193,7 +194,7 @@ func TestHandler_Endpoints(t *testing.T) {
}
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
tx1, err := holder.BeginTx(true, i1.Index)
tx1, err := holder.BeginTx(true, i1.Index, shard)
if err != nil {
t.Fatal(err)
}

25
server/server_test.go
View file

@ -22,6 +22,8 @@ import (
"fmt"
"io/ioutil"
"math/rand"
nethttp "net/http"
"os"
"reflect"
"sort"
"strconv"
@ -53,6 +55,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
for i := 0; i < 100; i++ {
//for i := 0; i < 10; i++ {
t.Run(fmt.Sprint(i), func(t *testing.T) {
t.Parallel()
@ -867,10 +870,13 @@ func TestMain_ImportTimestamp(t *testing.T) {
}
// Import data.
if err := m.API.Import(context.Background(), &data); err != nil {
qcx := m.API.Txf().NewQcx()
if err := m.API.Import(context.Background(), qcx, &data); err != nil { /// first write i/0 here. 2nd write here.
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
files, err := ioutil.ReadDir(dir)
@ -919,7 +925,11 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
}
// Import data.
if err := m.API.Import(context.Background(), &data); err != nil {
qcx := m.API.Txf().NewQcx()
if err := m.API.Import(context.Background(), qcx, &data); err != nil {
t.Fatal(err)
}
if err := qcx.Finish(); err != nil {
t.Fatal(err)
}
@ -1209,3 +1219,12 @@ Set("h", adec=100.22)
}
}
func TestMain(m *testing.M) {
port := pilosa.GetAvailPort()
fmt.Printf("server/ TestMain: online stack-traces: curl http://localhost:%v/debug/pprof/goroutine?debug=2\n", port)
go func() {
_ = nethttp.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", port), nil)
}()
os.Exit(m.Run())
}

13
stattx.go
View file

@ -264,6 +264,14 @@ var _ = kReadonly
var _ Tx = (*statTx)(nil)
func (c *statTx) Group() *TxGroup {
return c.b.Group()
}
func (c *statTx) Options() Txo {
return c.b.Options()
}
//IncrementOpN
func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
me := kIncrementOpN
@ -664,3 +672,8 @@ func (c *statTx) SliceOfShards(index, field, view, optionalViewPath string) (sli
}()
return c.b.SliceOfShards(index, field, view, optionalViewPath)
}
// Sn retreives the serial number of the Tx.
func (c *statTx) Sn() int64 {
return c.b.Sn()
}

3
test/cluster.go
View file

@ -90,7 +90,8 @@ func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uin
if com.API.Node().ID != node.ID {
continue
}
err := com.API.Import(context.Background(), &pilosa.ImportRequest{
err := com.API.Import(context.Background(), nil, &pilosa.ImportRequest{
Index: index,
Field: field,
Shard: shard,

18
test/field.go
View file

@ -27,13 +27,14 @@ type Field struct {
*pilosa.Field
}
// newField returns a new instance of Field d/0.
// newField returns a new instance of Field.
func newField(tb testing.TB, opts pilosa.FieldOption) *Field {
path, err := testhook.TempDir(tb, "pilosa-field-")
if err != nil {
panic(err)
}
field, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", "f", opts)
// This path is probably wrong, but we don't care much because it's a scratch holder anyway.
field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", "f", opts)
if err != nil {
panic(err)
}
@ -57,21 +58,10 @@ func (f *Field) close() error { // nolint: unparam
// reopen closes the index and reopens it.
func (f *Field) reopen() error {
var err error
if err := f.Field.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, index, name, pilosa.OptFieldTypeDefault())
if err != nil {
return err
}
if err := f.Open(); err != nil {
return err
}
return nil
return f.Field.Open()
}
// Ensure field can set its cache

40
test/holder.go
View file

@ -40,8 +40,7 @@ func NewHolder(tb testing.TB) *Holder {
panic(err)
}
h := &Holder{Holder: pilosa.NewHolder(pilosa.DefaultPartitionN)}
h.Path = path
h := &Holder{Holder: pilosa.NewHolder(path, nil)}
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h
@ -64,11 +63,6 @@ func (h *Holder) Close() error {
// Reopen instantiates and opens a new holder.
// Note that the holder must be Closed first.
func (h *Holder) Reopen() error {
path, logger := h.Path, h.Holder.Logger
h.Holder = pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Holder.Path = path
h.Holder.Logger = logger
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h.Holder.Open()
}
@ -88,8 +82,7 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
var tx pilosa.Tx
row, err := f.Row(tx, rowID)
if err != nil {
@ -111,8 +104,7 @@ func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row {
if f == nil {
panic(errors.Wrap(pilosa.ErrFieldNotFound, field))
}
tx := idx.Txf.NewTx(pilosa.Txo{Write: false, Field: f})
defer tx.Rollback()
var tx pilosa.Tx
row, err := f.Row(tx, rowID)
if err != nil {
@ -139,8 +131,7 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
var tx pilosa.Tx
row, err := f.RowTime(tx, rowID, t, quantum)
if err != nil {
@ -157,6 +148,9 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
h.SetBitTime(index, field, rowID, columnID, nil)
}
var vv = pilosa.VV
var _ = vv // happy linter
// SetBitTime sets a bit with timestamp on the given field.
func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time.Time) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
@ -165,7 +159,8 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
shard := columnID / pilosa.ShardWidth
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
defer tx.Rollback()
_, err = f.SetBit(tx, rowID, columnID, t)
@ -182,7 +177,9 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
shard := columnID / pilosa.ShardWidth
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
defer tx.Rollback()
_, err = f.ClearBit(tx, rowID, columnID)
@ -207,7 +204,8 @@ func (h *Holder) SetValue(index, field string, columnID uint64, value int64) *In
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
shard := columnID / pilosa.ShardWidth
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
defer tx.Rollback()
_, err = f.SetValue(tx, columnID, value)
@ -227,7 +225,8 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) {
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
shard := columnID / pilosa.ShardWidth
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index, Shard: shard})
defer tx.Rollback()
val, exists, err := f.Value(tx, columnID)
@ -245,10 +244,11 @@ func (h *Holder) Range(index, field string, op pql.Token, predicate int64) *pilo
if err != nil {
panic(err)
}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
row, err := f.Range(tx, field, op, predicate)
qcx := h.Txf().NewQcx()
defer qcx.Abort()
row, err := f.Range(qcx, field, op, predicate)
if err != nil {
panic(err)
}

14
test/index.go
View file

@ -32,11 +32,10 @@ func newIndex(tb testing.TB) *Index {
if err != nil {
panic(err)
}
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h := pilosa.NewHolder(path, nil)
testhook.Cleanup(tb, func() {
h.Close()
})
h.Path = path
index, err := h.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
panic(err)
@ -57,19 +56,10 @@ func (i *Index) Close() error {
// Reopen closes the index and reopens it.
func (i *Index) Reopen() error {
var err error
if err := i.Index.Close(); err != nil {
return err
}
path, name := i.Path(), i.Name()
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Path = h.HolderPathFromIndexPath(path, name)
i.Index, err = h.CreateIndex(name, pilosa.IndexOptions{})
if err != nil {
return err
}
return nil
return i.Index.Open(false)
}
// CreateField creates a field with the given options.

23
tx.go
View file

@ -74,7 +74,7 @@ type Tx interface {
// UseRowCache is used by fragment.go unprotectedRow() to determine
// dynamically at runtime if RoaringTx
// are in use, which for continuity want to continue to use the
// are in use, which for continuity wants to continue to use the
// rowCache, or if other storage engines (RBF, Badger) are in
// use, which will mean that the bitmap data stored by the
// rowCache can disappear as it is un-mmap-ed, causing crashes.
@ -193,8 +193,29 @@ type Tx interface {
// one that needs optionalViewPath; any other Tx implementation can ignore that.
SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error)
// Group returns nil or the TxGroup that this Tx is a part of.
Group() *TxGroup
// Dump is for debugging, what does this Tx see as its database?
Dump()
// Options returns the options used to create this Tx. This
// can be implementd by embedding Txo, and Txo provides the
// Options() method.
Options() Txo
// Sn retreives the serial number of the Tx.
Sn() int64
}
// Closer is used by Finders
type Closer interface {
Close()
}
type Dumper interface {
// Dump is for debugging, what does this Tx see as its database?
AllDump()
}
// TxStore has operations that will create and commit multiple

18
tx_test.go
View file

@ -163,10 +163,14 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
air := createAIRUpdate(expectedBalStartingAcct0, expectedBalStartingAcct1)
if err := m0api.ImportAtomicRecord(ctx, air); err != nil {
//vv("BEFORE the first ImportAtomicRecord!")
if err := m0api.ImportAtomicRecord(ctx, nil, air); err != nil {
t.Fatal(err)
}
//vv("AFTER the first ImportAtomicRecord!")
iraBit := queryIRABit(m0api, acctOwnerID, iraField, iraRowID, index)
if !iraBit {
panic("IRA bit should have been set")
@ -193,10 +197,16 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
air = createAIRUpdate(expectedBalEndingAcct0, expectedBalEndingAcct1)
err = m0api.ImportAtomicRecord(ctx, air, opt)
qcx := m0api.Txf().NewQcx()
//vv("just before the SECOND ImportAtomicRecord, qcx is %p, should NOT BE NIL", qcx)
err = m0api.ImportAtomicRecord(ctx, qcx, air, opt)
//err = m0api.ImportAtomicRecord(ctx, nil, air, opt)
if err != pilosa.ErrAborted {
panic(fmt.Sprintf("expected ErrTxnAborted but got err='%#v'", err))
}
// sad path, cleanup
qcx.Abort()
qcx = nil
b0, b1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
//vv("after power failure tx, balance: acct0=%v, acct1=%v", b0, b1)
@ -214,7 +224,7 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
// happy path with no power failure half-way through.
err = m0api.ImportAtomicRecord(ctx, air)
err = m0api.ImportAtomicRecord(ctx, nil, air)
panicOn(err)
eb0, eb1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
@ -231,7 +241,7 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
air.Ivr[1].Clear = true
air.Ir[0].Clear = true
err = m0api.ImportAtomicRecord(ctx, air)
err = m0api.ImportAtomicRecord(ctx, nil, air)
panicOn(err)
eb0, eb1 = queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)

677
txfactory.go
View file

@ -21,6 +21,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"text/tabwriter"
@ -33,7 +34,6 @@ import (
// public strings that pilosa/server/config.go can reference
const (
RoaringTxn string = "roaring"
BadgerTxn string = "badger"
LmdbTxn string = "lmdb"
RBFTxn string = "rbf"
)
@ -48,7 +48,7 @@ const DefaultTxsrc = RoaringTxn
// which the transaction has committed or rolled back. Since
// memory segments will be recycled by the underlying databases,
// this can lead to corruption. When DetectMemAccessPastTx is true,
// code in badger.go will copy the transactionally viewed memory before
// code in lmdb.go will copy the transactionally viewed memory before
// returning it for bitmap reading, and then zero it or overwrite it
// with -2 when the Tx completes.
//
@ -58,24 +58,331 @@ const DetectMemAccessPastTx = false
var sep = string(os.PathSeparator)
// Qcx is a (Pilosa) Query Context.
//
// It flexibly expresses the desired grouping of Tx for mass
// rollback at a query's end. It provides one-time commit for
// an atomic import write Tx that involves multiple fragments.
//
// The most common use of Qcx is to call GetTx() to obtain a Tx locally,
// once the index/shard pair is known:
//
// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) {
// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...})
// defer finisher(&err0)
// ...
// }
//
// Qcx reuses read-only Tx on the same index/shard pair. See
// the Qcx.GetTx() for further discussion. The caveat is of
// course that your "new" read Tx actually has an "old" view
// of the database.
//
// At the moment, given that LMDB demands that
// all write Tx are created and executed on the same C thread, most
// writes to individual shards are commited eagerly and locally
// when the `defer finisher(&err0)` is run.
// This is done by returning a finisher that actually Commits,
// thus freeing the one write slot for re-use. A single
// writer is also required by RBF, so this design accomodates
// both.
//
// In contrast, the default read Tx generated (or re-used) will
// return a no-op finisher and the group of reads as a whole
// will be rolled back (mmap memory released) en-mass when
// Qcx.Abort() is called at the top-most level.
//
// Local use of a (Tx, finisher) pair obtained from Qcx.GetTx()
// doesn't need to care about these details. Local use should
// always invoke finisher(&err0) or finisher(nil) to complete
// the Tx within the local function scope.
//
// In summary write Tx are typically "local"
// and are never saved into the TxGroup. The parallelism
// supplied by TxGroup typically applies only to read Tx.
//
// The one exception is this rule is for the one write Tx
// used during the api.ImportAtomicRecord routine. There
// we make a special write Tx and use it for all matching writes.
// This is then committed at the final, top-level, Qcx.Finish() call.
//
// See also the Qcx.GetTx() example and the TxGroup description below.
//
type Qcx struct {
Grp *TxGroup
Txf *TxFactory
// if we go back to using Qcx values, this must become a pointer,
// or otherwise be dealt with because copies of Mutex are a no-no.
mu sync.Mutex
// RequiredForAtomicWriteTx is used by api.ImportAtomicRecord
// to ensure that all writes happen on this one Tx.
RequiredForAtomicWriteTx *Tx
// efficient access to the options for RequiredForAtomicWriteTx
RequiredTxo *Txo
isRoaring bool
}
// Finish commits/rollsback all stored Tx and resets the
// Qcx for further operations, avoiding the need to call NewQxc() again.
func (q *Qcx) Finish() (err error) {
q.mu.Lock()
defer q.mu.Unlock()
if q.RequiredForAtomicWriteTx != nil {
if q.RequiredTxo.Write {
err = (*q.RequiredForAtomicWriteTx).Commit() // panic here on 2nd. is this a double commit?
} else {
(*q.RequiredForAtomicWriteTx).Rollback()
}
}
err2 := q.Grp.FinishGroup()
q.reset()
if err != nil {
return err
}
return err2
}
// Abort rolls back all Tx generated and stored within the Qcx.
// The Qcx is then reset and can be used again immediately.
func (q *Qcx) Abort() {
q.mu.Lock()
defer q.mu.Unlock()
if q.RequiredForAtomicWriteTx != nil {
(*q.RequiredForAtomicWriteTx).Rollback()
}
q.Grp.AbortGroup()
q.reset()
}
// reset forgets everything are starts fresh with an empty
// group, ready for use again as if NewQcx() had been called.
// q.mu must be held
func (q *Qcx) reset() {
q.RequiredForAtomicWriteTx = nil
q.RequiredTxo = nil
q.Grp = q.Txf.NewTxGroup()
}
// NewQcxWithGroup allocates a freshly allocated and empty Grp.
func (f *TxFactory) NewQcx() (qcx *Qcx) {
qcx = &Qcx{
Grp: f.NewTxGroup(),
Txf: f,
}
if f.typeOfTx == "roaring" {
qcx.isRoaring = true
}
return
}
var NoopFinisher = func(perr *error) {}
// GetTx is used like this:
//
// someFunc(ctx context.Context, shard uint64) (_ interface{}, err0 error) {
//
// tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
// defer finisher(&err0)
//
// return e.executeIncludesColumnCallShard(ctx, tx, index, c, shard, col)
// }
//
// Note we are tracking the returned err value of someFunc(). An option instead is to say
//
// defer finisher(nil)
//
// This means always Commit writes, ignoring if there were errors. This style
// is expected to be rare compared to the typical
//
// defer finisher(&err0)
//
// invocation, where err0 is your return from the enclosing function error.
// If the Tx is local and not a part of a group, then the finisher
// consults that error to decides whether to Commit() or Rollback().
//
// If instead the Tx becomes part of a group, then the local finisher() is
// always a no-op, in deference to the Qcx.Finish()
// or Qcx.Abort() calls.
//
// Take care the finisher(&err) is capturing the address of the
// enclosing function's err and that it has not been shadowed
// locally by another _, err := f() call. For this reason, it can
// be clearer (and much safer) to rename the enclosing functions 'err' to 'err0',
// to make it clear we are referring to the first and final error.
//
func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error)) {
qcx.mu.Lock()
defer qcx.mu.Unlock()
// roaring uses finer grain, a file per fragment rather than
// db per shard. So we can't re-use the readTx. Moreover,
// roaring Tx are No-ops anyway, so just give it a new Tx
// everytime.
if qcx.isRoaring {
return qcx.Txf.NewTx(o), NoopFinisher
}
// note: write Tx were re-using Tx across different goroutines,
// which lmdb will not be pleased with. For reads this
// should be okay, as the docs say
// "If you want to pass read-only transactions across threads,
// you can use the MDB_NOTLS option on the environment."
// -- http://www.lmdb.tech/doc/starting.html
// and we always use lmdb.NoTLS as the lmdb-go bindings ensure this.
//
// So we make ALL write transactions local, and never reuse them
// below.
//
// *However* there is one exception: when we have set RequiredForAtomicWriteTx
// for the importing of an AtomicRequest, then we must use that
// our single RequiredForAtomicWriteTx for all writes until it
// is cleared. This one is kept separately from the read TxGroup.
//
if o.Write && qcx.RequiredForAtomicWriteTx != nil {
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
panic(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
panic("o.Index annot be nil")
}
if ro.Index == nil {
panic("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
panic(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
return *qcx.RequiredForAtomicWriteTx, NoopFinisher
}
if !o.Write && qcx.Grp != nil {
// read, with a group in place.
finisher = func(perr *error) {}
already := false
tx, already = qcx.Grp.AlreadyHaveTx(o)
if already {
return
}
o.Group = qcx.Grp
tx = qcx.Txf.NewTx(o)
qcx.Grp.AddTx(tx)
return
}
// non atomic writes or not grouped reads
tx = qcx.Txf.NewTx(o)
if o.Write {
finisherDone := false
finisher = func(perr *error) {
if finisherDone {
return
}
finisherDone = true // only Commit once.
// so defer finisher(nil) means always Commit writes, ignoring
// the enclosing functions return status.
if perr == nil || *perr == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}
} else {
// read-only txn
finisher = func(perr *error) {
tx.Rollback()
}
}
return
}
// StartAtomicWriteTx allocates a Tx and stores it
// in qcx.RequiredForAtomicWriteTx. All subsequent writes
// to this shard/index will re-use it.
func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
if !o.Write {
panic("must have o.Write true")
}
qcx.mu.Lock()
defer qcx.mu.Unlock()
if qcx.RequiredForAtomicWriteTx == nil {
// new Tx needed
tx := qcx.Txf.NewTx(o)
qcx.RequiredForAtomicWriteTx = &tx
o := tx.Options()
qcx.RequiredTxo = &o
return
}
// re-using existing
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
panic(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
panic("o.Index annot be nil")
}
if ro.Index == nil {
panic("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
panic(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
}
func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) {
if tx == nil || NilInside(tx) {
panic("cannot set nil tx in SetRequiredForAtomicWriteTx")
}
qcx.mu.Lock()
qcx.RequiredForAtomicWriteTx = &tx
o := tx.Options()
qcx.RequiredTxo = &o
qcx.mu.Unlock()
}
func (qcx *Qcx) ClearRequiredForAtomicWriteTx() {
qcx.mu.Lock()
qcx.RequiredForAtomicWriteTx = nil
qcx.RequiredTxo = nil
qcx.mu.Unlock()
}
func (qcx *Qcx) ListOpenTx() string {
return qcx.Grp.String()
}
// TxFactory abstracts the creation of Tx interface-level
// transactions so that RBF, or Badger, or Roaring-fragment-files, or several
// transactions so that RBF, or LMDB, or Roaring-fragment-files, or several
// of these at once in parallel, is used as the storage and transction layer.
type TxFactory struct {
typeOfTx string
types []txtype // blue-green split individually here
mu sync.Mutex // group protection
badgerDB *BadgerDBWrapper
lmDB *LMDBWrapper
rbfDB *RbfDBWrapper
roaringDB *RoaringStore
types []txtype // blue-green split individually here
dbsClosed bool // idemopotent CloseDB()
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
dbPerShard *DBPerShard
holder *Holder
blueGreenReg *blueGreenRegistry
}
func (f *TxFactory) Types() []txtype {
return f.types
}
// integer types for fast switch{}
@ -84,11 +391,22 @@ type txtype int
const (
noneTxn txtype = 0
roaringTxn txtype = 1 // these don't really have any transactions
badgerTxn txtype = 2
rbfTxn txtype = 3
lmdbTxn txtype = 4
rbfTxn txtype = 2
lmdbTxn txtype = 3
)
func (ty txtype) FileSuffix() string {
switch ty {
case roaringTxn:
return ""
case rbfTxn:
return "-rbfdb"
case lmdbTxn:
return "-lmdb"
}
panic(fmt.Sprintf("unkown txtype %v", int(ty)))
}
func (txf *TxFactory) NeedsSnapshot() (b bool) {
for _, ty := range txf.types {
switch ty {
@ -115,8 +433,6 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
switch s {
case RoaringTxn: // "roaring"
types = append(types, roaringTxn)
case BadgerTxn: // "badger"
types = append(types, badgerTxn)
case RBFTxn: // "rbf"
types = append(types, rbfTxn)
case LmdbTxn: // "lmdb"
@ -137,53 +453,16 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
// want to a fresh database, os.RemoveAll on dir/name ahead of time.
// We always store files in a subdir of dir. If we are having one
// database or many can depend on name.
func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
//vv("NewTxFactory called for txsrc '%v'; dir='%v'; name='%v'", txsrc, dir, name)
func NewTxFactory(txsrc string, holderDir string, holder *Holder) (f *TxFactory, err error) {
types := MustTxsrcToTxtype(txsrc)
f = &TxFactory{
types: types,
typeOfTx: txsrc,
roaringDB: NewRoaringStore(),
types: types,
typeOfTx: txsrc,
holder: holder,
blueGreenReg: newBlueGreenReg(),
}
for _, ty := range f.types {
switch ty {
case roaringTxn:
// no-op. these are just files in a directory
case badgerTxn:
// one, big, bad-ass badger for all data: the honeyBadger.
//
// Note that having a single Tx backing store for all indexes
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = DetectMemAccessPastTx
case rbfTxn:
path := dir + sep + "all-in-one-rbfdb"
f.rbfDB, err = globalRbfDBReg.openRbfDB(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new rbf db. path='%v'", path))
}
case lmdbTxn:
path := dir + sep + "all-in-one"
f.lmDB, err = globalLMDBReg.openLMDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new lmdb db. path='%v'", path))
}
}
}
f.dbPerShard = f.NewDBPerShard(types, holderDir)
return f, err
}
@ -194,6 +473,15 @@ type Txo struct {
Index *Index
Fragment *fragment
Shard uint64
dbs *DBShard
per *DBPerShard
Group *TxGroup
}
func (o Txo) String() string {
return fmt.Sprintf("Txo{Write:%v, Index:%v Shard:%v Group:%p}", o.Write, o.Index.name, o.Shard, o.Group)
}
func (f *TxFactory) TxType() string {
@ -205,60 +493,21 @@ func (f *TxFactory) TxTypes() []txtype {
}
func (f *TxFactory) DeleteIndex(name string) (err error) {
for _, ty := range f.types {
switch ty {
case roaringTxn:
// from holder.go:955, by default is already done there with os.RemoveAll()
case badgerTxn:
err = f.badgerDB.DeleteIndex(name)
case rbfTxn:
err = f.rbfDB.DeleteIndex(name)
case lmdbTxn:
err = f.lmDB.DeleteIndex(name)
default:
panic(fmt.Sprintf("unknown txtyp : '%v'", ty))
}
}
return
return f.dbPerShard.DeleteIndex(name)
}
func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
for _, ty := range f.types {
switch ty {
case roaringTxn:
err = f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
err = f.badgerDB.DeleteField(index, field, fieldPath)
case rbfTxn:
err = f.rbfDB.DeleteField(index, field, fieldPath)
case lmdbTxn:
err = f.lmDB.DeleteField(index, field, fieldPath)
default:
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", ty))
}
}
return
return f.dbPerShard.DeleteFieldFromStore(index, field, fieldPath)
}
func (f *TxFactory) DeleteFragmentFromStore(
index, field, view string, shard uint64, frag *fragment,
) (err error) {
return f.dbPerShard.DeleteFragment(index, field, view, shard, frag)
}
for _, ty := range f.types {
switch ty {
case roaringTxn:
err = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case badgerTxn:
err = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
err = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
case lmdbTxn:
err = f.lmDB.DeleteFragment(index, field, view, shard, frag)
default:
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", ty))
}
}
return
func (f *TxFactory) DumpAll() {
f.dbPerShard.DumpAll()
}
func (f *TxFactory) CloseIndex(idx *Index) error {
@ -273,22 +522,7 @@ func (f *TxFactory) CloseDB() (err error) {
return nil
}
f.dbsClosed = true
for _, ty := range f.types {
switch ty {
case roaringTxn:
// no-op
case badgerTxn:
err = f.badgerDB.Close()
case rbfTxn:
err = f.rbfDB.Close()
case lmdbTxn:
err = f.lmDB.Close()
default:
panic(fmt.Sprintf("unknown txtype: '%v'", ty))
}
}
return
return f.dbPerShard.Close()
}
var globalUseStatTx = false
@ -300,44 +534,179 @@ func init() {
}
}
// TxGroup holds a set of read and a set of write transactions
// that will en-mass have Rollback() (for the read set) and
// Commit() (for the write set) called on
// them when TxGroup.Finish() is invoked.
// Alternatively, TxGroup.Abort() will call Rollback()
// on all Tx group memebers.
type TxGroup struct {
mu sync.Mutex
fac *TxFactory
reads []Tx
writes []Tx
finished bool
all map[grpkey]Tx
}
type grpkey struct {
write bool
index string
shard uint64
}
func mustHaveIndexShard(o *Txo) {
if o.Index == nil || o.Index.name == "" {
panic("index must be set on Txo")
}
}
func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) {
mustHaveIndexShard(&o)
g.mu.Lock()
defer g.mu.Unlock()
key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard}
tx, already = g.all[key]
return
}
func (g *TxGroup) String() (r string) {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.reads) == 0 && len(g.writes) == 0 {
return "<empty-TxGroup>"
}
i := 0
r += "\n"
for _, tx := range g.reads {
r += fmt.Sprintf("[%v]read: _sn_ %v %v, \n", i, tx.Sn(), tx.Options())
i++
}
for _, tx := range g.writes {
r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, tx.Sn(), tx.Options())
i++
}
return
}
// NewTxGroup
func (f *TxFactory) NewTxGroup() (g *TxGroup) {
g = &TxGroup{
fac: f,
all: make(map[grpkey]Tx),
}
return
}
// AddTx adds tx to the group.
func (g *TxGroup) AddTx(tx Tx) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
panic("in TxGroup.Finish(): TxGroup already finished")
}
if NilInside(tx) {
panic("Cannot add nil Tx to TxGroup")
}
if tx.Readonly() {
g.reads = append(g.reads, tx)
} else {
g.writes = append(g.writes, tx)
}
o := tx.Options()
mustHaveIndexShard(&o)
key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard}
prior, ok := g.all[key]
if ok {
panic(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx))
}
g.all[key] = tx
}
// Finish commits the write tx and calls Rollback() on
// the read tx contained in the group. Either Abort() or Finish() must
// be called on the TxGroup exactly once.
func (g *TxGroup) FinishGroup() (err error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
panic("in TxGroup.Finish(): TxGroup already finished")
}
g.finished = true
for i, tx := range g.writes {
_ = i
err0 := tx.Commit()
if err0 != nil {
if err == nil {
err = err0 // keep the first error, but Commit them all.
}
}
}
for _, r := range g.reads {
r.Rollback()
}
return
}
// Abort calls Rollback() on all the group Tx, and marks
// the group as finished. Either Abort() or Finish() must
// be called on the TxGroup exactly once.
func (g *TxGroup) AbortGroup() {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
// defer Abort() probably gets here often by default, just ignore.
return
}
g.finished = true
for _, r := range g.reads {
r.Rollback()
}
for _, tx := range g.writes {
tx.Rollback()
}
}
func (f *TxFactory) NewTx(o Txo) (txn Tx) {
f.mu.Lock() // deadlock here
defer f.mu.Unlock()
defer func() {
if globalUseStatTx {
txn = newStatTx(txn)
}
}()
indexName := ""
if o.Index != nil {
indexName = o.Index.name
}
var txns []Tx
for _, ty := range f.types {
switch ty {
case roaringTxn:
txns = append(txns, &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment})
case badgerTxn:
//tx := NewMultiTxWithIndex(o.Write, o.Index)
//txns = append(txns, tx
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
txns = append(txns, btx)
case rbfTxn:
//tx := NewMultiTxWithIndex(o.Write, o.Index)
tx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
txns = append(txns, tx)
case lmdbTxn:
txns = append(txns, f.lmDB.NewLMDBTx(o.Write, indexName, o.Fragment))
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
if o.Fragment != nil {
if o.Fragment.index != indexName {
panic(fmt.Sprintf("inconsistent NewTx request: o.Fragment.index='%v' but indexName='%v'", o.Fragment.index, indexName))
}
if o.Fragment.shard != o.Shard {
panic(fmt.Sprintf("inconsistent NewTx request: o.Fragment.shard='%v' but o.Shard='%v'", o.Fragment.shard, o.Shard))
}
}
if len(txns) > 1 {
return newBlueGreenTx(txns[0], txns[1], f.idx)
// look up in the collection of open databases, and get our
// per-shard database. Opens a new one if needed.
dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index)
panicOn(err)
o.dbs = dbs // our specific database per shard.
o.per = f.dbPerShard // for top level debug Dumps
tx, err := dbs.NewTx(o.Write, indexName, o)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return txns[0]
return tx
}
func (ty txtype) String() string {
@ -346,8 +715,6 @@ func (ty txtype) String() string {
return "noneTxn"
case roaringTxn:
return "roaringTxn"
case badgerTxn:
return "badgerTxn"
case rbfTxn:
return "rbfTxn"
case lmdbTxn:
@ -356,15 +723,6 @@ func (ty txtype) String() string {
panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
}
// StringifiedBadgerKeys displays the keys visible in BadgerDB for the idx *Index.
// If optionalUseThisTx is nil, it will start a new read-only transaction to
// do this query. Otherwise it will piggy back on the provided transaction.
// Hence to view uncommited keys, you must provide in optionalUseThisTx the
// Tx in which they have been added.
func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
return idx.Txf.badgerDB.StringifiedBadgerKeys(optionalUseThisTx)
}
// fragmentSpecFromRoaringPath takes a path releative to the
// index directory, not including the name of the index itself.
// The path should not start with the path separator sep ('/' or '\\') rune.
@ -425,7 +783,7 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) {
return "" // new convention that empty database => empty string returned.
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
r += "]\n all-in-blake3:" + Blake3sum16([]byte(r)) + "\n"
return "roaring-" + r
}
@ -503,17 +861,18 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
for citer.Next() {
ckey, ct := citer.Value()
by := containerToBytes(ct)
hash := blake3sum16(by)
hash := Blake3sum16(by)
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
var srbm string
if !hashOnly {
srbm = bitmapAsString(rbm)
srbm = BitmapAsString(rbm)
}
bkey := string(txkey.Key(index, field, view, shard, ckey))
bkey := txkey.ToString(txkey.Key(index, field, view, shard, ckey))
n := ct.N()
hotbits += int(n)

102
txfactory_internal_test.go Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"os"
"testing"
"time"
"github.com/glycerine/lmdb-go/lmdb"
)
func Test_TxFactory_Qcx_query_context(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src == "rbf" || src == "lmdb" {
// ok
} else {
t.Skip("this test only for lmdb and rbf")
}
shard := uint64(0)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "")
defer f.Clean(t)
tx.Rollback()
barrier := lmdb.NewBarrier()
defer barrier.Close()
done := make(chan bool)
setter := func(k int) {
for i := 0; ; i++ {
barrier.WaitAtGate(0)
select {
case <-done:
return
default:
}
// add to the group txn on the txf.
qcx := idx.Txf.NewQcx()
tx, finisher := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard})
// Set bits on the fragment.
if _, err := f.setBit(tx, 120, 1); err != nil {
panic(err)
} else if _, err := f.setBit(tx, 120, 6); err != nil {
panic(err)
} else if _, err := f.setBit(tx, 121, 0); err != nil {
panic(err)
}
// should have two containers set in the fragment.
// Verify counts on rows.
if n := f.mustRow(tx, 120).Count(); n != 2 {
panic(fmt.Sprintf("unexpected count: %d", n))
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
panic(fmt.Sprintf("unexpected count: %d", n))
}
finisher(nil) // hit the write tx.Commit path
// commit the change, and verify it is still there
panicOn(qcx.Finish())
tx, finread := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
if n := f.mustRow(tx, 120).Count(); n != 2 {
panic(fmt.Sprintf("unexpected count (reopen): %d", n))
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
panic(fmt.Sprintf("unexpected count (reopen): %d", n))
}
finread(nil) // no-op on reads that are in a group, so must qcx.Abort() to stop them.
qcx.Abort()
}
}
N := 1000
for i := 0; i < N; i++ {
go setter(i)
}
time.Sleep(time.Second * 1)
close(done)
// allow all goro to finish before Closing the lmdb.env, otherwise
// we will crash as the goroutines making Tx will try to use the env
// after it is closed. It can take quite a while.
// one writer might be blocking the other... so ask for only N-1 at first.
//barrier.BlockUntil(N - 1)
barrier.BlockUntil(N)
//barrier.UnblockReaders()
//time.Sleep(1 * time.Second)
}

265
txkey/txkey.go
View file

@ -13,131 +13,151 @@
// limitations under the License.
// Package txkey consolidates in one place the use of keys to index into our
// various storage/txn back-ends. Databases badgerDB and rbfDB both use it,
// various storage/txn back-ends. Databases LMDB and rbfDB both use it,
// so that debug Dumps are comparable.
package txkey
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
)
// Key produces the bytes that we use as a key to query the storage/tx engine.
// The roaringContainerKey argument is a container key into a roaring Container.
// Output examples:
// The roaringContainerKey argument to Key() is a container key into a roaring Container.
// The return value from Key() is constructed as follows:
//
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@00000000000000000000" // smallest container-key
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615" // largest container-key (math.MaxUint64)
// ~index%field;view:shard<ckey#
//
// where shard and ckey are always exactly 8 bytes, uint64 big-endian encoded.
//
// Keys always start with either '~' or '>'. Keys always end with '#'.
// Keys always contain exactly one each of '%', ';', ':' and '<', in that order.
// The index is between the first byte and the '%'. It must be at least 1 byte long.
// The field is between the '%' and the ';'. It must be at least 1 byte long.
// The view is between the ';' and the ':'. It must be at least 1 byte long.
// The shard is the 8 bytes between the ':' and the '<'.
// The ckey is the 8 bytes between the '<' and the '#'.
// The Prefix of a key ends at, and includes, the '<'. It is at least 16 bytes long.
// The index, field, and view are not allowed to contain these reserved bytes:
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable
// base-64 encoding.
//
// The shortest possible key is 25 bytes. It would be laid out like this:
// ~i%f;v:12345678<12345678#
// 1234567890123456789012345
//
// keys starting with '~' are regular value keys.
// keys starting with '>' are symlink keys.
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
//
func Key(index, field, view string, shard uint64, roaringContainerKey uint64) []byte {
// The %020d which adds zero padding up to 20 runes is required to
// allow the textual sort to accurately
// reflect a numeric sort order. This is because, as a string,
// math.MaxUint64 is 20 bytes long.
// Example of such a Key with a container-key that is math.MaxUint64:
// ...........................................12345678901234567890
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615
func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) {
prefix := Prefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
var ckeyPartExpected = []byte(";ckey@")
// MustValidatekey will panic on a bad Key with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
return append(prefix, ckey[:]...)
}
// ShardFromKey key example: index/field;view:shard<ckey
// n-9 n-1
// ... : 01234567 < 01234567 #
// shard ckey
func ShardFromKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
by := bkey[:n-27]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
// ckey is always exactly 8 bytes long
// shard is always exactly 8 bytes long
shard = binary.BigEndian.Uint64(bkey[(n - 18):(n - 10)])
return
}
func ShardFromPrefix(prefix []byte) (shard uint64) {
n := len(prefix)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@ -> idx:'i';fld:'f';vw:'standard';shd:'1
by := prefix[:n-7]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad prefix='%v' did not have single quote to being shard decoding", string(prefix)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
shard = binary.BigEndian.Uint64(prefix[(n - 9):(n - 1)])
return
}
// KeyAndPrefix returns the equivalent of Key() and Prefix() calls.
func KeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = Prefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey, prefix
var ckey [9]byte
binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey)
ckey[8] = byte('#')
key = append(prefix, ckey[:]...)
return
}
var _ = KeyAndPrefix // keep linter happy
// KeyExtractContainerKey extracts the containerKey from bkey.
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
MustValidateKey(bkey)
// The zero padding means that the container-key is always the last 20 bytes of the bkey.
//
// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
// ends in 'key@' rather than a full key that has 'key@00000000000000000001' (for example)
// at the end. The ParseUint call below will fail in that case.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 20 {
panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', too short!", string(bkey)))
if n < 25 {
panic(fmt.Sprintf("bkey too short, must have at least 25 bytes: '%v'", string(bkey)))
}
last := bkey[n-20:] // Key() and Prefix() always return more than 20 rune []byte.
var err error
containerKey, err = strconv.ParseUint(string(last), 10, 64) // has to be the container key
if err != nil {
panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', could not convert last 20 bytes ('%v') to a unit64: '%v'", string(bkey), string(last), err))
typ := bkey[0]
if typ != '~' && typ != '>' {
panic(fmt.Sprintf("bkey did not start with '~' for value nor '>' for symlink: '%v'", string(bkey)))
}
if bkey[n-10] != '<' {
panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey)))
}
if bkey[n-19] != ':' {
panic(fmt.Sprintf("bkey did not have '<' at 18 bytes from the end: '%v'", string(bkey)))
}
if bkey[n-1] != '#' {
panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey)))
}
}
// KeyExtractContainerKey extracts the containerKey from bkey.
// key example: index/field;view:shard<ckey
// shortest: =i%f;v:12345678<12345678#
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
n := len(bkey)
MustValidateKey(bkey)
containerKey = binary.BigEndian.Uint64(bkey[(n - 9):(n - 1)])
return
}
func AllShardPrefix(index, field, view string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
func AllShardPrefix(index, field, view string) (r []byte) {
r = make([]byte, 0, 64)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, ':')
return
}
// Prefix returns everything from Key up to and
// including the '@' fune in a Key. The prefix excludes the roaring container key itself.
// including the '<' byte in a Key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with Key() and KeyExtractContainerKey().
func Prefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
func Prefix(index, field, view string, shard uint64) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
r = append(r, []byte(view)...)
r = append(r, ':')
var sh [8]byte
binary.BigEndian.PutUint64(sh[:], shard)
r = append(r, sh[:]...)
r = append(r, '<')
return
}
// IndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
@ -145,22 +165,79 @@ func Prefix(index, field, view string, shard uint64) []byte {
//
// The full name of the index must be provided, no partial index names will work.
//
// The provided key is terminated by `';` and so DeleteIndex("i") will not delete the index "i2".
// The returned prefix is terminated by '%' and so DeleteIndex("i") will not delete the index "i2".
//
func IndexOnlyPrefix(indexName string) []byte {
return []byte(fmt.Sprintf("idx:'%v';", indexName))
func IndexOnlyPrefix(indexName string) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(indexName)...)
r = append(r, '%')
return
}
// same for deleting a whole field.
func FieldPrefix(index, field string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
func FieldPrefix(index, field string) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
r = append(r, []byte(index)...)
r = append(r, '%')
r = append(r, []byte(field)...)
r = append(r, ';')
return
}
// PrefixFromKey key example: index/field;view:shard<ckey
// n-9 n-1
// ... : 01234567 < 01234567 #
// shard ckey
func PrefixFromKey(bkey []byte) (prefix []byte) {
MustValidateKey(bkey)
beg := bytes.LastIndex(bkey, []byte("@"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have '@' extract prefix", string(bkey)))
}
return bkey[:beg+1]
n := len(bkey)
return bkey[:(n - 9)]
}
func ToString(bkey []byte) (r string) {
index, field, view, shard, ckey := Split(bkey)
return fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@%020d", index, field, view, shard, ckey)
}
func PrefixToString(pre []byte) (r string) {
index, field, view, shard := SplitPrefix(pre)
return fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';", index, field, view, shard)
}
func Split(bkey []byte) (index, field, view string, shard, ckey uint64) {
ckey = KeyExtractContainerKey(bkey)
n := len(bkey)
index, field, view, shard = SplitPrefix(bkey[:(n - 9)])
return
}
// full key: ~index%field;view:shard<ckey#
// prefix : ~index%field;view:shard<
func SplitPrefix(pre []byte) (index, field, view string, shard uint64) {
n := len(pre)
shard = binary.BigEndian.Uint64(pre[(n - 9):(n - 1)])
// prefix: =index%field;view:shard<
goal := byte('%')
beg := 1
for i := 1; i < n; i++ {
c := pre[i]
switch goal {
case '%':
if c == goal {
index = string(pre[beg:i])
beg = i + 1
goal = byte(';')
}
case ';':
if c == goal {
field = string(pre[beg:i])
beg = i + 1
view = string(pre[beg:(n - 10)])
return
}
}
}
panic(fmt.Sprintf("malformed prefix '%v' / '%#v', could not Split", string(pre), pre))
}

59
txkey/txkey_test.go
View file

@ -16,6 +16,7 @@ package txkey
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"testing"
@ -28,25 +29,27 @@ func Test_KeyPrefix(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
// prefix example: i%f;v:12345678<
prefix := Prefix(index, field, view, shard)
//fmt.Printf("needle = '%v'\n", string(needle))
//fmt.Printf("prefix = '%v'\n", string(prefix))
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("Prefix() output '%v'was not a prefix of Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("Prefix() output '%v'was 20 characters shorter than Key() '%v'", string(needle), string(prefix)))
npre := len(prefix)
nneed := len(needle)
if npre+9 != nneed {
panic(fmt.Sprintf("Prefix() output len %v '%v' was not 9 characters shorter than Key() len %v '%v'", npre, string(prefix), nneed, string(needle)))
}
// validate assumption that KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
shouldNotParse := "12345123451234';key<"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
@ -65,13 +68,18 @@ func Test_KeyPrefix(t *testing.T) {
}
func Test_ShardFromKey(t *testing.T) {
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
key := []byte("~i%f;v:12345678<12345678#")
binary.BigEndian.PutUint64(key[7:15], 1)
if ShardFromKey(key) != 1 {
panic("problem")
}
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
binary.BigEndian.PutUint64(key[7:15], 0)
if ShardFromKey(key) != 0 {
panic("problem")
}
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
binary.BigEndian.PutUint64(key[7:15], 18446744073709551615)
if ShardFromKey(key) != 18446744073709551615 {
panic("problem")
}
@ -83,14 +91,14 @@ func Test_ShardFromKey(t *testing.T) {
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
ShardFromKey([]byte("~i%f;v:12345678<1234567#"))
}()
}
func Test_PrefixFromKey(t *testing.T) {
k := []byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")
x := []byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@")
k := []byte("~i%f;v:12345678<12345678#")
x := []byte("~i%f;v:12345678<")
pre := PrefixFromKey(k)
if !bytes.Equal(pre, x) {
nx := len(x)
@ -106,3 +114,28 @@ func Test_PrefixFromKey(t *testing.T) {
panic(fmt.Sprintf("expected:\n%v\n, observed:\n%v\n", string(x), string(pre)))
}
}
func Test_Split(t *testing.T) {
bkey := []byte("~i%f;v:12345678<12345678#")
var xshard uint64 = 18446744073709551615
var xckey uint64 = 43
binary.BigEndian.PutUint64(bkey[7:15], xshard)
binary.BigEndian.PutUint64(bkey[16:24], xckey)
i, f, v, shard, ckey := Split(bkey)
if i != "i" {
panic("wrong index")
}
if f != "f" {
panic("wrong field")
}
if v != "v" {
panic("wrong view")
}
if shard != xshard {
panic("wrong shard")
}
if ckey != xckey {
panic("wrong ckey")
}
}

239
util.go Normal file
View file

@ -0,0 +1,239 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// util.go: a place for generic, reusable utilities.
import (
"fmt"
"io/ioutil"
"net"
"reflect"
"sort"
"unsafe"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// LeftShifted16MaxContainerKey is 0xffffffffffff0000. It is similar
// to the roaring.maxContainerKey 0x0000ffffffffffff, but
// shifted 16 bits to the left so its domain is the full [0, 2^64) bit space.
// It is used to match the semantics of the roaring.OffsetRange() API.
// This is the maximum endx value for Tx.OffsetRange(), because the lowbits,
// as in the roaring.OffsetRange(), are not allowed to be set.
// It is used in Tx.RoaringBitamp() to obtain the full contents of a fragment
// from a call from tx.OffsetRange() by requesting [0, LeftShifted16MaxContainerKey)
// with an offset of 0.
const LeftShifted16MaxContainerKey = uint64(0xffffffffffff0000) // or math.MaxUint64 - (1<<16 - 1), or 18446744073709486080
// NilInside checks if the provided iface is nil or
// contains a nil pointer, slice, array, map, or channel.
func NilInside(iface interface{}) bool {
if iface == nil {
return true
}
switch reflect.TypeOf(iface).Kind() {
case reflect.Ptr, reflect.Slice, reflect.Array, reflect.Map, reflect.Chan:
return reflect.ValueOf(iface).IsNil()
}
return false
}
// GetAvailPort asks the OS for an unused port.
// There's a race here, where the port could be grabbed by someone else
// before the caller gets to Listen on it, but we are only using
// it to find a random port for the test hang debugging.
// Moreover, in practice such races are rare. Just ask for
// it again if the port is taken.
// Uses net.Listen("tcp", ":0") to determine a free port, then
// releases it back to the OS with Listener.Close().
func GetAvailPort() int {
l, _ := net.Listen("tcp", ":0")
r := l.Addr()
l.Close()
return r.(*net.TCPAddr).Port
}
//////////////////////////////////
// helper utility functions
func highbits(v uint64) uint64 { return v >> 16 }
func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
func toArray16(a []byte) []uint16 {
return (*[4096]uint16)(unsafe.Pointer(&a[0]))[: len(a)/2 : len(a)/2]
}
func toArray64(a []byte) []uint64 {
return (*[1024]uint64)(unsafe.Pointer(&a[0]))[:1024:1024]
}
func toInterval16(a []byte) []roaring.Interval16 {
return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4]
}
func sliceToMap(slc []uint64) (m map[uint64]bool) {
m = make(map[uint64]bool)
for _, v := range slc {
m[v] = true
}
return
}
// return A - B
func mapDiff(mapA, mapB map[uint64]bool) (r []int) {
for a := range mapA {
_, ok := mapB[a]
if !ok {
r = append(r, int(a))
}
}
return
}
func asInts(a []uint64) (r []int) {
r = make([]int, len(a))
for i, v := range a {
r[i] = int(v)
}
return
}
func containerAsString(ckey uint64, rc *roaring.Container) (r string) {
rbm := roaring.NewBitmap()
rbm.Containers.Put(ckey, rc)
return BitmapAsString(rbm)
}
var _ = containerAsString // happy linter
func roaringBitmapDiff(a, b *roaring.Bitmap) error {
nA := a.Count()
nB := b.Count()
slcA := a.Slice()
slcB := b.Slice()
mapA := sliceToMap(slcA)
mapB := sliceToMap(slcB)
AminusB := mapDiff(mapA, mapB)
BminusA := mapDiff(mapB, mapA)
sort.Ints(AminusB)
sort.Ints(BminusA)
res := fmt.Sprintf("nA = %v; nB = %v;\n", nA, nB)
ndiff := 0
if nA != nB {
ndiff++
}
if len(AminusB) > 0 {
res += fmt.Sprintf("==> AminusB = (len %v) '%#v'; ", len(AminusB), AminusB)
ndiff++
}
if len(BminusA) > 0 {
res += fmt.Sprintf("\n==> BminusA = (len %v) '%#v'; ", len(BminusA), BminusA)
ndiff++
}
if ndiff == 0 {
return nil
}
res += fmt.Sprintf("\n ==> A = '%#v'\n ==> B = '%#v'", asInts(slcA), asInts(slcB))
return errors.New(res)
}
func dirAsString(path string) (r string) {
r = fmt.Sprintf("dump of directory '%v':\n", path)
files, err := ioutil.ReadDir(path)
panicOn(err)
for _, f := range files {
r += f.Name() + "\n"
}
return r
}
var _ = dirAsString // happy linter
var _ = zeroKeyContainerAsString // happy linter
// for debugging
func zeroKeyContainerAsString(ct *roaring.Container) (r string) {
cts := roaring.NewSliceContainers()
cts.Put(0, ct)
rbm := &roaring.Bitmap{Containers: cts}
r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + BitmapAsString(rbm)
return
}
var containerTypeNames = map[byte]string{
roaring.ContainerArray: "array",
roaring.ContainerBitmap: "bitmap",
roaring.ContainerRun: "run",
}
func BitmapAsString(rbm *roaring.Bitmap) (r string) {
r = "c("
slc := rbm.Slice()
width := 0
s := ""
for _, v := range slc {
if width == 0 {
s = fmt.Sprintf("%v", v)
} else {
s = fmt.Sprintf(", %v", v)
}
width += len(s)
r += s
if width > 70 {
r += ",\n"
width = 0
}
}
if width == 0 && len(r) > 2 {
r = r[:len(r)-2]
}
return r + ")"
}
// fromArray16 converts to an 8KB page
func fromArray16(a []uint16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 4096 {
panic(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
}
// fromArray64 converts to an 8KB page
func fromArray64(a []uint64) []byte {
if len(a) == 0 {
return []byte{}
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192]
}
// fromInterval16 converts to 8KB page
func fromInterval16(a []roaring.Interval16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 2048 {
panic(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
}

45
utils_internal_test.go
View file

@ -15,6 +15,7 @@
package pilosa
import (
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
@ -23,10 +24,40 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pkg/errors"
)
// utilities used by tests
// mustAddR is a helper for calling roaring.Container.Add() in tests to
// keep the linter happy that we are checking the error.
func mustAddR(changed bool, err error) {
panicOn(err)
}
// mustRemove is a helper for calling Tx.Remove() in tests to
// keep the linter happy that we are checking the error.
func mustRemove(changeCount int, err error) {
panicOn(err)
}
func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte {
b := roaring.NewBitmap()
changed := b.DirectAddN(bitsToSet...)
n := len(bitsToSet)
if changed != n {
panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n))
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
}
return buf.Bytes()
}
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewTestCluster(tb testing.TB, n int) *cluster {
path, err := testhook.TempDir(tb, "pilosa-cluster-")
@ -136,13 +167,12 @@ func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *tim
}
if err := func() error {
tx, err := c.holder.BeginTx(writable, c.holder.indexes[f.index])
idx := c.holder.indexes[f.index]
shard := colID / ShardWidth
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
if tx != nil {
defer tx.Rollback()
}
if err != nil {
return err
}
if _, err := f.SetBit(tx, rowID, colID, x); err != nil {
return err
@ -235,8 +265,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
}
// holder
h := NewHolder(DefaultPartitionN)
h.Path = path
h := NewHolder(path, nil)
// cluster
c := newCluster()
@ -459,11 +488,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
// there will be two -badgerdb directories/databases, we need to copy
// from src to dest the fragment. This simulates sending the fragment over the network.
srcIdx := srcCluster.holder.Index(src.Index)
srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment})
srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard})
destIdx := destCluster.holder.Index(src.Index)
desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment})
desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment, Shard: destFragment.shard})
citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
panicOn(err)

103
view.go
View file

@ -70,6 +70,8 @@ type view struct {
// newView returns a new instance of View.
func newView(holder *Holder, path, index, field, name string, fieldOptions FieldOptions) *view {
panicOn(validateName(name))
return &view{
path: path,
index: index,
@ -108,11 +110,13 @@ func newView(holder *Holder, path, index, field, name string, fieldOptions Field
// return that bitmap, and set knownShardsCopied to 1, but we rarely modify
// the list.
func (v *view) addKnownShard(shard uint64) {
v.notifyIfNewShard(shard)
if atomic.LoadUint32(&v.knownShardsCopied) == 1 {
v.knownShards = v.knownShards.Clone()
atomic.StoreUint32(&v.knownShardsCopied, 0)
}
_, _ = v.knownShards.Add(shard)
_, err := v.knownShards.Add(shard)
panicOn(err)
}
// removeKnownShard removes a known shard from v. See the notes on addKnownShard.
@ -168,14 +172,12 @@ var workQueue = make(chan struct{}, runtime.NumCPU()*2)
// replaces v.openFragments() with Tx generic code.
func (v *view) openFragmentsInTx() error {
tx := v.idx.Txf.NewTx(Txo{Write: false, Index: v.idx})
defer tx.Rollback()
shards, err := tx.SliceOfShards(v.index, v.field, v.name, v.path)
// we think this is correct for dbpershard, but might be slower. TODO
shards, err := DBPerShardGetShardsForIndex(v.idx, v.path)
if err != nil {
return errors.Wrap(err, "SliceOfShards")
return errors.Wrap(err, "DBPerShardGetShardsForIndex()")
}
eg, ctx := errgroup.WithContext(context.Background())
var mu sync.Mutex
@ -213,7 +215,7 @@ shardLoop:
v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
mu.Lock()
v.fragments[frag.shard] = frag
v.addKnownShard(frag.shard)
v.addKnownShard(shard)
mu.Unlock()
return nil
})
@ -315,6 +317,7 @@ func (v *view) recalculateCaches() {
func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
v.mu.Lock()
defer v.mu.Unlock()
// Find fragment in cache first.
if frag := v.fragments[shard]; frag != nil {
return frag, nil
@ -328,7 +331,6 @@ func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
frag.RowAttrStore = v.rowAttrStore
v.fragments[shard] = frag
v.notifyIfNewShard(shard)
v.addKnownShard(shard)
return frag, nil
}
@ -410,9 +412,16 @@ func (v *view) deleteFragment(shard uint64) error {
}
// row returns a row for a shard of the view.
func (v *view) row(tx Tx, rowID uint64) (*Row, error) {
func (v *view) row(txOrig Tx, rowID uint64) (*Row, error) {
row := NewRow()
for _, frag := range v.allFragments() {
tx := txOrig
if NilInside(tx) {
tx = v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard})
defer tx.Rollback()
}
fr, err := frag.row(tx, rowID)
if err != nil {
return nil, err
@ -426,59 +435,121 @@ func (v *view) row(tx Tx, rowID uint64) (*Row, error) {
}
// setBit sets a bit within the view.
func (v *view) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
func (v *view) setBit(txOrig Tx, rowID, columnID uint64) (changed bool, err error) {
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
var frag *fragment
frag, err = v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}
tx := txOrig
if NilInside(tx) {
tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}()
}
return frag.setBit(tx, rowID, columnID)
}
// clearBit clears a bit within the view.
func (v *view) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
func (v *view) clearBit(txOrig Tx, rowID, columnID uint64) (changed bool, err error) {
shard := columnID / ShardWidth
frag := v.Fragment(shard)
if frag == nil {
return false, nil
}
tx := txOrig
if NilInside(tx) {
tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}()
}
return frag.clearBit(tx, rowID, columnID)
}
// value uses a column of bits to read a multi-bit value.
func (v *view) value(tx Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) {
func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) {
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return value, exists, err
}
tx := txOrig
if NilInside(tx) {
tx = frag.idx.Txf.NewTx(Txo{Write: !writable, Index: frag.idx, Fragment: frag, Shard: frag.shard})
defer tx.Rollback()
}
return frag.value(tx, columnID, bitDepth)
}
// setValue uses a column of bits to set a multi-bit value.
func (v *view) setValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}
tx := txOrig
if NilInside(tx) {
tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}()
}
return frag.setValue(tx, columnID, bitDepth, value)
}
// clearValue removes a specific value assigned to columnID
func (v *view) clearValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
shard := columnID / ShardWidth
frag := v.Fragment(shard)
if frag == nil {
return false, nil
}
tx := txOrig
if NilInside(tx) {
tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
} else {
tx.Rollback()
}
}()
}
return frag.clearValue(tx, columnID, bitDepth, value)
}
// rangeOp returns rows with a field value encoding matching the predicate.
func (v *view) rangeOp(tx Tx, op pql.Token, bitDepth uint, predicate int64) (*Row, error) {
func (v *view) rangeOp(qcx *Qcx, op pql.Token, bitDepth uint, predicate int64) (_ *Row, err0 error) {
r := NewRow()
for _, frag := range v.allFragments() {
tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: v.idx, Shard: frag.shard})
defer finisher(&err0)
other, err := frag.rangeOp(tx, op, bitDepth, predicate)
if err != nil {
return nil, err

3
view_internal_test.go
View file

@ -34,8 +34,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
CacheSize: DefaultCacheSize,
}
h := NewHolder(DefaultPartitionN)
h.Path = path
h := NewHolder(path, nil)
// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
idx, err := h.createIndex(index, IndexOptions{})
testhook.Cleanup(tb, func() {

2
vprint.go
View file

@ -1,4 +1,4 @@
// home: https://github.com/glyerine/vprint
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//

175
vprint_test.go Normal file
View file

@ -0,0 +1,175 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package pilosa_test
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
// happy linter
var _ = stack
var _ = FileExists
var _ = FileSize
var _ = AlwaysPrintf
var _ = RFC3339MsecTz0
var _ = Caller
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}