scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #677 from molecula/lmdb_as_backend

Browse source 
add lmdb, Tx call stats, and prep for db/shard.
This commit is contained in:
tgruben 2020-08-17 18:43:57 -05:00 • committed by GitHub
commit d84ae88944
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 2948 additions and 1871 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -4,3 +4,4 @@ vendor
.protoc-gen-gofast
.DS_Store
build
*~

2
Dockerfile
View file

@ -5,7 +5,7 @@ ARG MAKE_FLAGS
COPY . pilosa
RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
RUN cd pilosa && make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
FROM alpine:3.9.4

2
Dockerfile-clustertests
View file

@ -8,7 +8,7 @@ LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& CGO_ENABLED=0 make install FLAGS="-a -mod=vendor"
&& make install FLAGS="-a -mod=vendor"
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

43
Makefile
View file

@ -184,6 +184,12 @@ topt-lmdb:
@echo " log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l
@echo " log.topt.lmdb red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l
topt-lmdb-race:
mv log.topt.lmdb log.topt.lmdb.prev || true
PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb-race
@echo " log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l
@echo " log.topt.lmdb-race red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@ -195,41 +201,64 @@ topt-race:
bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
mv log.bg-rr log.bg-rr.prev || true
set -o pipefail; PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr
@echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
@echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
mv log.bg.roar_bg log.bg.roar_bg.prev || true
set -o pipefail; PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
##PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
@echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
@echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
rbf-rr:
mv log.rbf-rr log.rbf-rr.prev || true
set -o pipefail; PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr
PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr
@echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
@echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
rr-rbf:
mv log.rr-rbf log.rr-rbf.prev || true
set -o pipefail; PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf
PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf
@echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
@echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
rbf-bg:
mv log.rbf-bg log.rbf-bg.prev || true
set -o pipefail; PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg
@echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
@echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
bg-rbf:
mv log.bg-rbf log.bg-rbf.prev || true
set -o pipefail; PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf
@echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
@echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
lm-rr:
mv log.lm-rr log.lm-rr.prev || true
PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr
@echo " log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l
@echo " log.lm-rr red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l
rr-lm:
mv log.rr-lm log.rr-lm.prev || true
PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-lm
@echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l
@echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l
bg-lm:
mv log.topt.bg-lm log.topt.bg-lm.prev || true
PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.bg-lm
@echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l
@echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l
lm-bg:
mv log.topt.lm-bg log.topt.lm-bg.prev || true
PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lm-bg
@echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l
@echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint

257
badger.go
View file

@ -20,6 +20,7 @@ import (
"io"
"io/ioutil"
"log"
"math"
"os"
"runtime"
"sort"
@ -31,7 +32,7 @@ import (
badger "github.com/dgraph-io/badger/v2"
badgeroptions "github.com/dgraph-io/badger/v2/options"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -345,7 +346,7 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
prefix := txpath.IndexOnlyPrefix(indexName)
prefix := txkey.IndexOnlyPrefix(indexName)
return w.DeletePrefix(prefix)
}
@ -438,6 +439,8 @@ type BadgerDBWrapper struct {
startStack string
DeleteEmptyContainer bool
writeBatch *badger.WriteBatch
}
// unprotectedListOpenTxAsString is a debugging helper.
@ -488,9 +491,7 @@ func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string, frag
doAllocZero: w.doAllocZero,
initialIndexName: initialIndexName,
DeleteEmptyContainer: w.DeleteEmptyContainer,
//initloc: "", // stack(),
}
return
}
@ -529,9 +530,6 @@ type BadgerTx struct {
frag *fragment
opcount int
// keep linter happy, comment out until needed again for debugging.
//initloc string // stack trace of where we were initially created.
doAllocZero bool
// for tracking txn boundary issues, track all the memory
@ -644,6 +642,7 @@ func (tx *BadgerTx) Commit() error {
if tx.doAllocZero {
tx.overWriteOurAllocs()
}
return err
}
@ -677,7 +676,7 @@ func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint
// you must use copy() to copy it to another byte slice.
// BUT here we are already inside the Txn.
bkey := txpath.Key(index, field, view, shard, ckey)
bkey := txkey.Key(index, field, view, shard, ckey)
tx.mu.Lock()
var item *badger.Item
item, err = tx.tx.Get(bkey)
@ -699,10 +698,41 @@ func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint
return
}
func (w *BadgerDBWrapper) NewWriteBatch() {
w.muDb.Lock()
defer w.muDb.Unlock()
if w.writeBatch != nil {
panic("must FlushWriteBatch() before calling NewWriteBatch()")
}
w.writeBatch = w.db.NewWriteBatch()
}
// Flush any remaining un-committed writes in progress.
func (w *BadgerDBWrapper) FlushWriteBatch() (err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
if w.writeBatch == nil {
panic("CommitWriteBatch error: no batch in progress")
}
err = w.writeBatch.Flush()
w.writeBatch = nil
return
}
// Cancel any remaining un-committed writes in progress.
func (w *BadgerDBWrapper) CancelWriteBatch() {
w.muDb.Lock()
defer w.muDb.Unlock()
if w.writeBatch == nil {
panic("CancelWriteBatch error: no batch in progress")
}
w.writeBatch.Cancel()
}
// PutContainer stores rc under the specified fragment and container ckey.
func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error {
bkey := txpath.Key(index, field, view, shard, ckey)
bkey := txkey.Key(index, field, view, shard, ckey)
var by []byte
ct := roaring.ContainerType(rc)
@ -719,13 +749,24 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
default:
panic(fmt.Sprintf("unknown roaring.Container type: %v", ct))
}
tx.writeCount++
sz := len(by) + len(bkey) + 2
tx.writeByteCount += sz
entry := badger.NewEntry(bkey, by).WithMeta(ct)
tx.Db.muDb.Lock()
if tx.Db.writeBatch != nil {
err := tx.Db.writeBatch.SetEntry(entry) // Will create txns as needed.
tx.Db.muDb.Unlock()
return err
}
tx.Db.muDb.Unlock()
tx.mu.Lock()
defer tx.mu.Unlock()
tx.writeCount++
sz := len(by) + len(bkey) + 2
tx.writeByteCount += sz
// The integration tests do large bit level loads that exceed 10MB.
// So we autocommit and start a new Txn if we are about to
// write too much into one Txn.
@ -736,15 +777,28 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
// However, emprirically we still get ErrTnTooBig when
// tx.writeByteCount=5884222; or when tx.writeCount=16197.
// So duck under both those thresholds by some margin.
if tx.writeCount > 10000 || tx.writeByteCount > 4000000 {
if tx.writeCount > 100 || tx.writeByteCount > 2000000 {
// avoid ErrTxnTooBig by commiting before going over the limits,
// because then we get a error: "Transaction Conflict. Please retry."
panicOn(tx.tx.Commit())
err := tx.tx.Commit()
panicOn(err)
// badger docs:
// `ErrConflict is returned when a transaction conflicts with another transaction. This can
// happen if the read rows had been updated concurrently by another transaction.
// ErrConflict = errors.New("Transaction Conflict. Please retry")`
//if err == badger.ErrConflict {
// problem is, we don't have the previous entry handy now.
//}
//if err != nil {
// ignore for now to get timings.
//panic(fmt.Sprintf("commit failed on bkey '%v': err '%v'", string(bkey), err))
//}
tx.tx = tx.Db.db.NewTransaction(tx.write)
//vv("NewBadgerTx write txn (p=%p) on gid=%v. b/c over thresholds writeCount=%v; writeByteCount=%v", tx.tx, curGID(), tx.writeCount, tx.writeByteCount)
tx.writeCount = 1
tx.writeByteCount = sz
}
entry := badger.NewEntry(bkey, by).WithMeta(ct)
err := tx.tx.SetEntry(entry)
// ErrTxnTooBig is returned if too many writes are fit into a single transaction.
@ -752,15 +806,30 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
// writes/deletes in the transaction exceeds a certain limit. In that case, it
// is best to commit the transaction and start a new transaction immediately."
//
if err == badger.ErrTxnTooBig {
panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; len(bkey)=%v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), len(bkey), tx.writeCount, tx.writeByteCount))
}
/*
if err == badger.ErrTxnTooBig {
err = tx.tx.Commit()
if err != nil {
panic(fmt.Sprintf("commit after TooBig failed on bkey '%v': err '%v'", string(bkey), err))
}
tx.tx = tx.Db.db.NewTransaction(tx.write)
//vv("NewBadgerTx write txn (p=%p) on gid=%v. b/c TooBig writeCount=%v; writeByteCount=%v", tx.tx, curGID(), tx.writeCount, tx.writeByteCount)
tx.writeCount = 1
tx.writeByteCount = sz
err = tx.tx.SetEntry(entry)
panicOn(err)
//panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; len(bkey)=%v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), len(bkey), tx.writeCount, tx.writeByteCount))
}
*/
return err
}
// RemoveContainer deletes the container specified by the shard and container key ckey
func (tx *BadgerTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error {
bkey := txpath.Key(index, field, view, shard, ckey)
bkey := txkey.Key(index, field, view, shard, ckey)
tx.mu.Lock()
err := tx.tx.Delete(bkey)
tx.mu.Unlock()
@ -769,80 +838,86 @@ func (tx *BadgerTx) RemoveContainer(index, field, view string, shard uint64, cke
// Add sets all the a bits hot in the specified fragment.
func (tx *BadgerTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
return tx.addOrRemove(index, field, view, shard, batched, false, a...)
}
// Remove clears all the specified a bits in the chosen fragment.
func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
const batched = false
const remove = true
return tx.addOrRemove(index, field, view, shard, batched, remove, a...)
}
func (tx *BadgerTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) {
// pure hack to match RoaringTx
defer func() {
if !batched {
if !remove && !batched {
if changeCount > 0 {
changeCount = 1
}
}
}()
// TODO: optimization: group 'a' elements into their containers,
// and then do all the Adds on that
// container at once, so we don't retrieve a container per bit.
// (maybe, for example, using ImportRoaringBits with clear=false).
for _, v := range a {
hi, lo := highbits(v), lowbits(v)
var rct *roaring.Container
rct, err = tx.Container(index, field, view, shard, hi)
panicOn(err)
if err != nil {
return 0, err
}
chng := false
// TODO optimization: set all the bits in the current container at once. group by container first.
rc1, chng := rct.Add(lo)
panicOn(err)
if chng {
changeCount++
}
if err != nil {
return changeCount, err
}
err = tx.PutContainer(index, field, view, shard, hi, rc1)
panicOn(err)
if len(a) == 0 {
return 0, nil
}
return
}
// Remove clears all the specified a bits in the chosen fragment.
func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
// have to sort, b/c input is not always sorted.
sort.Slice(a, func(i, j int) bool { return a[i] < a[j] })
// TODO: optimization: group 'a' elements into their containers,
// and then do all the Removes on that
// container at once, so we don't retrieve a container per bit.
// (maybe, for example, using ImportRoaringBits with clear=true).
for _, v := range a {
hi, lo := highbits(v), lowbits(v)
var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter.
var rc *roaring.Container
var hi uint64
var lo uint16
var rct *roaring.Container
rct, err = tx.Container(index, field, view, shard, hi)
panicOn(err)
if err != nil {
return 0, err
}
for i, v := range a {
hi, lo = highbits(v), lowbits(v)
if hi != lastHi {
// either first time through, or changed to a different container.
// do we need put the last updated container now?
if i > 0 {
// not first time through, write what we got.
if remove && (rc == nil || rc.N() == 0) {
err = tx.RemoveContainer(index, field, view, shard, lastHi)
panicOn(err)
} else {
err = tx.PutContainer(index, field, view, shard, lastHi, rc)
panicOn(err)
}
}
// get the next container
rc, err = tx.Container(index, field, view, shard, hi)
panicOn(err)
} // else same container, keep adding bits to rct.
chng := false
rc1, chng := rct.Remove(lo)
panicOn(err)
// rc can be nil before, and nil after, in both Remove/Add below.
// The roaring container add() and remove() methods handle this.
if remove {
rc, chng = rc.Remove(lo)
} else {
rc, chng = rc.Add(lo)
}
if chng {
changeCount++
}
if err != nil {
return changeCount, err
}
if rc1.N() == 0 {
lastHi = hi
}
// write the last updates.
if remove {
if rc == nil || rc.N() == 0 {
err = tx.RemoveContainer(index, field, view, shard, hi)
if err != nil {
return
}
panicOn(err)
} else {
err = tx.PutContainer(index, field, view, shard, hi, rc1)
err = tx.PutContainer(index, field, view, shard, hi, rc)
panicOn(err)
}
} else {
if rc == nil || rc.N() == 0 {
panic("there should be no way to have an empty bitmap AFTER an Add() operation")
}
err = tx.PutContainer(index, field, view, shard, hi, rc)
panicOn(err)
}
return
}
@ -852,7 +927,7 @@ func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64)
func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
lo, hi := lowbits(key), highbits(key)
bkey := txpath.Key(index, field, view, shard, hi)
bkey := txkey.Key(index, field, view, shard, hi)
tx.mu.Lock()
item, err := tx.tx.Get(bkey)
tx.mu.Unlock()
@ -873,7 +948,7 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := txpath.AllShardPrefix(index, field, view)
prefix := txkey.AllShardPrefix(index, field, view)
bi := NewBadgerIterator(tx, prefix)
defer bi.Close()
@ -886,7 +961,7 @@ func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (
for bi.Next() {
item := bi.it.Item()
key := item.Key()
shard := txpath.ShardFromKey(key)
shard := txkey.ShardFromKey(key)
if firstDone {
if shard != lastShard {
sliceOfShards = append(sliceOfShards, shard)
@ -912,10 +987,10 @@ func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (
func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
// needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000"
needle := txpath.Key(index, field, view, shard, firstRoaringContainerKey)
needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey)
// prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@"
prefix := txpath.Prefix(index, field, view, shard)
prefix := txkey.Prefix(index, field, view, shard)
bi := NewBadgerIterator(tx, prefix)
bi.Seek(needle)
@ -948,7 +1023,7 @@ type BadgerIterator struct {
}
// NewBadgerIterator creates an iterator on tx that will
// only return txpath.Keys that start with prefix.
// only return txkey.Keys that start with prefix.
func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
opts := badger.DefaultIteratorOptions
@ -1053,7 +1128,7 @@ func (bi *BadgerIterator) Value() (containerKey uint64, c *roaring.Container) {
panic("item was nil")
}
key := item.Key()
containerKey = txpath.KeyExtractContainerKey(key)
containerKey = txkey.KeyExtractContainerKey(key)
err := item.Value(func(v []byte) error {
c = bi.tx.toContainer(item.UserMeta(), v)
@ -1159,8 +1234,8 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) {
prefix := txpath.Prefix(index, field, view, shard)
seekto := txpath.Prefix(index, field, view, shard+1)
prefix := txkey.Prefix(index, field, view, shard)
seekto := txkey.Prefix(index, field, view, shard+1)
it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx.
defer it.Close()
@ -1319,15 +1394,15 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
off := highbits(offset)
hi0, hi1 := highbits(start), highbits(endx)
needle := txpath.Key(index, field, view, shard, hi0)
prefix := txpath.Prefix(index, field, view, shard)
needle := txkey.Key(index, field, view, shard, hi0)
prefix := txkey.Prefix(index, field, view, shard)
n2, pre2 := txpath.KeyAndPrefix(index, field, view, shard, hi0)
n2, pre2 := txkey.KeyAndPrefix(index, field, view, shard, hi0)
if string(n2) != string(needle) {
panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txpath.KeyAndPrefix not consitent with txpath.Key()", string(n2), string(needle)))
panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(n2), string(needle)))
}
if string(pre2) != string(prefix) {
panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txpath.KeyAndPrefix not consitent with txpath.Key()", string(pre2), string(prefix)))
panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(pre2), string(prefix)))
}
it := NewBadgerIterator(tx, prefix)
@ -1336,7 +1411,7 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
for ; it.it.ValidForPrefix(prefix); it.Next() {
item := it.it.Item()
bkey := item.Key()
k := txpath.KeyExtractContainerKey(bkey)
k := txkey.KeyExtractContainerKey(bkey)
// >= hi1 is correct b/c endx cannot have any lowbits set.
if uint64(k) >= hi1 {
@ -1606,7 +1681,7 @@ func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string)
}
// countBitsSet returns the number of bits set (or "hot") in
// the roaring container value found by the txpath.Key()
// the roaring container value found by the txkey.Key()
// formatted bkey.
func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) {
@ -1652,7 +1727,7 @@ func stringifiedBadgerKeysTx(tx *BadgerTx) (r string) {
item := it.Item()
bkey := item.Key()
key := string(bkey)
ckey := txpath.KeyExtractContainerKey(bkey)
ckey := txkey.KeyExtractContainerKey(bkey)
hash := ""
srbm := ""
err := item.Value(func(val []byte) error {
@ -1810,12 +1885,12 @@ func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error {
if err != nil {
return errors.Wrap(err, "removing directory")
}
prefix := txpath.FieldPrefix(index, field)
prefix := txkey.FieldPrefix(index, field)
return w.DeletePrefix(prefix)
}
func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
prefix := txpath.Prefix(index, field, view, shard)
prefix := txkey.Prefix(index, field, view, shard)
return w.DeletePrefix(prefix)
}

115
badger_test.go
View file

@ -1299,6 +1299,38 @@ func TestBadger_AutoCommit(t *testing.T) {
panicOn(err)
}
func TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index, nil)
containerKey := uint64(0)
// setup
bits := make([]uint64, 1024)
n := 0
for i := range bits {
bits[i] = ^uint64(0)
n += 64
}
rc := roaring.NewContainerBitmap(n, bits)
// if we go over 100K big writes, we should autocommit
// rather than panic.
for v := 0; v < 133444; v++ {
containerKey++
err := tx.PutContainer(index, field, view, shard, containerKey, rc)
panicOn(err)
}
err := tx.Commit()
panicOn(err)
}
func TestBadger_DeleteIndex(t *testing.T) {
// setup
@ -1626,3 +1658,86 @@ func TestMain(m *testing.M) {
//reportTestBadgersNeedingClose()
os.Exit(ret)
}
/*
func TestBadger_ConflictWriteWriteResolution(t *testing.T) {
// 1) when do we get write-write conflicts (probably different goroutines) but
// can we get them on different keys?
// 2) does having a lock registry that insures we are only ever writing
// different keys at once avoid write-write conflicts?
// 3) how should write-write conflicts be resolved?
// presumably just retying the write?
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ConflictWriteWriteResolution")
defer clean()
defer dbwrap.Close()
concur := 10
//bkey := []byte("a")
//by := []byte("value-for-a")
// read-loop:
for i := 0; i < concur*2; i++ {
go func() {
tx := dbwrap.NewBadgerTx(!writable, "", nil)
for j := 0; true; j++ {
bkey := []byte(fmt.Sprintf("key-for-a j=%v", j))
//by := []byte(fmt.Sprintf("value-for-a j=%v", -1))
_, err := tx.tx.Get(bkey)
if err != badger.ErrKeyNotFound {
panicOn(err)
}
if j%10 == 0 {
vv("committing after 10, gid=%v", curGID())
tx.Rollback()
tx = dbwrap.NewBadgerTx(!writable, "", nil)
}
}
}()
}
// write-loop:
for i := 0; i < concur; i++ {
go func() {
tx := dbwrap.NewBadgerTx(writable, "", nil)
for j := 0; true; j++ {
bkey := []byte(fmt.Sprintf("key-for-a j=%v", j))
by := []byte(fmt.Sprintf("value-for-a j=%v", j))
entry := badger.NewEntry(bkey, by)
err := tx.tx.SetEntry(entry)
panicOn(err)
if j%5 == 0 {
panicOn(tx.tx.Delete(bkey))
}
_, err = tx.tx.Get(bkey)
if err != badger.ErrKeyNotFound {
panicOn(err)
}
if j%10 == 0 {
///vv("committing after 10, gid=%v", curGID())
err = tx.tx.Commit()
panicOn(err)
tx = dbwrap.NewBadgerTx(writable, "", nil)
}
}
}()
}
select {}
}
*/

65
bluegreentx.go
View file

@ -51,7 +51,9 @@ type blueGreenTx struct {
func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
as := a.Type()
bs := b.Type()
return &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs}
c := &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs}
c.checker.c = c
return c
}
var _ = newBlueGreenTx // keep linter happy
@ -140,13 +142,14 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
}
bKey, bValue := bIter.Value()
if bKey != aKey {
//vv("6960 really ought to be present index='%v',field='%v';view='%v';shard='%v'; isIn=%v", index, field, view, shard, c.isIn(index, field, view, shard, 456130566))
AlwaysPrintf("problem in caller %v", Caller(2))
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack()))
}
if err := aValue.BitwiseCompare(bValue); err != nil {
c.Dump()
vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
//vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
}
}
@ -155,7 +158,7 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
AlwaysPrintf("bIter has more than it should. problem in caller %v", Caller(2))
c.Dump()
bKey, _ := bIter.Value()
vv("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack())
//vv("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack())
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack()))
}
}
@ -192,19 +195,24 @@ func (c *blueGreenTx) Rollback() {
panic(r)
}
}()
//fmt.Printf("blueGreenTx.Rollback() about to call (%v) a.Rollback()\n", c.as)
//vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as)
c.a.Rollback()
//fmt.Printf("blueGreenTx.Rollback() about to call (%v) b.Rollback()\n", c.bs)
//vv("blueGreenTx.Rollback() about to call (%v) b.Rollback()", c.bs)
c.b.Rollback()
//vv("blueGreenTx.Rollback() done. bgtx p=%p", c)
}
func (c *blueGreenTx) Commit() error {
c.mu.Lock()
defer c.mu.Unlock()
//fmt.Printf("blueGreenTx.Commit() called.\n")
// for rbf 6930 debug stuff:
//in := c.isIn("i", "x", "standard", 0, 456130566)
//fmt.Printf("blueGreenTx.Commit() called. bgtx p=%p; in rbf=%v\n", c, in[0])
if c.rollbackOrCommitDone {
return nil
}
//vv("blueGreenTx.Commit() called. bgtx p=%p", c)
c.rollbackOrCommitDone = true
c.checkDatabase()
defer func() {
@ -217,6 +225,13 @@ func (c *blueGreenTx) Commit() error {
_ = errA
errB := c.b.Commit()
/*
tx2, err := c.idx.Txf.rbfDB.NewRBFTx(false, "", nil)
panicOn(err)
inRbf, err := tx2.Contains("i", "x", "standard", 0, 456130566)
panicOn(err)
fmt.Printf("AFTER commits happened, blueGreenTx.Commit() called. bgtx p=%p; in rbf=%v\n", c, inRbf)
*/
compareErrors(errA, errB)
return errB
}
@ -281,9 +296,9 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
// these are the first port of call for debugging, so we leave them in.
// ================== begin save comments.
//c.checkDatabase()
//vv("got past database check at TOP of ImportRoaringBits")
////vv("got past database check at TOP of ImportRoaringBits")
//c.Dump()
//vv("done with top dump; clear=%v", clear)
////vv("done with top dump; clear=%v", clear)
// ================== end save comments.
defer func() {
if r := recover(); r != nil {
@ -344,9 +359,30 @@ func (c *blueGreenTx) UseRowCache() bool {
return false
}
var _ = (&blueGreenTx{}).isIn // happy linter
func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) {
r = make([]bool, 2)
inA, errA := c.a.Contains(index, field, view, shard, ckey)
panicOn(errA)
inB, errB := c.b.Contains(index, field, view, shard, ckey)
panicOn(errB)
r[0] = inA
r[1] = inB
return
}
func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
c.checker.see(index, field, view, shard)
//vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard)
defer func() {
// rbf 6960 debug code:
/*
in := c.isIn("i", "x", "standard", 0, 456130566)
if in[0] || in[1] {
vv("first time 6960 present isIn=%v; bgtx p=%p stack=\n%v", in, c, stack())
}
*/
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
panic(r)
@ -590,7 +626,6 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
c.checker.see(index, field, view, shard)
//vv("CountRange start=0x%x, endx=0x%x", start, end)
defer func() {
if r := recover(); r != nil {
c.Dump()
@ -626,6 +661,10 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
panicOn(err)
}
compareErrors(errA, errB)
//vv("end of blue-green OffsetRange, dump:")
//c.Dump()
return b, errB
}
@ -692,8 +731,9 @@ func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string)
}
for _, kb := range slcB {
if !ma[kb] {
//vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)
c.Dump()
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
}
delete(ma, kb)
}
@ -758,6 +798,8 @@ func (m *MultiReaderB) Close() error {
type blueGreenChecker struct {
visited map[string]map[string]map[string]map[uint64]struct{}
c *blueGreenTx
// lock mu when using visited.
// otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey
mu sync.Mutex
@ -768,6 +810,9 @@ func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
// keep this next Printf. Useful to see the sequence of Tx operations.
//fmt.Printf("blueGreenTx.%v on index='%v'\n", Caller(1), index)
// is ckey 6960 present in i/x/standard/0 ?
////vv("6960 present index='%v',field='%v';view='%v';shard='%v'; isIn=%v", index, field, view, shard, b.c.isIn(index, field, view, shard, 456130566))
b.mu.Lock()
defer b.mu.Unlock()

1
cmd/demo-lmdb/lmdb.go
View file

@ -13,7 +13,6 @@
// limitations under the License.
// +build !386
// +build skip_building_lmdb_for_now
package main

1
cmd/demo-lmdb/vprint.go
View file

@ -23,7 +23,6 @@
// SOFTWARE.
// +build !386
// +build skip_building_lmdb_for_now
package main

186
cmd/lmdb-keydump/keydump.go Normal file
View file

@ -0,0 +1,186 @@
// home https://github.com/glycerine/lmdb-go
// Copyright (c) 2020, the lmdb-go authors
// Copyright (c) 2015, Bryan Matsuo
// All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
// Neither the name of the author nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// +build amd64
package main
import (
"fmt"
"os"
"runtime"
"sort"
"github.com/glycerine/lmdb-go/lmdb"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/txkey"
)
// keydump simply prints all the keys in the database path specified
// as the first argument on the command line.
func main() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
if len(os.Args) < 2 {
fmt.Fprintf(os.Stderr, "must supply path to database as only arg\n")
os.Exit(1)
}
path := os.Args[1]
if !FileExists(path) {
fmt.Fprintf(os.Stderr, "path '%v' does not exist.\n", path)
os.Exit(1)
}
maxr := 1
env, err := lmdb.NewEnvMaxReaders(maxr)
panicOn(err)
defer env.Close()
panicOn(env.SetMapSize(256 << 30))
err = env.SetMaxDBs(10)
panicOn(err)
//var myflags uint = NoReadahead | NoSubdir
var myflags uint = lmdb.NoSubdir
err = env.Open(path, myflags, 0664)
panicOn(err)
// In any real application it is important to check for readers that were
// never closed by their owning process, and for which the owning process
// has exited. See the documentation on transactions for more information.
staleReaders, err := env.ReaderCheck()
panicOn(err)
if staleReaders > 0 {
vv("cleared %d reader slots from dead processes", staleReaders)
}
dbnames := []string{}
shardSize := make(map[string]int)
var dbiRoot lmdb.DBI
var dbi lmdb.DBI
err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) {
//txn.RawRead = true
dbiRoot, err = txn.OpenRoot(0)
panicOn(err)
cur, err := txn.OpenCursor(dbiRoot)
panicOn(err)
defer cur.Close()
for i := 0; true; i++ {
var k, v []byte
var err error
if i == 0 {
// must give it at least a zero byte here to start.
k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange)
panicOn(err)
} else {
k, v, err = cur.Get([]byte(nil), nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
} else {
panicOn(err)
}
}
dbnames = append(dbnames, string(k))
_ = v
}
cur.Close()
return
})
panicOn(err)
for _, dbn := range dbnames {
fmt.Printf(`
=========================
database '%v':
=========================
`, dbn)
err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) {
//txn.RawRead = true
dbi, err = txn.OpenDBI(dbn, 0)
panicOn(err)
cur, err := txn.OpenCursor(dbi)
panicOn(err)
defer cur.Close()
for i := 0; true; i++ {
var k, v []byte
var err error
if i == 0 {
// must give it at least a zero byte here to start.
k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange)
panicOn(err)
} else {
k, v, err = cur.Get([]byte(nil), nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
} else {
panicOn(err)
}
}
vs := ""
if len(v) < 100 {
vs = fmt.Sprintf("%x", v) + " "
}
fmt.Printf("%04v %v len value; key: '%v' len %v -> %v\n", i, len(v), string(k), len(k), vs)
pre := txkey.PrefixFromKey(k)
shardSize[string(pre)] += len(v)
}
return
})
panicOn(err)
} // for dbnames
fmt.Printf("=================== done.\n")
var lines []*pilosa.LineSorter
for k, v := range shardSize {
lines = append(lines, &pilosa.LineSorter{Line: k, Tot: float64(v)})
}
sort.Sort(pilosa.SortByTot(lines))
fd, err := os.Create("shardsize")
panicOn(err)
defer fd.Close()
for _, ln := range lines {
fmt.Fprintf(fd, "%v %v\n", int(ln.Tot), ln.Line)
}
}

179
cmd/lmdb-keydump/vprint.go Normal file
View file

@ -0,0 +1,179 @@
// home: https://github.com/glyerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
// +build amd64
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

191
dbshard.go Normal file
View file

@ -0,0 +1,191 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"sync"
)
// types to support a database file per shard
type DBnode struct {
Holder map[string]*DBholder
}
type DBholder struct {
Index map[string]*DBindex
}
type DBindex struct {
Field map[string]*DBfield
}
type DBfield struct {
View map[string]*DBview
}
type DBview struct {
Shard map[uint64]*DBshard
}
type DBWrapper interface {
DeleteDBPath(path string) error
Close() error
}
type DBRegistry interface {
OpenDBWrapper(path string) (DBWrapper, error)
}
type DBshard struct {
Path string
ID DBID
Open bool
// With RWMutex, the
// writer who calls Lock() automatically gets priority over
// any reader who arrives later, even if the lock is held
// by a reader to start with.
RWMut sync.RWMutex
W DBWrapper
ParentDBview *DBview
}
type DBID struct {
Node string
Holder string
Index string
Field string
View string
Shard uint64
}
func (id *DBID) Path() (s string) {
s = id.Node + sep + id.Holder + sep + id.Index +
sep + id.Field + sep + id.View + sep + fmt.Sprintf("%04v", id.Shard)
return
}
type DBPerShard struct {
Mu sync.Mutex
Dir string
Node map[string]*DBnode
// just flat, not buried within the Node heirarchy.
// Easily see how many we have.
Flatmap map[*DBshard]struct{}
}
func NewDBPerShard(dir string) (d *DBPerShard) {
d = &DBPerShard{
Dir: dir,
Node: make(map[string]*DBnode),
Flatmap: make(map[*DBshard]struct{}),
}
return
}
func (per *DBPerShard) GetDBshard(registry DBRegistry, id DBID) (dbs *DBshard, err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
dbn, ok := per.Node[id.Node]
if !ok {
dbn = &DBnode{
Holder: make(map[string]*DBholder),
}
per.Node[id.Node] = dbn
}
dbh, ok := dbn.Holder[id.Holder]
if !ok {
dbh = &DBholder{
Index: make(map[string]*DBindex),
}
dbn.Holder[id.Holder] = dbh
}
dbi, ok := dbh.Index[id.Index]
if !ok {
dbi = &DBindex{
Field: make(map[string]*DBfield),
}
dbh.Index[id.Index] = dbi
}
dbf, ok := dbi.Field[id.Field]
if !ok {
dbf = &DBfield{
View: make(map[string]*DBview),
}
dbi.Field[id.Field] = dbf
}
dbv, ok := dbf.View[id.View]
if !ok {
dbv = &DBview{
Shard: make(map[uint64]*DBshard),
}
dbf.View[id.View] = dbv
}
dbs, ok = dbv.Shard[id.Shard]
if !ok {
dbs = &DBshard{
ParentDBview: dbv,
ID: id,
Path: per.Dir + sep + id.Path(),
}
dbv.Shard[id.Shard] = dbs
}
if !dbs.Open {
dbs.W, err = registry.OpenDBWrapper(dbs.Path)
if dbs.W != nil {
per.Flatmap[dbs] = struct{}{}
}
}
return
}
func (per *DBPerShard) Del(dbs *DBshard) (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
err = dbs.W.Close()
if err != nil {
return
}
panicOn(dbs.W.DeleteDBPath(dbs.Path))
delete(per.Flatmap, dbs)
// delete from the heirarchy
delete(dbs.ParentDBview.Shard, dbs.ID.Shard)
return
}
func (per *DBPerShard) Close() (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
for _, dbn := range per.Node {
for _, dbh := range dbn.Holder {
for _, dbi := range dbh.Index {
for _, dbf := range dbi.Field {
for _, dbv := range dbf.View {
for _, dbs := range dbv.Shard {
err = dbs.W.Close()
panicOn(err)
}
}
}
}
}
}
return
}

13
field.go
View file

@ -22,6 +22,7 @@ import (
"io/ioutil"
"log"
"math"
"math/bits"
"os"
"path/filepath"
"sort"
@ -755,7 +756,7 @@ fileLoop:
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
if f.idx.Txf.TxType() == roaringFragmentFilesTxn {
if f.idx.Txf.TxType() == RoaringTxn {
// Automatically upgrade BSI v1 fragments if they exist & reopen view.
if bsig := f.bsiGroup(f.name); bsig != nil {
if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil {
@ -2151,17 +2152,9 @@ func isValidCacheType(v string) bool {
}
}
// TODO(jea): why isn't this bits.Len64(x) using import "math/bits"
// That would be much (80x or more) faster and correct if the high bit is set.
//
// bitDepth returns the number of bits required to store a value.
func bitDepth(v uint64) uint {
for i := uint(0); i < 63; i++ {
if v < (1 << i) {
return i
}
}
return 63
return uint(bits.Len64(v))
}
// bitDepthInt64 returns the required bit depth for abs(v).

6
fragment.go
View file

@ -2306,6 +2306,12 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
start := rowID * ShardWidth
end := (rowID + 1) * ShardWidth
// avoid a 2nd r/w iterator if possible,
// aiming to having fewer write/read Tx conflicts.
badgerTx, isBadger := tx.(*BadgerTx)
if isBadger {
badgerTx.frag = f
}
n, err := tx.CountRange(f.index, f.field, f.view, f.shard, start, end)
if err != nil {
return errors.Wrap(err, "CountRange")

27
fragment_internal_test.go
View file

@ -5179,7 +5179,7 @@ func TestImportClearRestart(t *testing.T) {
if err != nil {
t.Fatalf("initial small import: %v", err)
}
if idx.Txf.TxType() == roaringFragmentFilesTxn {
if idx.Txf.TxType() == RoaringTxn {
if expOpN <= maxOpN && f.opN != expOpN {
t.Errorf("unexpected opN - %d is not %d", f.opN, expOpN)
}
@ -5199,7 +5199,7 @@ func TestImportClearRestart(t *testing.T) {
t.Fatalf("reopening fragment: %v", err)
}
if idx.Txf.TxType() == roaringFragmentFilesTxn {
if idx.Txf.TxType() == RoaringTxn {
if expOpN <= maxOpN && f.opN != expOpN {
t.Errorf("unexpected opN after close/open %d is not %d", f.opN, expOpN)
}
@ -5237,7 +5237,7 @@ func TestImportClearRestart(t *testing.T) {
t.Fatalf("opening new fragment: %v", err)
}
if idx.Txf.TxType() == roaringFragmentFilesTxn {
if idx.Txf.TxType() == RoaringTxn {
if expOpN <= maxOpN && f2.opN != expOpN {
t.Errorf("unexpected opN after close/open %d is not %d", f2.opN, expOpN)
}
@ -5323,15 +5323,18 @@ func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{})
func TestImportValueConcurrent(t *testing.T) {
f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
switch idx.Txf.TxType() {
case blueGreenBadgerRoaring, blueGreenRoaringBadger:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"blueGreenTx because the lack of transactional consistency " +
"from Roaring-per-file will create false comparison " +
"failures."))
case lmdbTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"lmdb since only a single writer is allowed at once."))
types := idx.Txf.TxTypes()
for _, ty := range types {
switch ty {
case roaringTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"blueGreenTx because the lack of transactional consistency " +
"from Roaring-per-file will create false comparison " +
"failures."))
case lmdbTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"lmdb since only a single writer is allowed at once."))
}
}
// Since eg.Go gets called multiple times below, each

6
go.mod
View file

@ -12,18 +12,14 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
github.com/glycerine/lmdb-go v1.9.11
github.com/glycerine/lmdb-go v1.9.26
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.2.0
github.com/golang/protobuf v1.3.3
github.com/google/go-cmp v0.2.0
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1

4
go.sum
View file

@ -55,8 +55,8 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
github.com/glycerine/lmdb-go v1.9.11 h1:Jutsg5jgYxZIHf5DqV4Bu+JVYs3Ieax7DASNigr6TUg=
github.com/glycerine/lmdb-go v1.9.11/go.mod h1:iztA3wBlR0RO8jTYTqGTGoySIEa6vFAXWEByWusDfOY=
github.com/glycerine/lmdb-go v1.9.26 h1:4aIiCQhg5fLChuZuATDHD4Lr6y9CdEHLtvROkzCZKIg=
github.com/glycerine/lmdb-go v1.9.26/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=

10
holder.go
View file

@ -603,6 +603,11 @@ func (h *Holder) processForeignIndexFields() error {
// Close closes all open fragments.
func (h *Holder) Close() error {
if globalUseStatTx {
fmt.Printf("%v\n", globalCallStats.report())
}
h.Stats.Close()
// Notify goroutines of closing and wait for completion.
@ -1472,14 +1477,15 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
// Obtain a read lock on index to prevent Index.Close() from
// destroying the Index.translateStores map before this is
// done using it.
index.mu.RLock()
//
// Update: there was another path down to Index.Close(), so
// we shrink to lock to be inside index.TranslateStore() now.
for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
ownsPartition := s.Cluster.unprotectedOwnsPartition(s.Node.ID, partitionID)
if ts := index.TranslateStore(partitionID); ts != nil {
ts.SetReadOnly(!ownsPartition)
}
}
index.mu.RUnlock()
for _, field := range index.Fields() {
field.TranslateStore().SetReadOnly(!isCoordinator)

48
index.go
View file

@ -33,16 +33,6 @@ import (
"golang.org/x/sync/errgroup"
)
// debug: TODO(jea): remove this init() that does cpu profiling.
func init() {
go func() {
// give time for env var TXSRC to be set.
//time.Sleep(5 * time.Second)
//CPUProfileForDur(time.Minute, "cpu.pprof")
//CPUProfileForDur(15*time.Second, "cpu.pprof")
}()
}
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
@ -178,6 +168,8 @@ func (i *Index) TranslateStorePath(partitionID int) string {
// TranslateStore returns the translation store for a given partition.
func (i *Index) TranslateStore(partitionID int) TranslateStore {
i.mu.RLock() // avoid race with Index.Close() doing i.translateStores = make(map[int]TranslateStore)
defer i.mu.RUnlock()
return i.translateStores[partitionID]
}
@ -723,3 +715,39 @@ func (idx *Index) Dump(label string) {
fmt.Printf("\n%v Index.Dump('%v') for index '%v':\n", fileline, label, idx.name)
tx.Dump()
}
func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
// If we go to a database per shard then index will need this, or
// something like it, to read database files/directories
// and figure out what all the shards are so that a view
// can open its fragments.
file, err := os.Open(filepath.Join(viewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
idx.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", idx.name, field, view, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}

2
license.exceptions
View file

@ -14,3 +14,5 @@
./cmd/slurp/vprint.go
./cmd/demo-lmdb/vprint.go
./gid.go
./cmd/lmdb-keydump/vprint.go
./cmd/lmdb-keydump/keydump.go

600
lmdb/lmdb.go → lmdb.go
View file

File diff suppressed because it is too large Load diff

524
lmdb/txpool.go
View file

@ -1,524 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build skip_building_lmdb_for_now
package pilosa
import (
"fmt"
"io"
"github.com/pilosa/pilosa/v2/roaring"
)
// poolTx directs all Tx calls to a pre-made
// goroutine pool that are setup to do
// LMDB operations safely. Each of these
// goroutines has had runtime.LockOSThread()
// called, and we serialize write Tx onto
// a single writer goroutine.
type poolTx struct {
w *LMDBWrapper
b *LMDBTx
}
var _ = (*LMDBWrapper).newPoolTx // happy linter
func (w *LMDBWrapper) newPoolTx(write bool, initialIndexName string) (ptx *poolTx) {
var tx *LMDBTx
job := newLMDBJob(write, func(j *lmdbJob) {
tx = w.NewLMDBTx(write, initialIndexName)
})
if suberr := w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return &poolTx{
w: w,
b: tx,
}
}
var _ Tx = (*poolTx)(nil)
func (c *poolTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
c.b.IncrementOpN(index, field, view, shard, changedN)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
}
func (c *poolTx) NewTxIterator(index, field, view string, shard uint64) (rit *roaring.Iterator) {
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
rit = c.b.NewTxIterator(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changed, rowSet, err = c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Dump() {
c.b.Dump()
}
func (c *poolTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Readonly()
}
func (tx *poolTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (c *poolTx) Rollback() {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
c.b.Rollback()
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
}
func (c *poolTx) Commit() (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.Commit()
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RoaringBitmap(index, field, view string, shard uint64) (rbm *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
rbm, err = c.b.RoaringBitmap(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
ct, err = c.b.Container(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.PutContainer(index, field, view, shard, key, rc)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RemoveContainer(index, field, view string, shard uint64, key uint64) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.RemoveContainer(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *poolTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changeCount, err = c.b.Add(index, field, view, shard, batched, a...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changeCount, err = c.b.Remove(index, field, view, shard, a...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
exists, err = c.b.Contains(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
citer, found, err = c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.ForEach(index, field, view, shard, fn)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.ForEachRange(index, field, view, shard, start, end, fn)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Count(index, field, view string, shard uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.Count(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Max(index, field, view string, shard uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.Max(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Min(index, field, view string, shard uint64) (m uint64, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
m, found, err = c.b.Min(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.UnionInPlace(index, field, view, shard, others...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.CountRange(index, field, view, shard, start, end)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
other, err = c.b.OffsetRange(index, field, view, shard, offset, start, end)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
r, sz, err = c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Type() string {
return c.b.Type()
}
func (c *poolTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
sliceOfShards, err = c.b.SliceOfShards(index, field, view, optionalViewPath)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}

423
lmdb_other.go Normal file
View file

@ -0,0 +1,423 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !amd64
package pilosa
// this is a stubbed out file to let 386 build. lmdb won't work well
// on 32-bit; not enough memory map address space.
import (
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/pilosa/pilosa/v2/roaring"
)
var _ = time.Now
// lmdbRegistrar facilitates shutdown
// of all the lmdb databases started under
// tests. Its needed because most tests don't cleanup
// the *Index(es) they create. But we still
// want to shutdown lmdbDB goroutines
// after tests run.
//
// It also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
//
type lmdbRegistrar struct {
mu sync.Mutex
mp map[*LMDBWrapper]bool
path2db map[string]*LMDBWrapper
}
var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar()
func newLMDBTestRegistrar() *lmdbRegistrar {
return &lmdbRegistrar{
mp: make(map[*LMDBWrapper]bool),
path2db: make(map[string]*LMDBWrapper),
}
}
// register each lmdb created under tests, so we
// can clean them up. This is called by openLMDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
// return the existing instance.
func (r *lmdbRegistrar) unprotectedRegister(w *LMDBWrapper) {
panic("lmdb only available on 64-bit arch")
}
// unregister removes w from r
func (r *lmdbRegistrar) unregister(w *LMDBWrapper) {
panic("lmdb only available on 64-bit arch")
}
func DumpAllLMDB() {
panic("lmdb only available on 64-bit arch")
}
// lmdbPath is a helper for determining the full directory
// in which the lmdb database will be stored.
func lmdbPath(path string) string {
if !strings.HasSuffix(path, "-lmdb") {
return path + "-lmdb"
}
return path
}
// openLMDBDB opens the database in the bpath directoy
// without deleting any prior content. Any LMDBDB
// database directory will have the "-lmdb" suffix.
//
// openLMDBDB will check the registry and make a new instance only
// if one does not exist for its bpath. Otherwise it returns
// the existing instance. This insures only one lmdbDB
// per bpath in this pilosa node.
func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
panic("lmdb only available on 64-bit arch")
}
var ErrShutdown = fmt.Errorf("shutting down")
// DeleteIndex deletes all the containers associated with
// the named index from the lmdb database.
func (w *LMDBWrapper) DeleteIndex(indexName string) error {
panic("lmdb only available on 64-bit arch")
}
// statically confirm that LMDBTx satisfies the Tx interface.
var _ Tx = (*LMDBTx)(nil)
// LMDBWrapper provides the NewLMDBTx() method.
// Execute lmdbJob's via LMDBWrapper.submit(); these must
// be done by the lmdb goroutine worker pool.
type LMDBWrapper struct{}
func (w *LMDBWrapper) IsClosed() bool {
panic("lmdb only available on 64-bit arch")
}
// NewLMDBTx produces LMDB based ACID transactions. If
// the transaction will modify data, then the write flag must be true.
// Read-only queries should set write to false, to allow more concurrency.
// Methods on a LMDBTx are thread-safe, and can be called from
// different goroutines.
//
// initialIndexName is optional. It is set by the TxFactory from the Txo
// options provided at the Tx creation point. It allows us to recognize
// and isolate cross-index queries more quickly. It can always be empty ""
// but when set is highly useful for debugging. It has no impact
// on transaction behavior.
//
func (w *LMDBWrapper) NewLMDBTx(write bool, initialIndexName string, frag *fragment) (tx *LMDBTx) {
panic("lmdb only available on 64-bit arch")
}
// Close shuts down the LMDB database.
func (w *LMDBWrapper) Close() (err error) {
panic("lmdb only available on 64-bit arch")
}
// LMDBTx wraps a lmdb.Txn and provides the Tx interface
// method implementations.
// The methods on LMDBTx are thread-safe, and can be called
// from different goroutines.
type LMDBTx struct{}
func (tx *LMDBTx) Type() string {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) UseRowCache() bool {
panic("lmdb only available on 64-bit arch")
}
// Pointer gives us a memory address for the underlying transaction for debugging.
// It is public because we use it in roaring to report invalid container memory access
// outside of a transaction.
func (tx *LMDBTx) Pointer() string {
panic("lmdb only available on 64-bit arch")
}
// Rollback rolls back the transaction.
func (tx *LMDBTx) Rollback() {
panic("lmdb only available on 64-bit arch")
}
// Commit commits the transaction to permanent storage.
// Commits can handle up to 100k updates to fragments
// at once, but not more. This is a LMDBDB imposed limit.
func (tx *LMDBTx) Commit() error {
panic("lmdb only available on 64-bit arch")
}
// Readonly returns true iff the LMDBTx is read-only.
func (tx *LMDBTx) Readonly() bool {
panic("lmdb only available on 64-bit arch")
}
// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment.
func (tx *LMDBTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
panic("lmdb only available on 64-bit arch")
}
// Container returns the requested roaring.Container, selected by fragment and ckey
func (tx *LMDBTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) {
panic("lmdb only available on 64-bit arch")
}
// PutContainer stores rc under the specified fragment and container ckey.
func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error {
panic("lmdb only available on 64-bit arch")
}
// RemoveContainer deletes the container specified by the shard and container key ckey
func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error {
panic("lmdb only available on 64-bit arch")
}
// Add sets all the a bits hot in the specified fragment.
func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
panic("lmdb only available on 64-bit arch")
}
// Remove clears all the specified a bits in the chosen fragment.
func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
panic("lmdb only available on 64-bit arch")
}
// Contains returns exists true iff the bit chosen by key is
// hot (set to 1) in specified fragment.
func (tx *LMDBTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
panic("lmdb only available on 64-bit arch")
}
// key is the container key for the first roaring Container
// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will
// return the first container at or after key. found will be true if a
// container is found at key.
//
// LMDBTx notes: We auto-stop at the end of this shard, not going beyond.
func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
panic("lmdb only available on 64-bit arch")
}
// LMDBIterator is the iterator returned from a LMDBTx.ContainerIterator() call.
// It implements the roaring.ContainerIterator interface.
type LMDBIterator struct{}
// NewLMDBIterator creates an iterator on tx that will
// only return badgerKeys that start with prefix.
func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
panic("lmdb only available on 64-bit arch")
}
// Close tells the database and transaction that the user is done
// with the iterator.
func (bi *LMDBIterator) Close() {
panic("lmdb only available on 64-bit arch")
}
// Valid returns false if there are no more values in the iterator's range.
func (bi *LMDBIterator) Valid() bool {
panic("lmdb only available on 64-bit arch")
}
// Seek allows the iterator to start at needle instead of the global begining.
func (bi *LMDBIterator) Seek(needle []byte) (ok bool) {
panic("lmdb only available on 64-bit arch")
}
func (bi *LMDBIterator) ValidForPrefix(prefix []byte) bool {
panic("lmdb only available on 64-bit arch")
}
func (bi *LMDBIterator) String() (r string) {
panic("lmdb only available on 64-bit arch")
}
var oneByteSliceOfZero = []byte{0}
// Next advances the iterator.
func (bi *LMDBIterator) Next() (ok bool) {
panic("lmdb only available on 64-bit arch")
}
// Value retrieves what is pointed at currently by the iterator.
func (bi *LMDBIterator) Value() (containerKey uint64, c *roaring.Container) {
panic("lmdb only available on 64-bit arch")
}
// lmdbFinder implements roaring.IteratorFinder.
// It is used by LMDBTx.ForEach()
type lmdbFinder struct {
tx *LMDBTx
index string
field string
view string
shard uint64
needClose []Closer
}
// FindIterator lets lmdbFinder implement the roaring.FindIterator interface.
func (bf *lmdbFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) {
panic("lmdb only available on 64-bit arch")
}
// Close closes all bf.needClose listed Closers.
func (bf *lmdbFinder) Close() {
panic("lmdb only available on 64-bit arch")
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *LMDBTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
panic("lmdb only available on 64-bit arch")
}
// ForEach applies fn to each bitmap in the fragment.
func (tx *LMDBTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
panic("lmdb only available on 64-bit arch")
}
// ForEachRange applies fn on the selected range of bits on the chosen fragment.
func (tx *LMDBTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
panic("lmdb only available on 64-bit arch")
}
// Count operates on the full bitmap level, so it sums over all the containers
// in the bitmap.
func (tx *LMDBTx) Count(index, field, view string, shard uint64) (uint64, error) {
panic("lmdb only available on 64-bit arch")
}
// Max is the maximum bit-value in your bitmap.
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
func (tx *LMDBTx) Max(index, field, view string, shard uint64) (uint64, error) {
panic("lmdb only available on 64-bit arch")
}
// Min returns the smallest bit set in the fragment. If no bit is hot,
// the second return argument is false.
func (tx *LMDBTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
panic("lmdb only available on 64-bit arch")
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
panic("lmdb only available on 64-bit arch")
}
// OffsetRange creates a new roaring.Bitmap to return in other. For all the
// hot bits in [start, endx) of the chosen fragment, it stores
// them into other but with offset added to their bit position.
// The primary client is doing this, using ShardWidth, already; see
// fragment.rowFromStorage() in fragment.go. For example:
//
// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard,
// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
// ^ offset ^ start ^ endx
//
// The start and endx arguments are container keys that have been shifted left by 16 bits;
// their highbits() will be taken to determine the actual container keys. This
// is done to conform to the roaring.OffsetRange() argument convention.
//
func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) {
panic("lmdb only available on 64-bit arch")
}
// IncrementOpN increments the tx opcount by changedN
func (tx *LMDBTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
panic("lmdb only available on 64-bit arch")
}
// ImportRoaringBits handles deletes by setting clear=true.
// rowSet[rowID] returns the number of bit changed on that rowID.
func (tx *LMDBTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
panic("lmdb only available on 64-bit arch")
}
// StringifiedLMDBKeys returns a string with all the container
// keys available in lmdb.
func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
panic("lmdb only available on 64-bit arch")
}
// countBitsSet returns the number of bits set (or "hot") in
// the roaring container value found by the txkey.Key()
// formatted bkey.
func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) Dump() {
panic("lmdb only available on 64-bit arch")
}
// stringifiedLMDBKeysTx reports all the lmdb keys and a
// corresponding blake3 hash viewable by txn within the entire
// lmdb database.
// It also reports how many bits are hot in the roaring container
// (how many bits are set, or 1 rather than 0).
//
// By convention, we must return the empty string if there
// are no keys present. The tests use this to confirm
// an empty database.
func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
panic("lmdb only available on 64-bit arch")
}
func (w *LMDBWrapper) DeleteField(index, field, fieldPath string) error {
panic("lmdb only available on 64-bit arch")
}
func (w *LMDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
panic("lmdb only available on 64-bit arch")
}
func (w *LMDBWrapper) DeletePrefix(prefix []byte) error {
panic("lmdb only available on 64-bit arch")
}
func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
panic("lmdb only available on 64-bit arch")
}
// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the
// specified fragment.
func (tx *LMDBTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
panic("lmdb only available on 64-bit arch")
}

108
lmdb/lmdb_test.go → lmdb_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build skip_building_lmdb_for_now
// +build amd64
package pilosa
@ -25,6 +25,7 @@ import (
"testing"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txkey"
)
// helpers, each runs their own new txn, and commits if a change/delete
@ -32,7 +33,7 @@ import (
func LMDBMustHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -45,7 +46,7 @@ func LMDBMustHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard
func LMDBMustNotHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -56,7 +57,7 @@ func LMDBMustNotHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, sha
}
func LMDBMustSetBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
// add a bit
changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@ -74,14 +75,14 @@ func LMDBMustSetBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard u
}
func LMDBMustDeleteBitvalueContainer(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
}
func LMDBMustDeleteBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
@ -91,7 +92,7 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) {
var err error
fn := lmdbPath(path)
panicOn(os.RemoveAll(fn))
w, err = globalLMDBReg.newLMDBWrapper(fn)
w, err = globalLMDBReg.openLMDBWrapper(fn)
panicOn(err)
// verify it is empty
@ -103,6 +104,9 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) {
return w, func() {
w.Close() // stop any started background GC goroutine.
os.RemoveAll(fn)
if FileExists(fn + "-lock") {
os.RemoveAll(fn)
}
}
}
@ -119,7 +123,7 @@ func TestLMDB_DeleteFragment(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
shard1 := uint64(1)
@ -154,7 +158,7 @@ func TestLMDB_DeleteFragment(t *testing.T) {
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index)
tx = dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
for _, s := range shards {
@ -209,7 +213,7 @@ func TestLMDB_Max_on_many_containers(t *testing.T) {
}
}
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
for _, shard := range shards {
@ -242,7 +246,7 @@ func TestLMDB_SetBitmap(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
bitvalue := uint64(0)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -263,7 +267,7 @@ func TestLMDB_SetBitmap(t *testing.T) {
// commited, so should be visible outside the txn
//
tx2 := dbwrap.NewLMDBTx(!writable, index)
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
@ -283,7 +287,7 @@ func TestLMDB_OffsetRange(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
@ -317,7 +321,7 @@ func TestLMDB_OffsetRange(t *testing.T) {
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewLMDBTx(!writable, index)
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
@ -331,7 +335,7 @@ func TestLMDB_OffsetRange(t *testing.T) {
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewLMDBTx(!writable, index)
tx3 := dbwrap.NewLMDBTx(!writable, index, nil)
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
@ -359,7 +363,7 @@ func TestLMDB_Count_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
@ -375,7 +379,7 @@ func TestLMDB_Count_dense_containers(t *testing.T) {
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
expected := 0
for i := uint64(0); i < (1<<16)+2; i += 2 {
@ -401,7 +405,7 @@ func TestLMDB_ContainerIterator_on_empty(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
@ -419,7 +423,7 @@ func TestLMDB_ContainerIterator_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(42)
@ -481,19 +485,19 @@ func TestLMDB_badgerKey_badgerPrefix(t *testing.T) {
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := badgerKey(index, field, view, shard, 0)
needle := txkey.Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := badgerPrefix(index, field, view, shard)
prefix := txkey.Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("badgerPrefix() output '%v'was not a prefix of badgerKey() '%v'", string(needle), string(prefix)))
panic(fmt.Sprintf("txkey.Prefix() output '%v'was not a prefix of txkey.Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("badgerPrefix() output '%v'was 20 characters shorter than badgerKey() '%v'", string(needle), string(prefix)))
panic(fmt.Sprintf("txkey.Prefix() output '%v'was 20 characters shorter than txkey.Key() '%v'", string(needle), string(prefix)))
}
// validate assumption that badgerKeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// validate assumption that txkey.KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
@ -507,10 +511,10 @@ func TestLMDB_badgerKey_badgerPrefix(t *testing.T) {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to badgerKeyExtractContainerKey(prefix='%v')", prefix))
panic(fmt.Sprintf("should have seen panic on call to txkey.KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
badgerKeyExtractContainerKey(prefix) // should panic.
txkey.KeyExtractContainerKey(prefix) // should panic.
}()
}
@ -519,7 +523,7 @@ func TestLMDB_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -574,7 +578,7 @@ func TestLMDB_ContainerIterator_empty_iteration_loop(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -624,7 +628,7 @@ func TestLMDB_ForEach_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(42)
@ -683,7 +687,7 @@ func TestLMDB_RemoveContainer_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
@ -692,7 +696,7 @@ func TestLMDB_RemoveContainer_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewLMDBTx(writable, index)
tx = dbwrap.NewLMDBTx(writable, index, nil)
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
@ -744,7 +748,7 @@ func TestLMDB_Remove_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
@ -755,7 +759,7 @@ func TestLMDB_Remove_one_bit_test(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewLMDBTx(writable, index)
tx = dbwrap.NewLMDBTx(writable, index, nil)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
@ -787,7 +791,7 @@ func TestLMDB_Min_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
@ -804,7 +808,7 @@ func TestLMDB_Min_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewLMDBTx(!writable, index)
tx = dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
@ -825,7 +829,7 @@ func TestLMDB_CountRange_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
if n != 0 {
@ -841,7 +845,7 @@ func TestLMDB_CountRange_on_many_containers(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewLMDBTx(!writable, index)
tx = dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@ -869,7 +873,7 @@ func TestLMDB_CountRange_middle_container(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
// pick out just the middle container with the 1 bit set on it.
@ -894,7 +898,7 @@ func TestLMDB_CountRange_many_middle_container(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
// get them all
@ -925,7 +929,7 @@ func TestLMDB_UnionInPlace(t *testing.T) {
LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx2 := dbwrap.NewLMDBTx(!writable, index)
tx2 := dbwrap.NewLMDBTx(!writable, index, nil)
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
if n != 2 {
@ -940,7 +944,7 @@ func TestLMDB_UnionInPlace(t *testing.T) {
}
mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
@ -965,7 +969,7 @@ func TestLMDB_RoaringBitmap(t *testing.T) {
putme := expected
LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
@ -989,7 +993,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
@ -1062,7 +1066,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
// should have no keys
if allkeys != "" {
panic("badger should have no keys now")
panic("lmdb should have no keys now")
}
}
@ -1072,7 +1076,7 @@ func TestLMDB_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1122,7 +1126,7 @@ func TestLMDB_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1181,7 +1185,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
bitvalue := uint64(777)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
for _, v := range bits {
@ -1218,7 +1222,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index2)
tx = dbwrap.NewLMDBTx(!writable, index2, nil)
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1243,7 +1247,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewLMDBTx(writable, index)
tx := dbwrap.NewLMDBTx(writable, index, nil)
bitvalue := uint64(777)
limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
//limit := uint64(101)
@ -1256,7 +1260,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
panicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
tx = dbwrap.NewLMDBTx(writable, index)
tx = dbwrap.NewLMDBTx(writable, index, nil)
}
}
@ -1273,7 +1277,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewLMDBTx(!writable, index2)
tx = dbwrap.NewLMDBTx(!writable, index2, nil)
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1302,7 +1306,7 @@ func TestLMDB_SliceOfShards(t *testing.T) {
for _, shard := range shards {
LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewLMDBTx(!writable, index)
tx := dbwrap.NewLMDBTx(!writable, index, nil)
defer tx.Rollback()
slc, err := tx.SliceOfShards(index, field, view, "")

308
mtx.go Normal file
View file

@ -0,0 +1,308 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
)
// MultiTx implements the transaction interface to combine multiple transactions.
type MultiTx struct {
mu sync.Mutex
writable bool
holder *Holder
index *Index
txs map[multiTxKey]Tx
}
// NewMultiTx returns a new instance of MultiTx for a Holder.
func NewMultiTx(writable bool, holder *Holder) *MultiTx {
return &MultiTx{
writable: writable,
holder: holder,
txs: make(map[multiTxKey]Tx),
}
}
// NewMultiTxWithIndex returns a new instance of MultiTx for a single index.
func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
return &MultiTx{
writable: writable,
index: index,
txs: make(map[multiTxKey]Tx),
}
}
var _ Tx = (*MultiTx)(nil)
type multiTxKey struct {
index string
shard uint64
write bool
}
func (mtx *MultiTx) Type() string {
return mtx.index.Txf.TxType()
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
tx.Rollback()
}
}
// Commit commits all underlying transactions.
func (mtx *MultiTx) Commit() (err error) {
for _, tx := range mtx.txs {
if e := tx.Commit(); e != nil && err == nil {
err = e
}
}
return err
}
func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.RoaringBitmap(index, field, view, shard)
}
func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.Container(index, field, view, shard, key)
}
func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.PutContainer(index, field, view, shard, key, c)
}
func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
}
return tx.Contains(index, field, view, shard, v)
}
func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, false, err
}
return tx.ContainerIterator(index, field, view, shard, key)
}
func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEach(index, field, view, shard, fn)
}
func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEachRange(index, field, view, shard, start, end, fn)
}
func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Count(index, field, view, shard)
}
func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Max(index, field, view, shard)
}
func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, false, err
}
return tx.Min(index, field, view, shard)
}
func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.UnionInPlace(index, field, view, shard, others...)
}
func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.CountRange(index, field, view, shard, start, end)
}
func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.OffsetRange(index, field, view, shard, offset, start, end)
}
// tx returns a transaction by index/shard. Reuses transaction if already open.
// Otherwise begins a new transaction.
func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
// If transaction doesn't exist, lookup the index.
idx := mtx.index
if mtx.holder != nil {
if idx = mtx.holder.Index(index); idx == nil {
return nil, ErrIndexNotFound
}
}
// Begin tranaction & cache it.
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[mkey] = tx
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("txNoShard: no prior tx in MultiTx available, looking up index='%v'", index))
}

12
rbf.go
View file

@ -25,7 +25,7 @@ import (
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -248,7 +248,7 @@ func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fr
func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := string(txpath.AllShardPrefix(index, field, view))
prefix := string(txkey.AllShardPrefix(index, field, view))
names, err := tx.tx.BitmapNames()
if err != nil {
@ -260,7 +260,7 @@ func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sli
if !strings.HasPrefix(name, prefix) {
continue
}
shard := txpath.ShardFromPrefix([]byte(name))
shard := txkey.ShardFromPrefix([]byte(name))
sliceOfShards = append(sliceOfShards, shard)
}
return sliceOfShards, nil
@ -296,13 +296,13 @@ func (tx *RBFTx) UseRowCache() bool {
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
func rbfName(index, field, view string, shard uint64) string {
//return fmt.Sprintf("%s\x00%s\x00%s\x00%d", index, field, view, shard)
return string(txpath.Prefix(index, field, view, shard))
return string(txkey.Prefix(index, field, view, shard))
}
// rbfFieldPrefix returns a prefix for field keys in RBF.
func rbfFieldPrefix(index, field string) string {
//return fmt.Sprintf("%s\x00%s\x00", index, field)
return string(txpath.FieldPrefix(index, field))
return string(txkey.FieldPrefix(index, field))
}
func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error {
@ -330,7 +330,7 @@ func (w *RbfDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in RbfDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
prefix := txpath.IndexOnlyPrefix(indexName)
prefix := txkey.IndexOnlyPrefix(indexName)
w.muDb.Lock()
defer w.muDb.Unlock()

4
rbf/tx_test.go
View file

@ -21,7 +21,7 @@ import (
"time"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pilosa/pilosa/v2/txkey"
)
func TestTx_CommitRollback(t *testing.T) {
@ -490,5 +490,5 @@ func TestTx_Dump(t *testing.T) {
}
func rbfName(index, field, view string, shard uint64) string {
return string(txpath.Prefix(index, field, view, shard))
return string(txkey.Prefix(index, field, view, shard))
}

2
roaring/container_stash.go
View file

@ -492,7 +492,7 @@ func (c *Container) setBitmap(bitmap []uint64) {
// runs yields the data viewed as a slice of intervals.
func (c *Container) runs() []Interval16 {
if c == nil {
panic("attempt to read nil container's runs")
return nil
}
if roaringParanoia {
if c.typeID != ContainerRun {

16
roaring/roaring.go
View file

@ -3301,6 +3301,7 @@ func (c *Container) bitmapRemove(v uint16) (*Container, bool) {
if !c.bitmapContains(v) {
return c, false
}
// removing the last item? we can just return the empty container.
if c.N() == 1 {
return nil, true
@ -6853,22 +6854,21 @@ func AsRuns(c *Container) []Interval16 {
return c.runs()
}
func ConvertArrayToBitmap(c *Container) {
c.arrayToBitmap()
func ConvertArrayToBitmap(c *Container) *Container {
return c.arrayToBitmap()
}
func ConvertRunToBitmap(c *Container) {
c.runToBitmap()
func ConvertRunToBitmap(c *Container) *Container {
return c.runToBitmap()
}
func Optimize(c *Container) {
c.optimize()
func Optimize(c *Container) *Container {
return c.optimize()
}
func Union(a, b *Container) (c *Container) {
c = union(a, b)
// c can be have arrays that are too big, and need
// to be optimized into raw bitmaps.
c.optimize()
return c
return c.optimize()
}
func Difference(a, b *Container) *Container {

102
roaring/roaring_test.go
View file

@ -2158,3 +2158,105 @@ func BenchmarkIntersectInPlace(b *testing.B) {
bm.IntersectInPlace(data.a2, data.b, data.r1, data.r2)
}
}
func TestArrayAddRemoveAddRemove(t *testing.T) {
var ct *roaring.Container
for i := 0; i < 3; i++ {
switch i {
case 0:
ct = roaring.NewContainerArray(nil)
case 1:
ct = roaring.NewContainerBitmap(0, nil)
case 2:
ct = roaring.NewContainerRun(nil)
}
values := []uint16{1, 0, 13, 77, 1511}
var added, removed bool
for _, v := range values {
ct, added = ct.Add(v)
if !added {
t.Fatalf("expected added to be true for: %d", v)
}
if !ct.Contains(v) {
t.Fatalf("expected bitmap to contain: %d", v)
}
ct, removed = ct.Remove(v)
if !removed {
t.Fatalf("expected removed to be true for: %d", v)
}
if n := ct.N(); n != 0 {
t.Fatalf("expected bitmap count to be zero now, got n=%v", n)
}
if ct.Contains(v) {
t.Fatalf("expected bitmap to not contain: %d", v)
}
}
}
}
func TestBitmapAddRemoveAddRemove(t *testing.T) {
ct := roaring.NewContainerBitmap(0, nil)
putmeValues := []uint16{1, 0, 13, 77, 1511}
var added, removed bool
for _, v := range putmeValues {
ct, added = ct.Add(v)
if !added {
t.Fatalf("expected added to be true for: %d", v)
}
if !ct.Contains(v) {
t.Fatalf("expected bitmap to contain: %d", v)
}
ct, removed = ct.Remove(v)
if !removed {
t.Fatalf("expected removed to be true for: %d", v)
}
if ct.Contains(v) {
t.Fatalf("expected bitmap to not contain: %d", v)
}
if n := ct.N(); n != 0 {
t.Fatalf("expected bitmap count to be zero now, got n=%v", n)
}
if ct != nil {
b := roaring.AsBitmap(ct)
for i, e := range b {
if e != 0 {
t.Fatalf("expected ct AsBitmap() contents to be all zeros, uint64 i=%v as %v", i, e)
}
}
}
}
}
func TestRunAddRemoveAddRemove(t *testing.T) {
ct := roaring.NewContainerRun(nil)
putmeValues := []uint16{1, 0, 13, 77, 1511}
var added, removed bool
for _, v := range putmeValues {
ct, added = ct.Add(v)
if !added {
t.Fatalf("expected added to be true for: %d", v)
}
if !ct.Contains(v) {
t.Fatalf("expected bitmap to contain: %d", v)
}
ct, removed = ct.Remove(v)
if !removed {
t.Fatalf("expected removed to be true for: %d", v)
}
if n := len(roaring.AsRuns(ct)); n != 0 {
t.Fatalf("expected ct AsRuns() len to be zero now, got n=%v", n)
}
if n := ct.N(); n != 0 {
t.Fatalf("expected bitmap count to be zero now, got n=%v", n)
}
if ct.Contains(v) {
t.Fatalf("expected bitmap to not contain: %d", v)
}
}
}

289
rrtx.go
View file

@ -21,297 +21,11 @@ import (
"os"
"path/filepath"
"strconv"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// MultiTx implements the transaction interface to combine multiple transactions.
type MultiTx struct {
mu sync.Mutex
writable bool
holder *Holder
index *Index
txs map[multiTxKey]Tx
}
// NewMultiTx returns a new instance of MultiTx for a Holder.
func NewMultiTx(writable bool, holder *Holder) *MultiTx {
return &MultiTx{
writable: writable,
holder: holder,
txs: make(map[multiTxKey]Tx),
}
}
// NewMultiTxWithIndex returns a new instance of MultiTx for a single index.
func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
return &MultiTx{
writable: writable,
index: index,
txs: make(map[multiTxKey]Tx),
}
}
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) Type() string {
return RoaringTxn
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
tx.Rollback()
}
}
// Commit commits all underlying transactions.
func (mtx *MultiTx) Commit() (err error) {
for _, tx := range mtx.txs {
if e := tx.Commit(); e != nil && err == nil {
err = e
}
}
return err
}
func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.RoaringBitmap(index, field, view, shard)
}
func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.Container(index, field, view, shard, key)
}
func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.PutContainer(index, field, view, shard, key, c)
}
func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
}
return tx.Contains(index, field, view, shard, v)
}
func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, false, err
}
return tx.ContainerIterator(index, field, view, shard, key)
}
func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEach(index, field, view, shard, fn)
}
func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEachRange(index, field, view, shard, start, end, fn)
}
func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Count(index, field, view, shard)
}
func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Max(index, field, view, shard)
}
func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, false, err
}
return tx.Min(index, field, view, shard)
}
func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.UnionInPlace(index, field, view, shard, others...)
}
func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.CountRange(index, field, view, shard, start, end)
}
func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.OffsetRange(index, field, view, shard, offset, start, end)
}
// tx returns a transaction by index/shard. Reuses transaction if already open.
// Otherwise begins a new transaction.
func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
// If transaction doesn't exist, lookup the index.
idx := mtx.index
if mtx.holder != nil {
if idx = mtx.holder.Index(index); idx == nil {
return nil, ErrIndexNotFound
}
}
// Begin tranaction & cache it.
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[mkey] = tx
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
}
type multiTxKey struct {
index string
shard uint64
write bool
}
// RoaringTx represents a fake transaction object for Roaring storage.
type RoaringTx struct {
write bool
@ -356,8 +70,7 @@ func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string)
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
//AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
tx.Index.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)

23
server/cluster_test.go
View file

@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"reflect"
"strings"
"testing"
@ -183,6 +184,19 @@ func TestClusterResize_AddNode(t *testing.T) {
}
})
t.Run("ContinuousShards", func(t *testing.T) {
// Why are we skipping this test under blue-green with Roaring?
//
// We see red test: during resize during importRoaringBits
// PILOSA_TXSRC=rbf_roaring go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards"
// green:
// PILOSA_TXSRC=roaring_rbf go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards"
//
// but rbf_badger and badger_rbf are both green (use the same data values for containers).
//
// Conclude: roaring reads a different size of data []byte in (due to ops log) bits vs others (RBF, badger), so
// we can't do blue-green with roaring on this test.
skipTestUnderBlueGreenWithRoaring(t)
// Configure node0
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
@ -725,3 +739,12 @@ func TestClusterMutualTLS(t *testing.T) {
t.Fatal(err)
}
}
func skipTestUnderBlueGreenWithRoaring(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if strings.Contains(src, "_") {
if strings.Contains(src, "roaring") {
t.Skip("skip for roaring blue-green")
}
}
}

666
stattx.go Normal file
View file

@ -0,0 +1,666 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io"
"math"
"os"
"sort"
"sync"
"time"
"github.com/pilosa/pilosa/v2/roaring"
)
// statTx is useful to profile on a
// per method basis, and to play with
// read/write locking.
type statTx struct {
b Tx
stats *callStats
}
// for now, just track call stats globally. But each statTx has
// a pointer to a callStats, so could be made per index or per shard, etc.
var globalCallStats = newCallStats()
type callStats struct {
// protect elap
mu sync.Mutex
// track how much time each call took.
elap map[kall]*elapsed
}
type elapsed struct {
dur []float64
}
func newCallStats() *callStats {
w := &callStats{}
w.reset()
return w
}
func (w *callStats) reset() {
w.mu.Lock()
defer w.mu.Unlock()
w.elap = make(map[kall]*elapsed)
for i := kall(0); i < kLast; i++ {
w.elap[i] = &elapsed{}
}
}
type LineSorter struct {
Line string
Tot float64
}
type SortByTot []*LineSorter
func (p SortByTot) Len() int {
return len(p)
}
func (p SortByTot) Less(i, j int) bool {
return p[i].Tot < p[j].Tot
}
func (p SortByTot) Swap(i, j int) {
p[i], p[j] = p[j], p[i]
}
func (c *callStats) report() (r string) {
txsrc := os.Getenv("PILOSA_TXSRC")
r = fmt.Sprintf("callStats: (%v)\n", txsrc)
c.mu.Lock()
defer c.mu.Unlock()
var lines []*LineSorter
for i := kall(0); i < kLast; i++ {
slc := c.elap[i].dur
n := len(slc)
if n == 0 {
continue
}
mean, sd, totaltm := computeMeanSd(slc)
if n == 1 {
sd = 0
mean = slc[0]
totaltm = slc[0]
}
line := fmt.Sprintf(" %20v N=%8v avg/op: %12v sd: %12v total: %12v\n", i.String(), n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm))
lines = append(lines, &LineSorter{Line: line, Tot: totaltm})
}
sort.Sort(SortByTot(lines))
for i := range lines {
r += lines[i].Line
}
return
}
var NaN = math.NaN()
func computeMeanSd(slc []float64) (mean, sd, tot float64) {
if len(slc) < 2 {
return NaN, NaN, NaN
}
for _, v := range slc {
tot += v
}
n := float64(len(slc))
mean = tot / n
variance := 0.0
for _, v := range slc {
tmp := (v - mean)
variance += tmp * tmp
}
variance = variance / n // biased, but we don't care b/c we can have very small n
sd = math.Sqrt(variance)
if sd < 1e-8 {
// sd is super close to zero, NaN out the z-score rather than +/- Inf
sd = NaN
}
return
}
func (c *callStats) add(k kall, dur time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
e := c.elap[k]
e.dur = append(e.dur, float64(dur))
}
func newStatTx(b Tx) *statTx {
w := &statTx{
b: b,
// For now, just track call stats globally.
// But this could be made per-Tx by making this be stats: newCallStats(),
// for example.
stats: globalCallStats,
}
return w
}
type kall int
// constants for kall argument to callStats.add()
const (
kIncrementOpN kall = iota
kNewTxIterator
kImportRoaringBits
kRollback
kCommit
kRoaringBitmap
kContainer
kPutContainer
kRemoveContainer
kAdd
kRemove
kContains
kContainerIterator
kForEach
kForEachRange
kCount
kMax
kMin
kUnionInPlace
kCountRange
kOffsetRange
kRoaringBitmapReader
kSliceOfShards
kLast // mark the end, always keep this last. The following aren't tracked atm:
kType
kDump
kReadonly
kPointer
kUseRowCache
)
func (k kall) String() string {
switch k {
case kIncrementOpN:
return "kIncrementOpN"
case kNewTxIterator:
return "kNewTxIterator"
case kImportRoaringBits:
return "kImportRoaringBits"
case kRollback:
return "kRollback"
case kCommit:
return "kCommit"
case kRoaringBitmap:
return "kRoaringBitmap"
case kContainer:
return "kContainer"
case kPutContainer:
return "kPutContainer"
case kRemoveContainer:
return "kRemoveContainer"
case kAdd:
return "kAdd"
case kRemove:
return "kRemove"
case kContains:
return "kContains"
case kContainerIterator:
return "kContainerIterator"
case kForEach:
return "kForEach"
case kForEachRange:
return "kForEachRange"
case kCount:
return "kCount"
case kMax:
return "kMax"
case kMin:
return "kMin"
case kUnionInPlace:
return "kUnionInPlace"
case kCountRange:
return "kCountRange"
case kOffsetRange:
return "kOffsetRange"
case kRoaringBitmapReader:
return "kRoaringBitmapReader"
case kSliceOfShards:
return "kSliceOfShards"
case kLast:
return "kLast"
case kType:
return "kType"
case kDump:
return "kDump"
case kReadonly:
return "kReadonly"
case kPointer:
return "kPointer"
case kUseRowCache:
return "kUseRowCache"
}
panic(fmt.Sprintf("unknown kall '%v'", int(k)))
}
var _ = newStatTx // happy linter
var _ = kPointer
var _ = kUseRowCache
var _ = kType
var _ = kDump
var _ = kReadonly
var _ Tx = (*statTx)(nil)
//IncrementOpN
func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
me := kIncrementOpN
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
me := kNewTxIterator
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
me := kImportRoaringBits
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (c *statTx) Dump() {
c.b.Dump()
}
func (c *statTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Readonly()
}
func (tx *statTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (c *statTx) Rollback() {
me := kRollback
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
c.b.Rollback()
}
func (c *statTx) Commit() error {
me := kCommit
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Commit()
}
func (c *statTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
me := kRoaringBitmap
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
}
func (c *statTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
me := kContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Container(index, field, view, shard, key)
}
func (c *statTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
me := kPutContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
}
func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
me := kRemoveContainer
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *statTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *statTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
me := kAdd
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Add(index, field, view, shard, batched, a...)
}
func (c *statTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
me := kRemove
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
}
func (c *statTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
me := kContains
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
}
func (c *statTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
me := kContainerIterator
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
}
func (c *statTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
me := kForEach
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
}
func (c *statTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
me := kForEachRange
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
}
func (c *statTx) Count(index, field, view string, shard uint64) (uint64, error) {
me := kCount
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Count(index, field, view, shard)
}
func (c *statTx) Max(index, field, view string, shard uint64) (uint64, error) {
me := kMax
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Max(index, field, view, shard)
}
func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
me := kMin
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Min(index, field, view, shard)
}
func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
me := kUnionInPlace
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
}
func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
me := kCountRange
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
}
func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
me := kOffsetRange
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
me := kRoaringBitmapReader
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (c *statTx) Type() string {
return c.b.Type()
}
func (c *statTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
me := kSliceOfShards
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.SliceOfShards(index, field, view, optionalViewPath)
}

490
txfactory.go
View file

@ -24,7 +24,7 @@ import (
"text/tabwriter"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -34,15 +34,6 @@ const (
BadgerTxn string = "badger"
LmdbTxn string = "lmdb"
RBFTxn string = "rbf"
// A is listed first, B is second. blueGreenTx returns the B output.
BlueGreenBadgerRoaring string = "badger_roaring"
BlueGreenRoaringBadger string = "roaring_badger"
BlueGreenRBFRoaring string = "rbf_roaring"
BlueGreenRoaringRBF string = "roaring_rbf"
BlueGreenBadgerRBF string = "badger_rbf"
BlueGreenRBFBadger string = "rbf_badger"
)
// DefaultTxsrc is set here. pilosa/server/config.go references it
@ -69,18 +60,17 @@ var sep = string(os.PathSeparator)
// transactions so that RBF, or Badger, or Roaring-fragment-files, or several
// of these at once in parallel, is used as the storage and transction layer.
type TxFactory struct {
typeOfTx txtype
typeOfTx string
badgerDB *BadgerDBWrapper
rbfDB *RbfDBWrapper
types []txtype // blue-green split individually here
badgerDB *BadgerDBWrapper
lmDB *LMDBWrapper
rbfDB *RbfDBWrapper
roaringDB *RoaringStore
dbsClosed bool // idemopotent CloseDB()
//lmDB *LMDBWrapper
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
@ -90,77 +80,55 @@ type TxFactory struct {
type txtype int
const (
noneTxn txtype = 0
roaringFragmentFilesTxn txtype = 1 // these don't really have any transactions
badgerTxn txtype = 2
rbfTxn txtype = 3
// A is listed first, B is second. blueGreenTx returns the B output.
blueGreenBadgerRoaring txtype = 4
blueGreenRoaringBadger txtype = 5
blueGreenRBFRoaring txtype = 6
blueGreenRoaringRBF txtype = 7
blueGreenBadgerRBF txtype = 8
blueGreenRBFBadger txtype = 9
lmdbTxn txtype = 10
noneTxn txtype = 0
roaringTxn txtype = 1 // these don't really have any transactions
badgerTxn txtype = 2
rbfTxn txtype = 3
lmdbTxn txtype = 4
)
func (txf *TxFactory) NeedsSnapshot() bool {
switch txf.typeOfTx {
case noneTxn:
panic("noneTxn should not occur")
case roaringFragmentFilesTxn:
return true
case badgerTxn:
return false
case rbfTxn:
return false
case blueGreenBadgerRoaring:
return true
case blueGreenRoaringBadger:
return true
case blueGreenRBFRoaring:
return true
case blueGreenRoaringRBF:
return true
case blueGreenBadgerRBF:
return false
case blueGreenRBFBadger:
return false
case lmdbTxn:
return false
func (txf *TxFactory) NeedsSnapshot() (b bool) {
for _, ty := range txf.types {
switch ty {
case roaringTxn:
b = true
}
}
panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx))
return
}
func MustTxsrcToTxtype(txsrc string) txtype {
switch txsrc {
case RoaringTxn: // "roaring"
return roaringFragmentFilesTxn
case BadgerTxn: // "badger"
return badgerTxn
case RBFTxn: // "rbf"
return rbfTxn
case BlueGreenBadgerRoaring: //"badger_roaring"
return blueGreenBadgerRoaring
case BlueGreenRoaringBadger: // "roaring_badger"
return blueGreenRoaringBadger
case BlueGreenRBFRoaring: //"rbf_roaring"
return blueGreenRBFRoaring
case BlueGreenRoaringRBF: //"roaring_rbf"
return blueGreenRoaringRBF
case BlueGreenBadgerRBF: // "badger_rbf"
return blueGreenBadgerRBF
case BlueGreenRBFBadger: // "rbf_badger"
return blueGreenRBFBadger
case LmdbTxn:
return lmdbTxn
func MustTxsrcToTxtype(txsrc string) (types []txtype) {
var srcs []string
if strings.Contains(txsrc, "_") {
srcs = strings.Split(txsrc, "_")
if len(srcs) != 2 {
panic("only two blue-green comparisons permitted")
}
} else {
srcs = append(srcs, txsrc)
}
panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
for i, s := range srcs {
switch s {
case RoaringTxn: // "roaring"
types = append(types, roaringTxn)
case BadgerTxn: // "badger"
types = append(types, badgerTxn)
case RBFTxn: // "rbf"
types = append(types, rbfTxn)
case LmdbTxn: // "lmdb"
types = append(types, lmdbTxn)
default:
panic(fmt.Sprintf("unknown txsrc '%v'", s))
}
if i == 1 {
if types[1] == types[0] {
panic(fmt.Sprintf("cannot blue-green the same txsrc on both arms: '%v'", s))
}
}
}
return
}
// NewTxFactory always opens an existing database. If you
@ -170,55 +138,48 @@ func MustTxsrcToTxtype(txsrc string) txtype {
func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
//vv("NewTxFactory called for txsrc '%v'; dir='%v'; name='%v'", txsrc, dir, name)
ty := MustTxsrcToTxtype(txsrc)
if ty < 1 || ty > 10 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
types := MustTxsrcToTxtype(txsrc)
f = &TxFactory{
typeOfTx: ty,
types: types,
typeOfTx: txsrc,
roaringDB: NewRoaringStore(),
}
switch ty {
case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger:
for _, ty := range f.types {
switch ty {
case roaringTxn:
// no-op. these are just files in a directory
// one, big, bad-ass badger for all data: the honeyBadger.
//
// Note that having a single Tx backing store for all indexes
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
case badgerTxn:
// one, big, bad-ass badger for all data: the honeyBadger.
//
// Note that having a single Tx backing store for all indexes
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = DetectMemAccessPastTx
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = DetectMemAccessPastTx
switch ty {
case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
path := dir + sep + "all-in-one-rbfdb"
f.rbfDB, err = globalRbfDBReg.openRbfDB(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new rbf db. path='%v'", path))
}
}
switch ty {
case lmdbTxn:
panic("lmdb is unfinished and relocated to the ldmb/ subdirectory for the moment.")
/*
case rbfTxn:
path := dir + sep + "all-in-one-rbfdb"
f.rbfDB, err = globalRbfDBReg.openRbfDB(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new rbf db. path='%v'", path))
}
case lmdbTxn:
path := dir + sep + "all-in-one"
f.lmDB, err = globalLMDBReg.newLMDBWrapper(path)
f.lmDB, err = globalLMDBReg.openLMDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new lmdb db. path='%v'", path))
}
*/
}
}
return f, err
@ -233,227 +194,160 @@ type Txo struct {
Shard uint64
}
func (f *TxFactory) TxType() txtype {
func (f *TxFactory) TxType() string {
return f.typeOfTx
}
func (f *TxFactory) DeleteIndex(name string) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
// from holder.go:955, by default is already done there with os.RemoveAll()
return nil
case badgerTxn:
return f.badgerDB.DeleteIndex(name)
case rbfTxn:
return f.rbfDB.DeleteIndex(name)
case blueGreenBadgerRoaring:
return f.badgerDB.DeleteIndex(name)
case blueGreenRoaringBadger:
return f.badgerDB.DeleteIndex(name)
case blueGreenRBFBadger:
_ = f.rbfDB.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case blueGreenBadgerRBF:
_ = f.badgerDB.DeleteIndex(name)
return f.rbfDB.DeleteIndex(name)
case blueGreenRBFRoaring:
// roaring already done
return f.rbfDB.DeleteIndex(name)
case blueGreenRoaringRBF:
// roaring already done
return f.rbfDB.DeleteIndex(name)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
func (f *TxFactory) TxTypes() []txtype {
return f.types
}
func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
return f.badgerDB.DeleteField(index, field, fieldPath)
case rbfTxn:
return f.rbfDB.DeleteField(index, field, fieldPath)
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteField(index, field, fieldPath)
return f.roaringDB.DeleteField(index, field, fieldPath)
case blueGreenRoaringBadger:
_ = f.roaringDB.DeleteField(index, field, fieldPath)
return f.badgerDB.DeleteField(index, field, fieldPath)
case blueGreenRBFBadger:
_ = f.badgerDB.DeleteField(index, field, fieldPath)
return f.rbfDB.DeleteField(index, field, fieldPath)
case blueGreenBadgerRBF:
_ = f.rbfDB.DeleteField(index, field, fieldPath)
return f.badgerDB.DeleteField(index, field, fieldPath)
case blueGreenRBFRoaring:
_ = f.rbfDB.DeleteField(index, field, fieldPath)
return f.roaringDB.DeleteField(index, field, fieldPath)
case blueGreenRoaringRBF:
_ = f.roaringDB.DeleteField(index, field, fieldPath)
return f.rbfDB.DeleteField(index, field, fieldPath)
//case lmdbTxn:
//return f.lmDB.DeleteField(index, field, fieldPath)
func (f *TxFactory) DeleteIndex(name string) (err error) {
for _, ty := range f.types {
switch ty {
case roaringTxn:
// from holder.go:955, by default is already done there with os.RemoveAll()
case badgerTxn:
err = f.badgerDB.DeleteIndex(name)
case rbfTxn:
err = f.rbfDB.DeleteIndex(name)
case lmdbTxn:
err = f.lmDB.DeleteIndex(name)
default:
panic(fmt.Sprintf("unknown txtyp : '%v'", ty))
}
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
return
}
func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case badgerTxn:
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringBadger:
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRBFBadger:
_ = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenBadgerRBF:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRBFRoaring:
_ = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringRBF:
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
// case lmdbTxn:
// return f.lmDB.DeleteFragment(index, field, view, shard, frag)
func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
for _, ty := range f.types {
switch ty {
case roaringTxn:
err = f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
err = f.badgerDB.DeleteField(index, field, fieldPath)
case rbfTxn:
err = f.rbfDB.DeleteField(index, field, fieldPath)
case lmdbTxn:
err = f.lmDB.DeleteField(index, field, fieldPath)
default:
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", ty))
}
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
return
}
func (f *TxFactory) DeleteFragmentFromStore(
index, field, view string, shard uint64, frag *fragment,
) (err error) {
for _, ty := range f.types {
switch ty {
case roaringTxn:
err = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case badgerTxn:
err = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
err = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
case lmdbTxn:
err = f.lmDB.DeleteFragment(index, field, view, shard, frag)
default:
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", ty))
}
}
return
}
func (f *TxFactory) CloseIndex(idx *Index) error {
// under roaring and all the new databases, this is a no-op.
//idx.Dump("CloseIndex")
return nil
}
func (f *TxFactory) CloseDB() error {
func (f *TxFactory) CloseDB() (err error) {
if f.dbsClosed {
return nil
}
f.dbsClosed = true
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
return f.badgerDB.Close()
case rbfTxn:
return f.rbfDB.Close()
case blueGreenBadgerRoaring:
return f.badgerDB.Close()
case blueGreenRoaringBadger:
return f.badgerDB.Close()
case blueGreenRBFRoaring:
return f.rbfDB.Close()
case blueGreenRoaringRBF:
return f.rbfDB.Close()
case blueGreenBadgerRBF:
_ = f.badgerDB.Close()
return f.rbfDB.Close()
case blueGreenRBFBadger:
_ = f.rbfDB.Close()
return f.badgerDB.Close()
case lmdbTxn:
return nil
for _, ty := range f.types {
switch ty {
case roaringTxn:
// no-op
case badgerTxn:
err = f.badgerDB.Close()
case rbfTxn:
err = f.rbfDB.Close()
case lmdbTxn:
err = f.lmDB.Close()
default:
panic(fmt.Sprintf("unknown txtype: '%v'", ty))
}
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
return
}
func (f *TxFactory) NewTx(o Txo) Tx {
var globalUseStatTx = false
func init() {
v := os.Getenv("PILOSA_CALLSTAT")
if v != "" {
globalUseStatTx = true
}
}
func (f *TxFactory) NewTx(o Txo) (txn Tx) {
defer func() {
if globalUseStatTx {
txn = newStatTx(txn)
}
}()
indexName := ""
if o.Index != nil {
indexName = o.Index.name
}
var txns []Tx
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
case badgerTxn:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
return btx
case rbfTxn:
tx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
panicOn(err)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
for _, ty := range f.types {
switch ty {
case roaringTxn:
txns = append(txns, &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment})
case badgerTxn:
//tx := NewMultiTxWithIndex(o.Write, o.Index)
//txns = append(txns, tx
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
txns = append(txns, btx)
case rbfTxn:
//tx := NewMultiTxWithIndex(o.Write, o.Index)
tx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
txns = append(txns, tx)
case lmdbTxn:
txns = append(txns, f.lmDB.NewLMDBTx(o.Write, indexName, o.Fragment))
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
return tx
case lmdbTxn:
//return f.lmDB.newPoolTx(o.Write, indexName)
case blueGreenBadgerRoaring:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(btx, rtx, f.idx)
case blueGreenRoaringBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, btx, f.idx)
case blueGreenBadgerRBF:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return newBlueGreenTx(btx, rbftx, f.idx)
case blueGreenRBFBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return newBlueGreenTx(rbftx, btx, f.idx)
case blueGreenRBFRoaring:
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rbftx, rtx, f.idx)
case blueGreenRoaringRBF:
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, rbftx, f.idx)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
if len(txns) > 1 {
return newBlueGreenTx(txns[0], txns[1], f.idx)
}
return txns[0]
}
func (ty txtype) String() string {
switch ty {
case noneTxn:
return "noneTxn"
case roaringFragmentFilesTxn:
return "roaringFragmentFilesTxn"
case roaringTxn:
return "roaringTxn"
case badgerTxn:
return "badgerTxn"
case rbfTxn:
return "rbfTxn"
case blueGreenBadgerRoaring:
return "blueGreenBadgerRoaring"
case blueGreenRoaringBadger:
return "blueGreenRoaringBadger"
case blueGreenRBFRoaring:
return "blueGreenRBFRoaring"
case blueGreenRoaringRBF:
return "blueGreenRoaringRBF"
case blueGreenBadgerRBF:
return "blueGreenBadgerRBF"
case blueGreenRBFBadger:
return "blueGreenRBFBadger"
case lmdbTxn:
return "lmdbTxn"
}
@ -600,7 +494,7 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
srbm := bitmapAsString(rbm)
panicOn(err)
bkey := string(txpath.Key(index, field, view, shard, ckey))
bkey := string(txkey.Key(index, field, view, shard, ckey))
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
r += " ......." + srbm + "\n"

13
txpath/txpath.go → txkey/txkey.go
View file

@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package txpath consolidates in one place the use of keys to index into our
// Package txkey consolidates in one place the use of keys to index into our
// various storage/txn back-ends. Databases badgerDB and rbfDB both use it,
// so that debug Dumps are comparable.
package txpath
package txkey
import (
"bytes"
@ -155,3 +155,12 @@ func IndexOnlyPrefix(indexName string) []byte {
func FieldPrefix(index, field string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
}
func PrefixFromKey(bkey []byte) (prefix []byte) {
MustValidateKey(bkey)
beg := bytes.LastIndex(bkey, []byte("@"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have '@' extract prefix", string(bkey)))
}
return bkey[:beg+1]
}

21
txpath/txpath_test.go → txkey/txkey_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package txpath
package txkey
import (
"bytes"
@ -87,3 +87,22 @@ func Test_ShardFromKey(t *testing.T) {
}()
}
func Test_PrefixFromKey(t *testing.T) {
k := []byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")
x := []byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@")
pre := PrefixFromKey(k)
if !bytes.Equal(pre, x) {
nx := len(x)
npre := len(pre)
if nx != npre {
panic(fmt.Sprintf("nx=%v, npre=%v; expected '%v', observed '%v'", nx, npre, string(x), string(pre)))
}
for i := 0; i < nx; i++ {
if x[i] != pre[i] {
panic(fmt.Sprintf("first diff at index %v, expected '%v', observed '%v'", i, string(x[:i]), string(pre[:i])))
}
}
panic(fmt.Sprintf("expected:\n%v\n, observed:\n%v\n", string(x), string(pre)))
}
}

15
txpath/txprefix.go~
View file

@ -1,15 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package txprefix

44
txpath/txprefix_test.go~
View file

@ -1,44 +0,0 @@
package txprefix
func TestBadger_KeyPrefix(t *testing.T) {
// txprefix.Prefix() must agree with txprefix.Key(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := txprefix.Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := txprefix.Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("txprefix.Prefix() output '%v'was not a prefix of txprefix.Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("txprefix.Prefix() output '%v'was 20 characters shorter than txprefix.Key() '%v'", string(needle), string(prefix)))
}
// validate assumption that txprefix.KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to txprefix.KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
txprefix.KeyExtractContainerKey(prefix) // should panic.
}()
}