Merge pull request #812 from molecula/bgdev_rb

blue_green migration; holdbkg.go holder goroutine.

.circleci/config.yml

@@ -201,6 +201,7 @@ workflows:
       - test:
           name: test-shardwidth-22
           shard_width: "22"
+          resource_class: large
           requires:
             - setup
       - cluster-tests:

Makefile

@@ -17,6 +17,7 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
 NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
+TEST_TAGS = roaringparanoia
 define LICENSE_HASH_CODE
     head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
 endef
@@ -38,11 +39,11 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)
+	go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v
 
 # Run test suite with race flag
 test-race:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
+	go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
 
 testv: topt testvsub
 
@@ -57,7 +58,7 @@ testvsub:
 	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
            echo; echo "___ testing subpkg $$i"; \
            cd $$i; pwd; \
-           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
+           go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
            echo; echo "999 done testing subpkg $$i"; \
            cd ..; \
         done
@@ -66,7 +67,7 @@ testvsub-race:
 	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
            echo; echo "___ testing subpkg $$i -race"; \
            cd $$i; pwd; \
-           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
+           go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
            echo; echo "999 done testing subpkg $$i -race"; \
            cd ..; \
         done
@@ -194,55 +195,55 @@ pilosa-chk:
 
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
 
 # run top tests, not subdirs. print summary red/green after.
 # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
 topt:
 	mv log.topt.roar log.topt.roar.prev || true
-	go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.roar
+	go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.roar
 	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
 	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
 
 topt-badger:
 	mv log.topt.badger log.topt.badger.prev || true
-	PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
+	PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
 	@echo "   log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
 	@echo "   log.topt.badger   red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
 
 topt-badger-race:
 	mv log.topt.badger-race log.topt.badger-race.prev || true
-	PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger-race
+	PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger-race
 	@echo "   log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
 	@echo "   log.topt.badger-race   red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
 
 topt-rbf:
 	mv log.topt.rbf log.topt.rbf.prev || true
-	PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.rbf
+	PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.rbf
 	@echo "   log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
 	@echo "   log.topt.rbf   red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
 
 topt-rbf-race:
 	mv log.topt.rbf-race log.topt.rbf-race.prev || true
-	PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m  2>&1 | tee log.topt.rbf-race
+	PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m  2>&1 | tee log.topt.rbf-race
 	@echo "   log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l
 	@echo "   log.topt.rbf-race   red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l
 
 topt-lmdb:
 	mv log.topt.lmdb log.topt.lmdb.prev || true
-	PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb
+	PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb
 	@echo "   log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l
 	@echo "   log.topt.lmdb   red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l
 
 topt-lmdb-race:
 	mv log.topt.lmdb log.topt.lmdb.prev || true
-	PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb-race
+	PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb-race
 	@echo "   log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l
 	@echo "   log.topt.lmdb-race   red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l
 
 topt-race:
 	mv log.topt.race log.topt.race.prev || true
-	go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.race
+	go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.race
 	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
 	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
 
@@ -251,73 +252,73 @@ topt-race:
 
 bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
 	mv log.bg-rr log.bg-rr.prev || true
-	PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rr
+	PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rr
 	@echo "   log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
 	@echo "   log.bg-rr   red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
 
 rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
 	mv log.bg.roar_bg log.bg.roar_bg.prev || true
-	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-bg
+	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-bg
 	@echo "   log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
 	@echo "   log.rr-bg   red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
 
 rbf-rr:
 	mv log.rbf-rr log.rbf-rr.prev || true
-	PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-rr
+	PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-rr
 	@echo "   log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
 	@echo "   log.rbf-rr   red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
 
 rr-rbf:
 	mv log.rr-rbf log.rr-rbf.prev || true
-	PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-rbf
+	PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-rbf
 	@echo "   log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
 	@echo "   log.rr-rbf   red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
 
 rbf-bg:
 	mv log.rbf-bg log.rbf-bg.prev || true
-	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-bg
+	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-bg
 	@echo "   log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
 	@echo "   log.rbf-bg   red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
 
 bg-rbf:
 	mv log.bg-rbf log.bg-rbf.prev || true
-	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rbf
+	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rbf
 	@echo "   log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
 	@echo "   log.bg-rbf   red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
 
 rbf-lm:
 	mv log.rbf-lm log.rbf-lm.prev || true
-	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-lm
+	PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-lm
 	@echo "   log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l
 	@echo "   log.rbf-lm   red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l
 
 lm-rbf:
 	mv log.lm-rbf log.lm-rbf.prev || true
-	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rbf
+	PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rbf
 	@echo "   log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l
 	@echo "   log.lm-rbf   red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l
 
 lm-rr:
 	mv log.lm-rr log.lm-rr.prev || true
-	PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rr
+	PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rr
 	@echo "   log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l
 	@echo "   log.lm-rr   red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l
 
 rr-lm:
 	mv log.rr-lm log.rr-lm.prev || true
-	PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-lm
+	PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-lm
 	@echo "   log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l
 	@echo "   log.rr-lm   red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l
 
 bg-lm:
 	mv log.topt.bg-lm log.topt.bg-lm.prev || true
-	PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.bg-lm
+	PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.bg-lm
 	@echo "   log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l
 	@echo "   log.topt.bg-lm   red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l
 
 lm-bg:
 	mv log.topt.lm-bg log.topt.lm-bg.prev || true
-	PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lm-bg
+	PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lm-bg
 	@echo "   log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l
 	@echo "   log.topt.lm-bg   red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l
 

api.go

@@ -54,6 +54,10 @@ type API struct {
 	Serializer Serializer
 }
 
+func (api *API) Holder() *Holder {
+	return api.holder
+}
+
 // apiOption is a functional option type for pilosa.API
 type apiOption func(*API) error
 

bluegreentx.go

@@ -32,7 +32,8 @@ import (
 // The checkDatabase() call will do reads of the fragments at Commit/Rollback,
 // while the snapshotqueue may be doing writes.
 //
-// Do not run with go test -race and expect it to be race free.
+// Do not run with go test -race and expect it to be race free with RoaringTx
+// on one arm.
 //
 type blueGreenTx struct {
 	a Tx
@@ -42,7 +43,8 @@ type blueGreenTx struct {
 	as string
 	bs string
 
-	types []txtype
+	types      []txtype
+	hasRoaring bool
 
 	// roaring will not create as many Tx (they are
 	// psuedo Tx anyway), espcially when deleting
@@ -57,6 +59,8 @@ type blueGreenTx struct {
 	rollbackOrCommitDone bool
 
 	txf *TxFactory
+
+	short bool // short Dump or long
 }
 
 // blueGreenRegistry is used to force checking of (read) transactions
@@ -65,19 +69,29 @@ type blueGreenTx struct {
 // roaring will show up, while writes to the DB won't show up on
 // readTx that have already started.
 type blueGreenRegistry struct {
-	mu sync.Mutex
-	m  map[int64]*blueGreenTx
+	mu         sync.Mutex
+	m          map[int64]*blueGreenTx
+	types      []txtype
+	hasRoaring bool
 }
 
-func newBlueGreenReg() *blueGreenRegistry {
+// if we have raoring in the mix we cannot expect reads
+// to match up, but otherwise do.
+func newBlueGreenReg(types []txtype) *blueGreenRegistry {
+
+	hasRoaring := false
+	if types[0] == roaringTxn || types[1] == roaringTxn {
+		hasRoaring = true
+	}
 	return &blueGreenRegistry{
-		m: make(map[int64]*blueGreenTx),
+		m:          make(map[int64]*blueGreenTx),
+		types:      types,
+		hasRoaring: hasRoaring,
 	}
 }
 
-// add remembers the tx so we can check it
-// should we see a write after creation
-// but before rollback/commit.
+// add remembers the tx so we can check that
+// all tx were finished before Close().
 func (b *blueGreenRegistry) add(c *blueGreenTx) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -100,26 +114,33 @@ func (b *blueGreenRegistry) Close() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	if len(b.m) > 0 {
-		panic(fmt.Sprintf("still have unchecked blueGreenTx: '%#v'", b.m))
+		panic(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m))
 		//AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m)
 	}
 }
 
 func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx {
-	//func (reg *blueGreenRegistry) newBlueGreenTx(a, b Tx, idx *Index, o Txo, openTxSn []int64) *blueGreenTx {
 	as := a.Type()
 	bs := b.Type()
-	c := &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs, txf: txf, types: txf.types}
+	c := &blueGreenTx{a: a,
+		b:          b,
+		idx:        idx,
+		as:         as,
+		bs:         bs,
+		txf:        txf,
+		types:      txf.types,
+		hasRoaring: txf.blueGreenReg.hasRoaring,
+		short:      true,
+	}
 
 	if c.types[1] == roaringTxn {
 		c.useSnA = true
 	}
+	//vv("newBlueGreenTx with a.sn=%v with o.Shard=%v", c.Sn(), int(o.Shard))
 
 	c.checker.c = c
 	c.o = o
-	if o.Write {
-		txf.blueGreenReg.add(c)
-	}
+	txf.blueGreenReg.add(c)
 	return c
 }
 
@@ -131,24 +152,27 @@ func (c *blueGreenTx) Type() string {
 
 var blueGreenTxDumpMut sync.Mutex
 
-func (c *blueGreenTx) Dump() {
+func (c *blueGreenTx) Dump(short bool) {
+
 	blueGreenTxDumpMut.Lock()
 	defer blueGreenTxDumpMut.Unlock()
 	fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2))
 	fmt.Printf("A(%v) Dump:\n", c.as)
-	c.a.Dump()
+	c.a.Dump(short)
 	fmt.Printf("B(%v) Dump:\n", c.bs)
-	c.b.Dump()
+	c.b.Dump(short)
 
-	fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx)
-	c.idx.Txf.dbPerShard.DumpAll()
+	if !short {
+		fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx)
+		c.idx.holder.txf.dbPerShard.DumpAll()
+	}
 }
 
 func (c *blueGreenTx) Readonly() bool {
 	a := c.a.Readonly()
 	b := c.b.Readonly()
 	if a != b {
-		panic(fmt.Sprintf("a=%v, but b =%v", a, b))
+		panic(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b))
 	}
 	return b
 }
@@ -172,6 +196,9 @@ func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, chan
 
 // compareTxState is called for the first Commit or Rollback a blueGreenTx sees.
 func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
+	if c.o.blueGreenOff {
+		return
+	}
 	here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
 	//vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID())
 	aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
@@ -184,21 +211,21 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 	}
 
 	if aFound != bFound {
-		c.Dump()
+		c.Dump(c.short)
 		panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack()))
 	}
 
 	if aErr != nil || bErr != nil {
 		if aErr != nil && bErr != nil {
-			c.Dump()
+			c.Dump(c.short)
 			panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack()))
 		}
 		if aErr != nil {
-			c.Dump()
+			c.Dump(c.short)
 			panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack()))
 		}
 		if bErr != nil {
-			c.Dump()
+			c.Dump(c.short)
 			panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack()))
 		}
 	}
@@ -207,17 +234,17 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 
 		if !bIter.Next() {
 			AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, stack())
-			c.Dump()
+			c.Dump(c.short)
 			panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack()))
 		}
 		bKey, bValue := bIter.Value()
 		if bKey != aKey {
 			AlwaysPrintf("problem in caller %v", Caller(2))
-			c.Dump()
+			c.Dump(c.short)
 			panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack()))
 		}
 		if err := aValue.BitwiseCompare(bValue); err != nil {
-			c.Dump()
+			c.Dump(c.short)
 			//vv("compareTxState[%v]: key %v differs: %v;  A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
 			panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v;  A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
 		}
@@ -226,7 +253,7 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 	// end checking everything in A, but does B have more?
 	if bIter.Next() {
 		AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn())
-		c.Dump()
+		c.Dump(c.short)
 		bKey, _ := bIter.Value()
 		panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), stack()))
 	}
@@ -234,10 +261,17 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 }
 
 func (c *blueGreenTx) checkDatabase() {
-	if !c.o.Write {
-		// We only need to check the we are A/B consistent after every write.
-		// Then reads can only see that consitent state, and don't need
-		// to be checked themselves. Sketch of proof by induction:
+	if c.o.blueGreenOff {
+		return
+	}
+	if c.hasRoaring && !c.o.Write {
+		// With roaring on one arm, we only check the we are A/B
+		// consistent after every write.
+		//
+		// Ideally reads can only see that consitent state, and don't need
+		// to be checked themselves-- but we do try if both A and B
+		// are transactional. Sketch of proof by induction that
+		// write checking should, theoretically, suffice:
 		// Starting with zero data, if we have agreement in both A/B
 		// database state after each write, then
 		// because there is only ever a single
@@ -245,7 +279,8 @@ func (c *blueGreenTx) checkDatabase() {
 		// data state between A and B as long as every prior
 		// A/B check of the serialized writes suceeded.
 		//
-		// This avoids a key problem we discovered when A/B checking reads.
+		// This avoids a key problem we discovered when A/B checking reads
+		// with roaring on one arm.
 		// The MVCC of the transactional engines means that reads that
 		// start before a write commit will look very different
 		// when comparing to roaring's non-transactional state.
@@ -259,8 +294,6 @@ func (c *blueGreenTx) checkDatabase() {
 	}
 	c.checker.checkDone = true
 
-	// seen() returns nil on 2nd or any further call,
-	// so only the first Commit() or Rollback() does this.
 	for index, fields := range c.checker.seen() {
 		for field, views := range fields {
 			for view, shards := range views {
@@ -272,6 +305,10 @@ func (c *blueGreenTx) checkDatabase() {
 	}
 }
 
+func (c *blueGreenTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
 func (c *blueGreenTx) Rollback() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -308,7 +345,9 @@ func (c *blueGreenTx) Commit() error {
 	//vv("blueGreenTx.Commit() called. bgtx p=%p", c)
 	c.rollbackOrCommitDone = true
 	if c.o.Write {
-		c.checkDatabase()
+		if !c.o.blueGreenOff {
+			c.checkDatabase()
+		}
 	}
 	defer func() {
 		if r := recover(); r != nil {
@@ -336,14 +375,15 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
 	a, errA := c.a.RoaringBitmap(index, field, view, shard)
 	_, _ = a, errA
 	b, errB := c.b.RoaringBitmap(index, field, view, shard)
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
 
-	slcA := a.Slice()
-	slcB := b.Slice()
-	if !reflect.DeepEqual(slcA, slcB) {
-		panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
+		slcA := a.Slice()
+		slcB := b.Slice()
+		if !reflect.DeepEqual(slcA, slcB) {
+			panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
+		}
 	}
-
 	return b, errB
 }
 
@@ -357,10 +397,12 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
 	}()
 	a, errA := c.a.Container(index, field, view, shard, key)
 	b, errB := c.b.Container(index, field, view, shard, key)
-	compareErrors(errA, errB)
-	err = a.BitwiseCompare(b)
-	panicOn(err)
 
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+		err = a.BitwiseCompare(b)
+		panicOn(err)
+	}
 	return b, errB
 }
 
@@ -374,8 +416,10 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
 	}()
 	errA := c.a.PutContainer(index, field, view, shard, key, rc)
 	errB := c.b.PutContainer(index, field, view, shard, key, rc)
-	compareErrors(errA, errB)
 
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return errB
 }
 
@@ -386,7 +430,7 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
 	// ================== begin save comments.
 	//c.checkDatabase()
 	////vv("got past database check at TOP of ImportRoaringBits")
-	//c.Dump()
+	//c.Dump(c.short)
 	////vv("done with top dump; clear=%v", clear)
 	// ==================   end save comments.
 	defer func() {
@@ -403,28 +447,31 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
 	changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 	changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
 
-	if len(data) == 0 {
-		// okay to check! otherwise we are in the fragment.fillFragmentFromArchive
-		// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
-		// be inaccurate.
-		if changedA != changedB {
-			panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
-		}
-		if len(rowSetA) != len(rowSetB) {
-			panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
-		}
-		for k, va := range rowSetA {
-			vb, ok := rowSetB[k]
-			if !ok {
-				panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
+	if !c.o.blueGreenOff {
+
+		if len(data) == 0 {
+			// okay to check! otherwise we are in the fragment.fillFragmentFromArchive
+			// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
+			// be inaccurate.
+			if changedA != changedB {
+				panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
 			}
-			if va != vb {
-				panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
+			if len(rowSetA) != len(rowSetB) {
+				panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
+			}
+			for k, va := range rowSetA {
+				vb, ok := rowSetB[k]
+				if !ok {
+					panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
+				}
+				if va != vb {
+					panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
+				}
 			}
 		}
+		compareErrors(errA, errB)
+		c.checkDatabase()
 	}
-	compareErrors(errA, errB)
-	c.checkDatabase()
 	return changedB, rowSetB, errB
 }
 
@@ -438,7 +485,10 @@ func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, k
 	}()
 	errA := c.a.RemoveContainer(index, field, view, shard, key)
 	errB := c.b.RemoveContainer(index, field, view, shard, key)
-	compareErrors(errA, errB)
+
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return errB
 }
 
@@ -481,10 +531,14 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool,
 
 	bch, errB := c.b.Add(index, field, view, shard, batched, a2...)
 
-	if ach != bch {
-		panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB))
+	if !c.o.blueGreenOff {
+
+		if ach != bch {
+			panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB))
+		}
+		compareErrors(errA, errB)
 	}
-	compareErrors(errA, errB)
+
 	return bch, errB
 }
 
@@ -516,7 +570,10 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6
 	ach, errA := c.a.Remove(index, field, view, shard, a...)
 	_, _ = ach, errA
 	bch, errB := c.b.Remove(index, field, view, shard, a...)
-	compareErrors(errA, errB)
+
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return bch, errB
 }
 
@@ -532,7 +589,9 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint
 	_, _ = ax, errA
 	bx, errB := c.b.Contains(index, field, view, shard, key)
 
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return bx, errB
 }
 
@@ -550,8 +609,10 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
 
 	bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 
-	compareErrors(errA, errB)
-	// INVAR: errA == errB, so only need to check one.
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
+
 	if errB != nil {
 		// RoaringTx can return an iterator and an error, so be sure Close it we have it.
 		if ait != nil {
@@ -562,6 +623,13 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
 		}
 		return nil, bfound, errB
 	}
+	if errA != nil {
+		// RoaringTx can return an iterator and an error, so be sure Close it we have it.
+		if ait != nil {
+			ait.Close()
+		}
+	}
+
 	// INVAR: errA == errB == nil
 	bgi := NewBlueGreenIterator(c, ait, bit)
 	return bgi, bfound, errB
@@ -598,11 +666,14 @@ func (bgi *blueGreenIterator) Next() bool {
 func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) {
 	ka, ca := bgi.ait.Value()
 	kb, cb := bgi.bit.Value()
-	if ka != kb {
-		panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
+
+	if !bgi.tx.o.blueGreenOff {
+		if ka != kb {
+			panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
+		}
+		err := ca.BitwiseCompare(cb)
+		panicOn(err)
 	}
-	err := ca.BitwiseCompare(cb)
-	panicOn(err)
 	return kb, cb
 }
 func (bgi *blueGreenIterator) Close() {
@@ -611,7 +682,7 @@ func (bgi *blueGreenIterator) Close() {
 }
 
 // ForEach is read-only on the database, and so we only pass through to B.
-// Avoids the side-effects of calling fn too many times.
+// Avoids the side-effects of calling fn too many times, which can cause serious false alarms.
 func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
 	defer func() {
 		if r := recover(); r != nil {
@@ -654,7 +725,9 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er
 	b, errB := c.b.Count(index, field, view, shard)
 	_, _ = b, errB
 
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return b, errB
 }
 
@@ -671,7 +744,9 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro
 	b, errB := c.b.Max(index, field, view, shard)
 	_, _ = b, errB
 
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return b, errB
 }
 
@@ -688,7 +763,9 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool
 	bmin, bfound, errB := c.b.Min(index, field, view, shard)
 	_, _, _ = bmin, bfound, errB
 
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return bmin, bfound, errB
 }
 
@@ -702,7 +779,9 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
 	}()
 	errA := c.a.UnionInPlace(index, field, view, shard, others...)
 	errB := c.b.UnionInPlace(index, field, view, shard, others...)
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 	return errB
 }
 
@@ -710,7 +789,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
 	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
-			c.Dump()
+			c.Dump(c.short)
 			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
@@ -718,11 +797,13 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
 	a, errA := c.a.CountRange(index, field, view, shard, start, end)
 	b, errB := c.b.CountRange(index, field, view, shard, start, end)
 
-	if a != b {
-		panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
-	}
+	if !c.o.blueGreenOff {
+		if a != b {
+			panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
+		}
 
-	compareErrors(errA, errB)
+		compareErrors(errA, errB)
+	}
 	return b, errB
 }
 
@@ -730,19 +811,22 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
 	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
-			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			AlwaysPrintf("see OffsetRange() on _sn_ %v, panic '%v' at '%v'", c.Sn(), r, stack())
 			panic(r)
 		}
 	}()
 	a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end)
 	b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end)
 
-	err = roaringBitmapDiff(a, b)
-	if err != nil {
-		c.Dump()
-		panicOn(err)
+	if !c.o.blueGreenOff {
+
+		err = roaringBitmapDiff(a, b)
+		if err != nil {
+			c.Dump(false)
+			panicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err))
+		}
+		compareErrors(errA, errB)
 	}
-	compareErrors(errA, errB)
 
 	return b, errB
 }
@@ -751,7 +835,7 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
 	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
-			c.Dump()
+			c.Dump(c.short)
 			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
@@ -760,14 +844,19 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
 	rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
 	rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
 
-	compareErrors(errA, errB)
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
+	}
 
 	// We are seeing Roaring vs Badger size differences on
 	// server/ test TestClusterResize_AddNode/ContinuousShards,
 	// so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to
 	// check RBF vs Badger for byte-for-byte compatiblity (we
 	// suspect the ops log or optimized bitmaps are accounting for the difference).
-	sizeMustMatch := false
+	sizeMustMatch := false // !c.hasRoaring
+	if c.o.blueGreenOff {
+		sizeMustMatch = false
+	}
 	if sizeMustMatch {
 		if szA != szB {
 			panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v;  fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring))
@@ -806,41 +895,44 @@ func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string)
 	//c.checker.see(index, field, view, shard) // don't have shard.
 	defer func() {
 		if r := recover(); r != nil {
-			c.Dump()
+			c.Dump(c.short)
 			AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
 	}()
 	slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath)
 	slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath)
-	compareErrors(errA, errB)
 
-	// sort order may be different, and that's ok.
-	cpa := append([]uint64{}, slcA...)
-	cpb := append([]uint64{}, slcB...)
-	sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] })
-	sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] })
+	if !c.o.blueGreenOff {
+		compareErrors(errA, errB)
 
-	if !reflect.DeepEqual(cpa, cpb) {
-		// report the first difference
-		ma := make(map[uint64]bool)
-		for _, ka := range slcA {
-			ma[ka] = true
-		}
-		for _, kb := range slcB {
-			if !ma[kb] {
-				//vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)
-				c.Dump()
-				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
+		// sort order may be different, and that's ok.
+		cpa := append([]uint64{}, slcA...)
+		cpb := append([]uint64{}, slcB...)
+		sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] })
+		sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] })
+
+		if !reflect.DeepEqual(cpa, cpb) {
+			// report the first difference
+			ma := make(map[uint64]bool)
+			for _, ka := range slcA {
+				ma[ka] = true
 			}
-			delete(ma, kb)
-		}
-		if len(ma) != 0 {
-			for firstDifference := range ma {
-				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb))
+			for _, kb := range slcB {
+				if !ma[kb] {
+					//vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)
+					c.Dump(c.short)
+					panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
+				}
+				delete(ma, kb)
 			}
+			if len(ma) != 0 {
+				for firstDifference := range ma {
+					panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb))
+				}
+			}
+			panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb))
 		}
-		panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb))
 	}
 	return slcB, errB
 }

bluegreentx_test.go

@@ -16,13 +16,20 @@ package pilosa
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
+	"os"
+	"strings"
 	"testing"
 
 	cryrand "crypto/rand"
 )
 
+var _ = context.Background
+var _ = os.Open
+var _ = strings.Split
+
 func TestMultiReaderB(t *testing.T) {
 	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
 	// member io.Readers, else it should panic. This should hold for

catcher.go

@@ -58,8 +58,8 @@ func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, r
 	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 }
 
-func (c *catcherTx) Dump() {
-	c.b.Dump()
+func (c *catcherTx) Dump(short bool) {
+	c.b.Dump(short)
 }
 
 func (c *catcherTx) Readonly() bool {
@@ -145,6 +145,10 @@ func (c *catcherTx) UseRowCache() bool {
 	return c.b.UseRowCache()
 }
 
+func (c *catcherTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
 func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
 
 	defer func() {

cluster.go

@@ -62,7 +62,7 @@ const (
 	resizeJobActionAdd    = "ADD"
 	resizeJobActionRemove = "REMOVE"
 
-	defaultConfirmDownRetries = 10
+	defaultConfirmDownRetries = 120
 	defaultConfirmDownSleep   = 1 * time.Second
 )
 
@@ -758,7 +758,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost {
 			fieldViews.addView(field.Name(), view.name)
 		}
 	}
-	return c.fragCombos(idx.Name(), idx.AvailableShards(), fieldViews)
+	return c.fragCombos(idx.Name(), idx.AvailableShards(includeRemote), fieldViews)
 }
 
 // fragCombos returns a map (by uri) of lists of fragments for a given index
@@ -2218,7 +2218,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
 		is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
 		for _, f := range idx.Fields {
 			if field := c.holder.Field(idx.Name, f.Name); field != nil {
-				availableShards = field.AvailableShards()
+				availableShards = field.AvailableShards(includeRemote)
 			} else {
 				availableShards = roaring.NewBitmap()
 			}

cluster_internal_test.go

@@ -97,6 +97,7 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
 		panic(err)
 	}
 	h := NewHolder(path, nil)
+	panicOn(h.Open())
 	index, err := h.CreateIndex(name, IndexOptions{})
 	testhook.Cleanup(tb, func() {
 		h.Close()
@@ -170,7 +171,7 @@ func TestFragSources(t *testing.T) {
 
 	// Obtain transaction.
 	var shard uint64
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = field.SetBit(tx, 1, 101, nil)
@@ -180,7 +181,7 @@ func TestFragSources(t *testing.T) {
 	panicOn(tx.Commit())
 
 	shard = 1
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 	defer tx.Rollback()
 	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
@@ -189,7 +190,7 @@ func TestFragSources(t *testing.T) {
 	panicOn(tx.Commit())
 
 	shard = 2
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
@@ -199,7 +200,7 @@ func TestFragSources(t *testing.T) {
 	panicOn(tx.Commit())
 
 	shard = 3
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)

dbshard.go

@@ -22,6 +22,9 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
 )
 
 var _ = sort.Sort
@@ -50,6 +53,9 @@ type DBWrapper interface {
 	DeleteField(index, field, fieldPath string) error
 	OpenListString() string
 	OpenSnList() (sns []int64)
+	Path() string
+	HasData() (has bool, err error)
+	SetHolder(h *Holder)
 }
 
 type DBRegistry interface {
@@ -68,7 +74,9 @@ type DBShard struct {
 	// by a reader to start with.
 	mut sync.RWMutex
 
-	types         []txtype
+	types      []txtype
+	hasRoaring bool // if either of the types is roaringTxn
+
 	W             []DBWrapper
 	ParentDBIndex *DBIndex
 
@@ -76,6 +84,7 @@ type DBShard struct {
 	per *DBPerShard
 
 	useOpenList int
+	closed      bool
 }
 
 func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) {
@@ -105,6 +114,7 @@ func (dbs *DBShard) Close() (err error) {
 			return err
 		}
 	}
+	dbs.closed = true
 	return
 }
 
@@ -112,10 +122,49 @@ func (dbs *DBShard) String() string {
 	return dbs.Path
 }
 
-func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
-	dbs.mut.Lock()
-	defer dbs.mut.Unlock()
+// Cleanup must be called at every commit/rollback of a Tx, in
+// order to release the read-write mutex that guarantees a single
+// writer at a time. Each tx must take care to call cleanup()
+// exactly once. examples:
+//   tx.o.dbs.Cleanup(tx)
+//   tx.Options().dbs.Cleanup(tx)
+//
+func (dbs *DBShard) Cleanup(tx Tx) {
+	if dbs == nil {
+		return // some tests are using Tx only, no dbs available.
+	}
+	if useRWLock {
+		if !dbs.hasRoaring {
+			if tx.Readonly() {
+				dbs.mut.RUnlock()
+			} else {
+				dbs.mut.Unlock()
+			}
+		}
+	}
+}
 
+// experimental feature, off for now.
+const useRWLock = false
+
+func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
+	if useRWLock {
+		// enforce only one writer at a time. The dbs.mut is held until
+		// the Tx finishes.
+		if !dbs.hasRoaring {
+			if write {
+				dbs.mut.Lock()
+			} else {
+				dbs.mut.RLock()
+			}
+		}
+	}
+	if o.dbs != dbs {
+		panic(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs))
+	}
+	if o.Shard != dbs.Shard {
+		panic(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)))
+	}
 	var txns []Tx
 
 	for _, w := range dbs.W {
@@ -153,7 +202,8 @@ type DBPerShard struct {
 	// Easily see how many we have.
 	Flatmap map[*DBShard]struct{}
 
-	types []txtype
+	types      []txtype
+	hasRoaring bool
 
 	txf *TxFactory
 
@@ -163,6 +213,25 @@ type DBPerShard struct {
 	useOpenList int
 }
 
+// HasData returns true if the database has at least one key.
+// For roaring it returns the number of fragments stored.
+// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green.
+// If you pass 1, be sure you have a blue-green configuration.
+func (per *DBPerShard) HasData(which int) (hasData bool, err error) {
+	// has to aggregate across all available DBShard for each index and shard.
+
+	for v := range per.Flatmap {
+		hasData, err = v.W[which].HasData()
+		if err != nil {
+			return
+		}
+		if hasData {
+			return
+		}
+	}
+	return
+}
+
 func (per *DBPerShard) ListOpenString() (r string) {
 	for v := range per.Flatmap {
 		r += v.Path + " -> " + v.W[per.useOpenList].OpenListString() + "\n"
@@ -173,12 +242,19 @@ func (per *DBPerShard) ListOpenString() (r string) {
 func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string) (d *DBPerShard) {
 
 	useOpenList := 0
+	hasRoaring := false
+	if types[0] == roaringTxn {
+		hasRoaring = true
+	}
 	if len(types) == 2 {
 		// blue-green, avoid the empty roaring Tx open list.
 		// Prefer B's open list if neither is roaring.
 		if types[0] == roaringTxn || types[1] != roaringTxn {
 			useOpenList = 1
 		}
+		if types[1] == roaringTxn {
+			hasRoaring = true
+		}
 	}
 
 	d = &DBPerShard{
@@ -188,6 +264,7 @@ func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string) (d *DBPerS
 		Flatmap:     make(map[*DBShard]struct{}),
 		txf:         txf,
 		useOpenList: useOpenList,
+		hasRoaring:  hasRoaring,
 	}
 	return
 }
@@ -239,28 +316,33 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err
 
 func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
 
-	dbs, err := per.GetDBShard(index, shard, nil)
+	idx := per.txf.holder.Index(index)
+	dbs, err := per.GetDBShard(index, shard, idx)
 	panicOn(err)
 	return dbs.DeleteFragment(index, field, view, shard, frag)
 }
 
 func (dbs *DBShard) DumpAll() {
-
+	short := false
+	fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard))
 	for i, ty := range dbs.types {
 		_ = i
 		tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx})
 		panicOn(err)
 		defer tx.Rollback()
-		tx.Dump()
+		fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty)
+		tx.Dump(short)
 
 		switch ty {
 		case roaringTxn:
 		case rbfTxn:
 		case lmdbTxn:
+		case badgerTxn:
 		default:
 			panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
 		}
 	}
+	fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard))
 }
 
 func (per *DBPerShard) DumpAll() {
@@ -285,6 +367,7 @@ func (per *DBPerShard) Path(index string, shard uint64) string {
 }
 
 func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
+
 	per.Mu.Lock()
 	defer per.Mu.Unlock()
 
@@ -296,8 +379,10 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
 		per.dbh.Index[index] = dbi
 	}
 	dbs, ok = dbi.Shard[shard]
+	if dbs != nil && dbs.closed {
+		panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs))
+	}
 	if !ok {
-
 		dbs = &DBShard{
 			types:         per.types,
 			ParentDBIndex: dbi,
@@ -307,6 +392,7 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
 			idx:           idx,
 			per:           per,
 			useOpenList:   per.useOpenList,
+			hasRoaring:    per.hasRoaring,
 		}
 		dbi.Shard[shard] = dbs
 	}
@@ -320,11 +406,15 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
 				registry = globalRbfDBReg
 			case lmdbTxn:
 				registry = globalLMDBReg
+			case badgerTxn:
+				registry = globalBadgerReg
 			default:
 				panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
 			}
 			w, err := registry.OpenDBWrapper(dbs.Path, DetectMemAccessPastTx)
 			panicOn(err)
+			h := idx.Holder()
+			w.SetHolder(h)
 			dbs.Open = true
 			if w != nil && len(dbs.W) == 0 {
 				per.Flatmap[dbs] = struct{}{}
@@ -364,18 +454,39 @@ func (per *DBPerShard) Close() (err error) {
 	return
 }
 
-// requiredSuffix should be "-badgerdb" for badger, etc.
+// DBPerShardGetShardsForIndex returns the indexes from the B (green) database if
+// blue-green comparison is in use, rather than from the A (blue) database.
 func DBPerShardGetShardsForIndex(idx *Index, roaringViewPath string) (sliceOfShards []uint64, err error) {
 
 	// follow the blueGreen convention of returning the answer for 'B' or
 	// the last wrapper type.
 	types := idx.holder.txf.Types()
 	ty := types[len(types)-1]
+	return TypedDBPerShardGetLocalShardsForIndex(ty, idx, roaringViewPath)
+}
+
+// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover
+// all the view paths under idx for type ty.
+func TypedDBPerShardGetLocalShardsForIndex(ty txtype, idx *Index, roaringViewPath string) (sliceOfShards []uint64, err error) {
 
 	if ty == roaringTxn {
 		rx := &RoaringTx{
 			Index: idx,
 		}
+		if roaringViewPath == "" {
+			for _, field := range idx.Fields() {
+				for _, view := range field.views() {
+					sos, err := rx.SliceOfShards("", "", "", view.path)
+					if err != nil {
+						return nil,
+							errors.Wrap(err, fmt.Sprintf(
+								"TypedDBPerShardGetLocalShardsForIndex roaringTxn view.path='%v'", view.path))
+					}
+					sliceOfShards = append(sliceOfShards, sos...)
+				}
+			}
+			return dedupShardSlice(sliceOfShards), nil
+		}
 		return rx.SliceOfShards("", "", "", roaringViewPath)
 	}
 	requiredSuffix := ty.FileSuffix()
@@ -435,3 +546,192 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor
 	})
 	return
 }
+
+// populateBlueFromGreen prepares for a blue_green run at startup time.
+//
+// It is called at the end of Holder.Open(). This allows the application
+// of blue-green checking to pilosa instances that
+// were previously run only with a single (solo) backend.
+//
+// PRE: This operation requires, at its start, either:
+//
+// (1) an empty blue database -- this allows transitioning from
+//     a solo database to blue_green checking where the solo
+//     becomes the green; or
+//
+// (2) that the blue data, if present, be logically
+//     identical to the green data -- this allows one to restart
+//     a pilosa that was already running in blue_green mode
+//     and remain in blue_green mode.
+//
+// In either case, the goal to to finish populateBlueFromGreen()
+// and have the exact same logical set of data in both backends.
+//
+// Why must the data be identical after Holder.Open() finishes?
+// Otherwise subsequent blue-green checks have no hope of
+// being accurate.
+//
+// The blue is the destination -- this is always types[0].
+// The green source is always types[1]. The mnemonic is blue_geen.
+// The blue is first, so it is in types[0]. The green
+// is second, in types[1]. For example, with PILOSA_TXSRC=lmdb_roaring
+// we have lmdb as blue, and roaring as green. The contents of
+// lmdb must be empty or exactly match roaring. If lmdb
+// starts empty, it will be populated from roaring by
+// populateBlueFromGreen().
+//
+func (dbs *DBShard) populateBlueFromGreen() (err error) {
+	n := len(dbs.W)
+	if n != 2 {
+		panic(fmt.Sprintf("copyGreenToBlue did not find 2 open DBs: have %v", n))
+	}
+
+	dest := dbs.W[0] // blue
+	src := dbs.W[1]  // green
+
+	// copy all the key/container pairs.
+	// Since a shard is fairly small, we think one Tx will suffice.
+
+	readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer readtx.Rollback()
+
+	writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer writetx.Rollback()
+
+	for _, fld := range dbs.idx.Fields() {
+		field := fld.Name()
+		for _, vw := range fld.views() {
+			view := vw.name
+			citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				return errors.Wrap(err, "DBShard.copyGreenToBlue readtx.ContainerIterator")
+			}
+
+			for citer.Next() {
+				ckey, rc := citer.Value()
+				err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc)
+				if err != nil {
+					citer.Close()
+					return errors.Wrap(err, "DBShard.copyGreenToBlue writetx.PutContainer")
+				}
+			}
+			citer.Close()
+		}
+	}
+	err = writetx.Commit()
+	if err != nil {
+		return errors.Wrap(err, "writetx.Commit()")
+	}
+	return nil
+}
+
+// verifyBlueEqualsGreen checks that blue and green are identical.
+func (dbs *DBShard) verifyBlueEqualsGreen(numCtVerified *int64) (err error) {
+
+	n := len(dbs.W)
+	if n != 2 {
+		panic(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n))
+	}
+
+	blue := dbs.W[0]
+	green := dbs.W[1]
+
+	greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer greentx.Rollback()
+
+	bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer bluetx.Rollback()
+
+	for _, fld := range dbs.idx.Fields() {
+		field := fld.Name()
+		for _, vw := range fld.views() {
+
+			view := vw.name
+			gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator")
+			}
+
+			bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				gCiter.Close()
+				if bCiter != nil {
+					bCiter.Close()
+				}
+				return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator")
+			}
+
+			for gCiter.Next() {
+				greenCkey, greenc := gCiter.Value()
+
+				if !bCiter.Next() {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees missing blue container at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' the greenCkey: '%v'",
+						dbs.Index, field, view, dbs.Shard, greenCkey))
+				}
+				blueCkey, bluec := bCiter.Value()
+
+				if blueCkey != greenCkey {
+					bCiter.Close()
+					gCiter.Close()
+					return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+
+						"difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v'",
+						blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard)
+				}
+				nGreen := greenc.N()
+				nBlue := bluec.N()
+				if nBlue != nGreen {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v",
+						dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue))
+				}
+				err = bluec.BitwiseCompare(greenc)
+				if err != nil {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'",
+						dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err))
+				}
+				atomic.AddInt64(numCtVerified, 1)
+
+			}
+			if bCiter.Next() {
+				blueCkey, _ := bCiter.Value()
+				bCiter.Close()
+				gCiter.Close()
+				return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+					"sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+
+					"shard: '%v' the ckey: '%v'",
+					dbs.Index, field, view, dbs.Shard, blueCkey))
+			}
+			bCiter.Close()
+			gCiter.Close()
+		}
+	}
+
+	return nil
+}
+
+func dedupShardSlice(sos []uint64) (r []uint64) {
+	m := make(map[uint64]struct{})
+	for _, s := range sos {
+		m[s] = struct{}{}
+	}
+	for k := range m {
+		r = append(r, k)
+	}
+	return
+}

dbshard_internal_test.go

@@ -54,7 +54,7 @@ func TestShardPerDB_SetBit(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	if n := f.mustRow(tx, 120).Count(); n != 2 {
@@ -72,12 +72,13 @@ func Test_DBPerShard_GetShardsForIndex(t *testing.T) {
 	orig := os.Getenv("PILOSA_TXSRC")
 	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
 
-	for _, src := range []string{"lmdb", "roaring", "rbf"} {
-		makeSampleRoaringDir(tmpdir, src)
+	for _, src := range []string{"lmdb", "roaring", "badger", "rbf"} {
+		makeSampleRoaringDir(tmpdir, src, 0)
 		os.Setenv("PILOSA_TXSRC", src)
 
 		// must make Holder AFTER setting src.
 		holder := NewHolder(tmpdir, nil)
+
 		idx, err := NewIndex(holder, tmpdir, "rick")
 		panicOn(err)
 		estd := "rick/_exists/views/standard"
@@ -100,6 +101,7 @@ func Test_DBPerShard_GetShardsForIndex(t *testing.T) {
 				}
 			}
 		}
+		holder.Close()
 	}
 }
 
@@ -135,36 +137,62 @@ rick/_exists/views/standard/fragments/219
 rick/_exists/views/standard/fragments/223
 `,
 	"lmdb": `
-rick/0219-lmdb/data.mdb
-rick/0219-lmdb/lock.mdb
-rick/0093-lmdb/data.mdb
-rick/0093-lmdb/lock.mdb
-rick/0223-lmdb/data.mdb
-rick/0223-lmdb/lock.mdb
-rick/0215-lmdb/data.mdb
-rick/0215-lmdb/lock.mdb
-rick/0217-lmdb/data.mdb
-rick/0217-lmdb/lock.mdb
-rick/0221-lmdb/data.mdb
-rick/0221-lmdb/lock.mdb
+rick/0219-lmdb@/data.mdb
+rick/0219-lmdb@/lock.mdb
+rick/0093-lmdb@/data.mdb
+rick/0093-lmdb@/lock.mdb
+rick/0223-lmdb@/data.mdb
+rick/0223-lmdb@/lock.mdb
+rick/0215-lmdb@/data.mdb
+rick/0215-lmdb@/lock.mdb
+rick/0217-lmdb@/data.mdb
+rick/0217-lmdb@/lock.mdb
+rick/0221-lmdb@/data.mdb
+rick/0221-lmdb@/lock.mdb
+`,
+	"badger": `
+rick/0219-badgerdb@/000000.vlog
+rick/0219-badgerdb@/KEYREGISTRY
+rick/0219-badgerdb@/MANIFEST
+rick/0219-badgerdb@/LOCK
+rick/0221-badgerdb@/000000.vlog
+rick/0221-badgerdb@/KEYREGISTRY
+rick/0221-badgerdb@/MANIFEST
+rick/0221-badgerdb@/LOCK
+rick/0223-badgerdb@/000000.vlog
+rick/0223-badgerdb@/KEYREGISTRY
+rick/0223-badgerdb@/MANIFEST
+rick/0223-badgerdb@/LOCK
+rick/0093-badgerdb@/000000.vlog
+rick/0093-badgerdb@/KEYREGISTRY
+rick/0093-badgerdb@/MANIFEST
+rick/0093-badgerdb@/LOCK
+rick/0217-badgerdb@/000000.vlog
+rick/0217-badgerdb@/KEYREGISTRY
+rick/0217-badgerdb@/MANIFEST
+rick/0217-badgerdb@/LOCK
+rick/0215-badgerdb@/000000.vlog
+rick/0215-badgerdb@/KEYREGISTRY
+rick/0215-badgerdb@/MANIFEST
+rick/0215-badgerdb@/LOCK
 `,
 	"rbf": `
-rick/0223-rbfdb/wal/0000000000000001.wal
-rick/0223-rbfdb/data
-rick/0093-rbfdb/wal/0000000000000001.wal
-rick/0093-rbfdb/data
-rick/0217-rbfdb/wal/0000000000000001.wal
-rick/0217-rbfdb/data
-rick/0215-rbfdb/wal/0000000000000001.wal
-rick/0215-rbfdb/data
-rick/0221-rbfdb/wal/0000000000000001.wal
-rick/0221-rbfdb/data
-rick/0219-rbfdb/wal/0000000000000001.wal
-rick/0219-rbfdb/data
+rick/0223-rbfdb@/wal/0000000000000001.wal
+rick/0223-rbfdb@/data
+rick/0093-rbfdb@/wal/0000000000000001.wal
+rick/0093-rbfdb@/data
+rick/0217-rbfdb@/wal/0000000000000001.wal
+rick/0217-rbfdb@/data
+rick/0215-rbfdb@/wal/0000000000000001.wal
+rick/0215-rbfdb@/data
+rick/0221-rbfdb@/wal/0000000000000001.wal
+rick/0221-rbfdb@/data
+rick/0219-rbfdb@/wal/0000000000000001.wal
+rick/0219-rbfdb@/data
 `,
 }
 
-func makeSampleRoaringDir(root, txsrc string) {
+func makeSampleRoaringDir(root, txsrc string, minBytes int) {
 	fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
 	for _, fn := range fns {
 		if fn == "" {
@@ -174,6 +202,10 @@ func makeSampleRoaringDir(root, txsrc string) {
 		panicOn(os.MkdirAll(path, 0755))
 		fd, err := os.Create(root + sep + fn)
 		panicOn(err)
+		if minBytes > 0 {
+			_, err := fd.Write(make([]byte, minBytes))
+			panicOn(err)
+		}
 		fd.Close()
 	}
 }

dbshard_test.go

@@ -18,9 +18,7 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"reflect"
-	"strings"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
@@ -39,7 +37,8 @@ func skipForNonLMDB(t *testing.T) {
 	}
 }
 
-var _ = skipForNonLMDB // happy linter
+var _ = skipForNonLMDB   // happy linter
+var _ = skipForNonBadger // happy linter
 
 func skipForNonBadger(t *testing.T) {
 	src := os.Getenv("PILOSA_TXSRC")
@@ -50,7 +49,7 @@ func skipForNonBadger(t *testing.T) {
 
 // Can't write it all to one shard like we do (did).
 func Test_DBPerShard_multiple_shards_used(t *testing.T) {
-	skipForNonBadger(t)
+	skipForNonLMDB(t)
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
 	hldr := c.GetHolder(0)
@@ -62,8 +61,8 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) {
 	hldr.SetBit(index, "general", 11, 2)
 	hldr.SetBit(index, "general", 11, ShardWidth+2)
 
-	//tx_suffix := "-lmdb"
-	tx_suffix := "-badgerdb"
+	types := pilosa.MustTxsrcToTxtype("lmdb")
+	tx_suffix := types[0].FileSuffix()
 	root := hldr.Path() + sep + index
 	shards := []string{"0000", "0001", "0002"}
 	pathShard := []string{}
@@ -75,7 +74,7 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) {
 		if !DirExists(pathShard[i]) {
 			panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i]))
 		}
-		sz, err := DiskUse(pathShard[i], "")
+		sz, err := pilosa.DiskUse(pathShard[i], "")
 		panicOn(err)
 
 		if sz < 100 {
@@ -90,28 +89,6 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) {
 	}
 }
 
-func DiskUse(root string, requiredSuffix string) (tot int, err error) {
-	if !DirExists(root) {
-		return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
-	}
-
-	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-		if info == nil {
-			panic(fmt.Sprintf("info was nil for path = '%v'", path))
-		}
-		if info.IsDir() {
-			// skip directories.
-		} else {
-			sz := info.Size()
-			if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
-				tot += int(sz)
-			}
-		}
-		return nil
-	})
-	return
-}
-
 func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
 
 	c := test.MustRunCluster(t, 1,

diagnostics.go

@@ -229,7 +229,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 	timeQuantumEnabled := false
 
 	for _, index := range d.server.holder.Indexes() {
-		numShards += index.AvailableShards().Count()
+		numShards += index.AvailableShards(includeRemote).Count()
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++

executor.go

@@ -506,7 +506,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
 		if idx == nil {
 			return nil, newNotFoundError(ErrIndexNotFound, index)
 		}
-		shards = idx.AvailableShards().Slice()
+		shards = idx.AvailableShards(includeRemote).Slice()
 		if len(shards) == 0 {
 			shards = []uint64{0}
 		}
@@ -780,7 +780,7 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
 		if idx == nil {
 			return nil, newNotFoundError(ErrIndexNotFound, index)
 		}
-		shards = idx.AvailableShards().Slice()
+		shards = idx.AvailableShards(includeRemote).Slice()
 		if len(shards) == 0 {
 			shards = []uint64{0}
 		}
@@ -4808,7 +4808,7 @@ func (e *executor) translateCalls(ctx context.Context, defaultIndexName string,
 	// Perform a separate batch translation for each separate index used.
 	keyMaps := make(map[string]map[string]uint64)
 	for indexName, keySet := range keySets {
-		idx := e.Holder.indexes[indexName]
+		idx := e.Holder.Index(indexName)
 		if idx == nil {
 			return fmt.Errorf("cannot find index %q", indexName)
 		}
@@ -4851,8 +4851,8 @@ func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c *
 
 	// Collect foreign index keys.
 	if fieldName != "" {
-		idx, exists := e.Holder.indexes[indexName]
-		if !exists {
+		idx := e.Holder.Index(indexName)
+		if idx == nil {
 			return newNotFoundError(ErrIndexNotFound, indexName)
 		}
 		if field := idx.Field(fieldName); field != nil && field.ForeignIndex() != "" {
@@ -4899,8 +4899,8 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C
 
 	// Translate column key.
 	colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel)
-	idx, exists := e.Holder.indexes[indexName]
-	if !exists {
+	idx := e.Holder.Index(indexName)
+	if idx == nil {
 		return newNotFoundError(ErrIndexNotFound, indexName)
 	}
 	if idx.Keys() {

executor_internal_test.go

@@ -155,7 +155,7 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
 	}
 
 	shard := uint64(0)
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 	defer tx.Rollback()
 
 	fb, errb := idx.CreateField("b", OptFieldTypeBool())

executor_test.go

@@ -936,7 +936,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
 		// Obtain transaction.
 		idx := index.Index
 		shard := uint64(0)
-		tx := idx.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idx, Shard: shard})
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: !writable, Index: idx, Shard: shard})
 		defer tx.Rollback()
 
 		f := hldr.Field("i", "f")

field.go

@@ -412,11 +412,31 @@ func (f *Field) TranslateStore() TranslateStore {
 func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore }
 
 // AvailableShards returns a bitmap of shards that contain data.
-func (f *Field) AvailableShards() *roaring.Bitmap {
+func (f *Field) AvailableShards(localOnly bool) *roaring.Bitmap {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 
-	b := f.remoteAvailableShards.Clone()
+	var b *roaring.Bitmap
+	if localOnly {
+		b = roaring.NewBitmap()
+	} else {
+		b = f.remoteAvailableShards.Clone()
+	}
+	for _, view := range f.viewMap {
+		//b.Union(view.availableShards())
+		b.UnionInPlace(view.availableShards())
+	}
+	return b
+}
+
+// LocalAvailableShards returns a bitmap of shards that contain data, but
+// only from the local node. This prevents txfactory from making
+// db-per-shard for remote shards.
+func (f *Field) LocalAvailableShards() *roaring.Bitmap {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+
+	b := roaring.NewBitmap()
 	for _, view := range f.viewMap {
 		//b.Union(view.availableShards())
 		b.UnionInPlace(view.availableShards())
@@ -687,7 +707,7 @@ func (f *Field) applyTranslateStore() error {
 	// In the case where the field has a foreign index, set
 	// the usesKeys value accordingly.
 	if foreignIndexName := f.ForeignIndex(); foreignIndexName != "" {
-		if foreignIndex := f.holder.indexes[foreignIndexName]; foreignIndex != nil {
+		if foreignIndex := f.holder.Index(foreignIndexName); foreignIndex != nil {
 			f.usesKeys = foreignIndex.Keys()
 		}
 	}
@@ -1483,7 +1503,7 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err
 	if view.idx == nil {
 		panic("view.idx should not be nil")
 	}
-	view.holder.addIndexFromField(view.idx)
+	view.holder.addIndex(view.idx)
 
 	return view.setValue(tx, columnID, bsig.BitDepth, baseValue)
 }
@@ -1571,7 +1591,7 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
 
 	var localTx Tx
 	if NilInside(tx) {
-		localTx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard})
+		localTx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard})
 		defer localTx.Rollback()
 	} else {
 		localTx = tx

field_internal_test.go

@@ -208,6 +208,8 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
 		t.Fatal(err)
 	}
 	h := NewHolder(path, nil)
+	panicOn(h.Open())
+
 	idx, err := h.CreateIndex("i", IndexOptions{})
 	if err != nil {
 		panic(err)
@@ -232,7 +234,7 @@ func OpenField(t *testing.T, opts FieldOption) *TestField {
 // Close closes the field and removes the underlying data.
 func (f *TestField) Close() error {
 	if f.idx != nil {
-		panicOn(f.idx.Txf.CloseIndex(f.idx))
+		panicOn(f.idx.holder.txf.CloseIndex(f.idx))
 	}
 	defer os.RemoveAll(f.Path())
 	return f.Field.Close()
@@ -245,7 +247,7 @@ func (f *TestField) Reopen() error {
 		f.parent = nil
 		return err
 	}
-	if err := f.parent.Open(false); err != nil {
+	if err := f.parent.Open(); err != nil {
 		f.parent = nil
 		return err
 	}
@@ -318,7 +320,7 @@ func TestField_RowTime(t *testing.T) {
 	defer f.Close()
 
 	// Obtain transaction.
-	tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
+	tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
 	defer tx.Rollback()
 
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
@@ -334,7 +336,7 @@ func TestField_RowTime(t *testing.T) {
 	panicOn(tx.Commit())
 
 	// obtain 2nd transaction to read it back.
-	tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
+	tx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
 	defer tx.Rollback()
 
 	if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
@@ -574,7 +576,7 @@ func TestBSIGroup_importValue(t *testing.T) {
 	f := OpenField(t, OptFieldTypeInt(-100, 200))
 	defer f.Close()
 
-	qcx := f.idx.Txf.NewQcx()
+	qcx := f.idx.holder.txf.NewQcx()
 	defer qcx.Abort()
 
 	options := &ImportOptions{}
@@ -620,7 +622,7 @@ func TestIntField_MinMaxForShard(t *testing.T) {
 	f := OpenField(t, OptFieldTypeInt(-100, 200))
 	defer f.Close()
 
-	qcx := f.idx.Txf.NewQcx()
+	qcx := f.idx.holder.txf.NewQcx()
 	defer qcx.Abort()
 
 	options := &ImportOptions{}
@@ -679,7 +681,7 @@ func TestIntField_MinMaxForShard(t *testing.T) {
 			panicOn(qcx.Finish())
 
 			shard := uint64(0)
-			tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
 			// Rollback below manually, because we are in a loop.
 
 			maxvc, err := f.MaxForShard(tx, shard, nil)
@@ -705,6 +707,7 @@ func TestIntField_MinMaxForShard(t *testing.T) {
 // Ensure we get errors when they are expected.
 func TestDecimalField_MinMaxBoundaries(t *testing.T) {
 	th := newTestHolder(t)
+	defer th.Close()
 	for i, test := range []struct {
 		scale  int64
 		min    pql.Decimal
@@ -785,7 +788,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
 	f := OpenField(t, OptFieldTypeDecimal(3))
 	defer f.Close()
 
-	qcx := f.idx.Txf.NewQcx()
+	qcx := f.idx.holder.txf.NewQcx()
 	defer qcx.Abort()
 
 	options := &ImportOptions{}
@@ -843,7 +846,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
 			}
 
 			shard := uint64(0)
-			tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
 			defer tx.Rollback()
 
 			maxvc, err := f.MaxForShard(tx, shard, nil)
@@ -869,7 +872,7 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
 	f := OpenField(t, OptFieldTypeInt(-100, 200))
 	defer f.Close()
 
-	qcx := f.idx.Txf.NewQcx()
+	qcx := f.idx.holder.txf.NewQcx()
 	defer qcx.Abort()
 
 	options := &ImportOptions{}

field_test.go

@@ -205,6 +205,8 @@ func TestField_NameValidation(t *testing.T) {
 	}
 }
 
+const includeRemote = false // for calls to Index.AvailableShards(localOnly bool)
+
 // Ensure can update and delete available shards.
 func TestField_AvailableShards(t *testing.T) {
 	idx := test.MustOpenIndex(t)
@@ -223,7 +225,7 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(err)
 	} else if _, err := f.SetBit(tx, 0, ShardWidth*2, nil); err != nil {
 		t.Fatal(err)
-	} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	} else if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -231,7 +233,7 @@ func TestField_AvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
 		t.Fatalf("adding remote shards: %v", err)
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 1, 2, 4}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -242,7 +244,7 @@ func TestField_AvailableShards(t *testing.T) {
 			t.Fatalf("removing shard %d: %v", i, err)
 		}
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 }

fragment.go

@@ -173,8 +173,9 @@ type fragment struct {
 // newFragment returns a new instance of Fragment.
 func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment {
 	idx := holder.Index(index)
+
 	if idx == nil {
-		panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder))
+		panic(fmt.Sprintf("got nil idx back for '%v' from holder!", index))
 	}
 	f := &fragment{
 		path:  path,
@@ -478,7 +479,7 @@ func (f *fragment) openCache() error {
 		return nil
 	}
 
-	tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f, Shard: f.shard})
+	tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Read in all rows by ID.
@@ -996,7 +997,7 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})
+		tx = f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})
 		defer func() {
 			if err == nil {
 				panicOn(tx.Commit())
@@ -1996,9 +1997,9 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) {
 
 	idx := f.holder.Index(f.index)
 	if idx == nil {
-		panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'; f.holder.indexes='%#v'\n", f.index, f.holder.indexes))
+		panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'\n", f.index))
 	}
-	tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 	// no Commit below, b/c is read-only.
 
@@ -2083,7 +2084,7 @@ func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64, err error) {
 	defer f.mu.Unlock()
 
 	idx := f.holder.Index(f.index)
-	tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Shard: f.shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: f.shard})
 	defer tx.Rollback()
 	// readonly, so no Commit()
 
@@ -2785,7 +2786,7 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) {
 // used in shipping the slices across the network for a resize.
 func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
 
-	tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard})
+	tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard})
 	defer tx.Rollback()
 	file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path)
 	if err != nil {
@@ -2859,7 +2860,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
 		switch hdr.Name {
 		case "data":
 			idx := f.holder.Index(f.index)
-			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 			if err := f.fillFragmentFromArchive(tx, tr); err != nil {
 				return 0, errors.Wrap(err, "reading storage")
@@ -3639,7 +3640,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
 	}
 
 	idx := f.holder.Index(f.index)
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: f.shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Merge blocks together.

fragment_internal_test.go

@@ -81,7 +81,7 @@ func TestFragment_SetBit(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	if n := f.mustRow(tx, 120).Count(); n != 2 {
@@ -114,7 +114,7 @@ func TestFragment_ClearBit(t *testing.T) {
 	// In that spirit, we will check that the Tx Commit is visible afterwards.
 	panicOn(tx.Commit())
 
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Close and reopen the fragment & verify the data.
@@ -199,7 +199,7 @@ func TestFragment_ClearRow(t *testing.T) {
 		t.Fatalf("unexpected count: %d", n)
 	}
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Close and reopen the fragment & verify the data.
@@ -246,7 +246,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Verify data on row.
@@ -259,7 +259,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Close and reopen the fragment & verify the data.
@@ -304,7 +304,7 @@ func TestFragment_SetValue(t *testing.T) {
 		if err := tx.Commit(); err != nil {
 			t.Fatal(err)
 		}
-		tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		// Read value.
@@ -356,7 +356,7 @@ func TestFragment_SetValue(t *testing.T) {
 		if err := tx.Commit(); err != nil {
 			t.Fatal(err)
 		}
-		tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		if value, exists, err := f.value(tx, 100, 16); err != nil {
@@ -401,7 +401,7 @@ func TestFragment_SetValue(t *testing.T) {
 		if err := tx.Commit(); err != nil {
 			t.Fatal(err)
 		}
-		tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		if value, exists, err := f.value(tx, 100, 16); err != nil {
@@ -478,7 +478,7 @@ func TestFragment_SetValue(t *testing.T) {
 			if err := tx.Commit(); err != nil {
 				t.Fatal(err)
 			}
-			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Ensure values are set.
@@ -525,7 +525,7 @@ func TestFragment_Sum(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	t.Run("NoFilter", func(t *testing.T) {
@@ -549,7 +549,7 @@ func TestFragment_Sum(t *testing.T) {
 	})
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// verify that clearValue clears values
@@ -558,7 +558,7 @@ func TestFragment_Sum(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	t.Run("ClearValue", func(t *testing.T) {
@@ -599,7 +599,7 @@ func TestFragment_MinMax(t *testing.T) {
 	panicOn(tx.Commit())
 
 	// the new tx is shared by Min/Max below.
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	t.Run("Min", func(t *testing.T) {
@@ -1197,7 +1197,7 @@ func TestFragment_Snapshot(t *testing.T) {
 		t.Fatal(err)
 	}
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Snapshot bitmap and verify data.
@@ -1290,7 +1290,7 @@ func TestFragment_Top_Filter(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Retrieve top rows.
@@ -1467,7 +1467,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
 	defer f.Clean(t)
 
 	// Obtain transaction.
-	tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
+	tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Set bits on various rows.
@@ -1546,7 +1546,7 @@ func TestFragment_Blocks(t *testing.T) {
 	}
 	prev = blocks
 
-	tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
 	// Set bit on different row.
 	if _, err := f.setBit(tx, 20, 0); err != nil {
 		t.Fatal(err)
@@ -1561,7 +1561,7 @@ func TestFragment_Blocks(t *testing.T) {
 	prev = blocks
 
 	// Set bit on different column.
-	tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 	if _, err := f.setBit(tx, 20, 100); err != nil {
 		t.Fatal(err)
@@ -1658,7 +1658,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 	}
 
 	// Obtain transaction.
-	tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
+	tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Set bits on the fragment.
@@ -1669,7 +1669,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard})
+	tx = index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Verify correct cache type and size.
@@ -1808,7 +1808,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Snapshot to disk before benchmarking.
@@ -2149,7 +2149,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2161,7 +2161,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Clear import.
@@ -2171,7 +2171,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2199,12 +2199,12 @@ func TestFragment_ConcurrentImport(t *testing.T) {
 
 		eg := errgroup.Group{}
 		eg.Go(func() error {
-			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
+			tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
 			defer func() { panicOn(tx.Commit()) }()
 			return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{})
 		})
 		eg.Go(func() error {
-			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
+			tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
 			defer func() { panicOn(tx.Commit()) }()
 			return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{})
 		})
@@ -2428,7 +2428,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2440,7 +2440,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Clear import.
@@ -2450,7 +2450,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2678,7 +2678,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2690,7 +2690,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Clear import.
@@ -2700,7 +2700,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
 			}
 
 			panicOn(tx.Commit())
-			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 
 			// Check for expected results.
@@ -2763,7 +2763,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 			i++
 		}
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		if err := f.bulkImport(tx, rows, cols, options); err != nil {
@@ -3137,6 +3137,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		fi.Close()
 
 		h := NewHolder(fi.Name(), nil)
+		panicOn(h.Open())
 		idx, err := h.CreateIndex("i", IndexOptions{})
 		panicOn(err)
 
@@ -3147,7 +3148,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		}
 
 		// Obtain transaction.
-		tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		copy(rows, rowsOrig)
@@ -3195,7 +3196,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
 		f := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0)
 		defer f.Clean(b)
 
-		tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		err = f.Open()
@@ -3432,6 +3433,7 @@ func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64)
 func newTestHolder(tb testing.TB) *Holder {
 	path, _ := testhook.TempDirInDir(tb, *TempDir, "holder-dir")
 	h := NewHolder(path, nil)
+	panicOn(h.Open())
 	//h.SnapshotQueue = newSnapshotQueue(1, 1, nil)
 	return h
 }
@@ -3446,7 +3448,7 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde
 	idx.keys = opt.Keys
 	idx.trackExistence = opt.TrackExistence
 
-	if err := idx.Open(false); err != nil {
+	if err := idx.Open(); err != nil {
 		panic(err)
 	}
 	return idx
@@ -3472,10 +3474,10 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
 	fragPath := fragDir + fmt.Sprintf("%v", shard)
 	f := newFragment(th, fragPath, index, field, view, shard, flags)
 
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
 	testhook.Cleanup(tb, func() {
 		tx.Rollback()
-		panicOn(idx.Txf.CloseIndex(idx))
+		panicOn(idx.holder.txf.CloseIndex(idx))
 	})
 
 	f.CacheType = cacheType
@@ -3582,7 +3584,7 @@ func TestFragment_RowsIteration(t *testing.T) {
 		}
 		panicOn(tx.Commit())
 
-		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		ids, err := f.rows(context.Background(), tx, 0)
@@ -4010,7 +4012,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 		f.mustSetBits(tx, 3, 0)
 
 		panicOn(tx.Commit())
-		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		iter, err := f.rowIterator(tx, false)
@@ -4058,7 +4060,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 		f.mustSetBits(tx, 7, 0)
 
 		panicOn(tx.Commit())
-		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		iter, err := f.rowIterator(tx, false)
@@ -4106,7 +4108,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 		f.mustSetBits(tx, 3, 0)
 
 		panicOn(tx.Commit())
-		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		iter, err := f.rowIterator(tx, true)
@@ -4143,7 +4145,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 		f.mustSetBits(tx, 7, 0)
 
 		panicOn(tx.Commit())
-		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 		defer tx.Rollback()
 
 		iter, err := f.rowIterator(tx, true)
@@ -4166,7 +4168,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 				t.Errorf("should have wrapped")
 			}
 			if !reflect.DeepEqual(row.Columns(), []uint64{0}) {
-				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
+				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but got %v", i, row.Columns())
 			}
 		}
 	})
@@ -4392,7 +4394,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 
 	t.Run("<", func(t *testing.T) {
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4416,7 +4418,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	})
 	t.Run("<=", func(t *testing.T) {
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4440,7 +4442,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	})
 	t.Run(">", func(t *testing.T) {
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4463,7 +4465,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 		}
 	})
 	t.Run(">=", func(t *testing.T) {
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4487,7 +4489,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	})
 	t.Run("Range", func(t *testing.T) {
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4523,7 +4525,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	})
 	t.Run("==", func(t *testing.T) {
 
-		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard})
 		defer tx.Rollback()
 
 		for i := minCheck; i < maxCheck; i++ {
@@ -4563,7 +4565,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Generate a list of columns.
@@ -4726,7 +4728,7 @@ func TestFragmentBSISigned(t *testing.T) {
 	}
 
 	panicOn(tx.Commit())
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	// Generate a list of columns.
@@ -4926,7 +4928,7 @@ func TestImportClearRestart(t *testing.T) {
 				if err != nil {
 					t.Fatalf("initial small import: %v", err)
 				}
-				if idx.Txf.TxType() == RoaringTxn {
+				if idx.holder.txf.TxType() == RoaringTxn {
 					if expOpN <= maxOpN && f.opN != expOpN {
 						t.Errorf("unexpected opN - %d is not %d", f.opN, expOpN)
 					}
@@ -4940,13 +4942,13 @@ func TestImportClearRestart(t *testing.T) {
 				panicOn(tx.Commit())
 
 				err = f.Open()
-				tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+				tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 				defer tx.Rollback()
 				if err != nil {
 					t.Fatalf("reopening fragment: %v", err)
 				}
 
-				if idx.Txf.TxType() == RoaringTxn {
+				if idx.holder.txf.TxType() == RoaringTxn {
 					if expOpN <= maxOpN && f.opN != expOpN {
 						t.Errorf("unexpected opN after close/open %d is not %d", f.opN, expOpN)
 					}
@@ -4970,7 +4972,7 @@ func TestImportClearRestart(t *testing.T) {
 
 				panicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation.
 
-				tx2 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard})
+				tx2 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard})
 				defer tx2.Rollback()
 
 				err = f.closeStorage()
@@ -4983,7 +4985,7 @@ func TestImportClearRestart(t *testing.T) {
 					t.Fatalf("opening new fragment: %v", err)
 				}
 
-				if idx.Txf.TxType() == RoaringTxn {
+				if idx.holder.txf.TxType() == RoaringTxn {
 					if expOpN <= maxOpN && f2.opN != expOpN {
 						t.Errorf("unexpected opN after close/open %d is not %d", f2.opN, expOpN)
 					}
@@ -5016,7 +5018,7 @@ func TestImportClearRestart(t *testing.T) {
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
-				tx3 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard})
+				tx3 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard})
 				defer tx3.Rollback()
 
 				err = f2.closeStorage()
@@ -5072,7 +5074,7 @@ func TestImportValueConcurrent(t *testing.T) {
 	// we will be making a new Tx each time, so we can rollback the default provided one.
 	tx.Rollback()
 
-	types := idx.Txf.TxTypes()
+	types := idx.holder.txf.TxTypes()
 	for _, ty := range types {
 		switch ty {
 		case roaringTxn:
@@ -5090,7 +5092,7 @@ func TestImportValueConcurrent(t *testing.T) {
 	for i := 0; i < 4; i++ {
 		i := i
 		eg.Go(func() error {
-			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+			tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 			defer tx.Rollback()
 			for j := uint64(0); j < 10; j++ {
 				err := f.importValue(tx, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0)
@@ -5138,7 +5140,7 @@ func TestImportMultipleValues(t *testing.T) {
 
 				// probably too slow, would hit disk alot:
 				//panicOn(tx.Commit())
-				//tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true})
+				//tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true})
 				//defer tx.Rollback()
 
 				for i := range test.checkCols {
@@ -5232,13 +5234,11 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 	f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
 	defer f.Clean(t)
 	tx.Rollback()
-	// Obtain transaction, but don't start another b/c the
-	// two goroutines below need the same view.
 
 	eg := &errgroup.Group{}
 	eg.Go(func() error {
 
-		tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+		tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 
 		for i := uint64(0); i < 1000; i++ {
 			_, err := f.setBit(tx, i%4, i)
@@ -5251,7 +5251,7 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 	})
 
 	// need read-only Tx so as not to block on the writer finishing above.
-	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	acc := uint64(0)

go.mod

@@ -11,6 +11,8 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
+	github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
 	github.com/glycerine/lmdb-go v1.9.32
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.1
@@ -31,7 +33,6 @@ require (
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.4.0
@@ -45,7 +46,6 @@ require (
 	golang.org/x/text v0.3.3 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible

go.sum

@@ -5,6 +5,8 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
+github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@@ -33,15 +35,25 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
+github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
+github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -76,6 +88,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
@@ -185,6 +199,7 @@ github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Ung
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -206,6 +221,7 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
@@ -214,6 +230,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -231,6 +248,7 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
@@ -247,6 +265,7 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -288,9 +307,11 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

gossip/gossip.go

@@ -330,7 +330,7 @@ func (g *memberSet) LocalState(join bool) []byte {
 		for _, f := range idx.Fields {
 			availableShards := roaring.NewBitmap()
 			if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
-				availableShards = field.AvailableShards()
+				availableShards = field.AvailableShards(false)
 			}
 
 			fs := &pilosa.FieldStatus{

holdbkg.go

@@ -0,0 +1,135 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/glycerine/idem"
+)
+
+// holderBackgroundGoro avoids the deadlock versus race dilmena
+// when creating a new index. The troubles were in having
+// the field/view code try to inform the Holder of the indexes
+// from a platoon of worker goroutines and tests; see index.go:273 inside
+// Index.openFields(). There calling i.holder.addIndex(i) instead
+// of locking cleans things up alot. There's no need anymore
+// to track which goroutines are holding the Holder's
+// mu lock.
+type holderBackgroundGoro struct {
+	h          *Holder
+	halt       *idem.Halter
+	reqIndexCh chan *indexReq
+	setIdxCh   chan *Index
+	getAllCh   chan *indexReq
+	delIdxCh   chan string
+}
+
+// indexReq is used to ask the holderBackgrounGoro
+// for index information.
+type indexReq struct {
+	// basic request to map an index name to *Index
+	index string
+	idx   *Index
+
+	// double duty as a getAll indexes request
+	getAll bool
+	all    []*Index
+
+	done chan struct{}
+}
+
+func newIndexReq(index string) *indexReq {
+	return &indexReq{
+		index: index,
+		done:  make(chan struct{}),
+	}
+}
+
+func (b *holderBackgroundGoro) index(index string) *Index {
+	if b == nil {
+		// utils_internal_test.go:448 from
+		// TestCluster_ResizeStates/Multiple_nodes,_with_data
+		// wants to pass a nil b/nil Holder. sigh. don't panic.
+		return nil
+	}
+	req := newIndexReq(index)
+	b.reqIndexCh <- req
+	<-req.done
+	return req.idx
+}
+
+func (b *holderBackgroundGoro) isClosed() bool {
+	select {
+	case <-b.halt.Done.Chan:
+		return true
+	default:
+		return false
+	}
+}
+
+func (b *holderBackgroundGoro) start() {
+	go func() {
+		defer b.halt.Done.Close()
+		for {
+			select {
+			case <-b.halt.ReqStop.Chan:
+				return
+
+			case r := <-b.reqIndexCh:
+				r.idx = b.h.indexesOwnedByBkgr[r.index]
+				close(r.done)
+
+			case idx := <-b.setIdxCh:
+				b.h.indexesOwnedByBkgr[idx.Name()] = idx
+				// atomic operation, client doesn't need to wait on done,
+				// so don't take the time to do another channel, just leave done open.
+
+			case target := <-b.delIdxCh:
+				delete(b.h.indexesOwnedByBkgr, target)
+				// atomic operation, client doesn't need to wait on done,
+				// so don't take the time to do another channel, just leave done open.
+
+			case r := <-b.getAllCh:
+				r.all = make([]*Index, 0, len(b.h.indexesOwnedByBkgr))
+				for _, index := range b.h.indexesOwnedByBkgr {
+					r.all = append(r.all, index)
+				}
+				close(r.done)
+			}
+		}
+	}()
+}
+
+func (h *Holder) stopBkgr() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if h.bkgr != nil {
+		h.bkgr.halt.ReqStop.Close()
+		<-h.bkgr.halt.Done.Chan
+	}
+}
+
+func (h *Holder) newHolderBackgroundGoro() (b *holderBackgroundGoro) {
+
+	b = &holderBackgroundGoro{
+		h:          h,
+		halt:       idem.NewHalter(),
+		reqIndexCh: make(chan *indexReq),
+		setIdxCh:   make(chan *Index),
+		getAllCh:   make(chan *indexReq),
+		delIdxCh:   make(chan string),
+	}
+	b.start()
+	return b
+}

holder.go

@@ -60,12 +60,12 @@ func init() {
 type Holder struct {
 	mu sync.RWMutex
 
+	// our configuration
+	cfg *HolderConfig
+
 	// Partition count used by translation.
 	partitionN int
 
-	// Indexes by name.
-	indexes map[string]*Index
-
 	// opened channel is closed once Open() completes.
 	opened lockedChan
 
@@ -116,6 +116,32 @@ type Holder struct {
 	Auditor testhook.Auditor
 
 	txf *TxFactory
+
+	bkgr *holderBackgroundGoro
+
+	// indexesOwnedByBkgr is owned by h.bkgr. Do not touch.
+	// Only the owning holdbkg.go goroutine should query or
+	// modify the indexesOwnedByBkgr map. indexesOwnedByBkgr replaces
+	// the old indexes map which was the source of deadlock vs
+	// race issues with a dedicated goroutine
+	// associated with the Holder.
+	//
+	// Normally this map would live inside the holderBackgroundGoro
+	// struct, but tests expect the map to
+	// persist cross Holder ReOpens(), so it still lives here --
+	// since a ReOpen will kill and restart the goroutine and
+	// lose its state.
+	//
+	// Again, do not touch this directly. Use these methods instead:
+	//
+	//  h.Index()       // index name -> *Index
+	//  h.Indexes()     // copy of the full list of *Indexes
+	//  h.addIndex()    // add one *Index
+	//  h.deleteIndex() // delete one *Index
+	//
+	indexesOwnedByBkgr map[string]*Index
+
+	numCtBlueGreenVerified int64
 }
 
 // HolderOpts holds information about the holder which other things might want
@@ -209,6 +235,8 @@ func DefaultHolderConfig() *HolderConfig {
 }
 
 // NewHolder returns a new instance of Holder for the given path.
+// It starts the bkgr background goroutine that provides
+// exclusive access to the indexesOwnedByBkgr map.
 func NewHolder(path string, cfg *HolderConfig) *Holder {
 	if cfg == nil {
 		cfg = DefaultHolderConfig()
@@ -222,7 +250,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
 	}
 
 	h := &Holder{
-		indexes: make(map[string]*Index),
+		cfg:     cfg,
 		closing: make(chan struct{}),
 
 		opened: lockedChan{ch: make(chan struct{})},
@@ -245,7 +273,11 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
 		Auditor: NewAuditor(),
 
 		path: path,
+
+		indexesOwnedByBkgr: make(map[string]*Index),
 	}
+	h.bkgr = h.newHolderBackgroundGoro()
+
 	txf, err := NewTxFactory(cfg.Txsrc, path, h)
 	panicOn(err)
 	h.txf = txf
@@ -535,9 +567,24 @@ func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo,
 
 // Open initializes the root data directory for the holder.
 func (h *Holder) Open() error {
+
 	h.opening = true
 	defer func() { h.opening = false }()
 
+	if h.bkgr == nil || h.bkgr.isClosed() {
+		h.bkgr = h.newHolderBackgroundGoro()
+	}
+
+	if h.txf == nil {
+		txf, err := NewTxFactory(h.cfg.Txsrc, h.path, h)
+		if err != nil {
+			return errors.Wrap(err, "Holder.Open NewTxFactory()")
+		}
+		h.txf = txf
+	}
+
+	h.txf.blueGreenOffIfRunningBlueGreen()
+
 	// Reset closing in case Holder is being reopened.
 	h.closing = make(chan struct{})
 
@@ -580,9 +627,7 @@ func (h *Holder) Open() error {
 			continue
 		}
 		// Skip embedded db files too.
-		if strings.HasSuffix(fi.Name(), "-badgerdb") ||
-			strings.HasSuffix(fi.Name(), "-lmdb") ||
-			strings.HasSuffix(fi.Name(), "-rbfdb") {
+		if h.txf.IsTxDatabasePath(fi.Name()) {
 			continue
 		}
 
@@ -598,22 +643,19 @@ func (h *Holder) Open() error {
 
 		if h.isCoordinator() {
 			index.createdAt = timestamp()
-			err = index.OpenWithTimestamp(false)
+			err = index.OpenWithTimestamp()
 		} else {
-			err = index.Open(false)
+			err = index.Open()
 		}
 		if err != nil {
-			// FIXME: The holder shouldn't be responsible for closing these, probably.
-			_ = index.Txf.CloseDB()
+			_ = h.txf.Close()
 			if err == ErrName {
 				h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
 				continue
 			}
 			return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
 		}
-		h.mu.Lock()
-		h.indexes[index.Name()] = index
-		h.mu.Unlock()
+		h.addIndex(index)
 	}
 
 	// If any fields were opened before their foreign index
@@ -631,7 +673,14 @@ func (h *Holder) Open() error {
 
 	_ = testhook.Opened(h.Auditor, h, nil)
 
+	// under blue_green, we must sync blue from green before we turn on checking.
+	if err := h.txf.green2blue(h); err != nil {
+		return errors.Wrap(err, "Holder.Open h.txf.UpdateBlueFromGreen(h)")
+	}
+	h.txf.blueGreenOnIfRunningBlueGreen()
+
 	return nil
+
 }
 
 // Activate runs the background tasks relevant to keeping a holder in a stable
@@ -676,10 +725,14 @@ func (h *Holder) processForeignIndexFields() error {
 // Close closes all open fragments.
 func (h *Holder) Close() error {
 
+	defer h.stopBkgr()
+
 	if globalUseStatTx {
 		fmt.Printf("%v\n", globalCallStats.report())
 	}
-	h.txf.blueGreenReg.Close()
+	if h.txf.blueGreenReg != nil {
+		h.txf.blueGreenReg.Close()
+	}
 
 	h.Stats.Close()
 
@@ -687,16 +740,17 @@ func (h *Holder) Close() error {
 	close(h.closing)
 	h.wg.Wait()
 
-	for _, index := range h.indexes {
+	for _, index := range h.Indexes() {
 		if err := index.Close(); err != nil {
 			return errors.Wrap(err, "closing index")
 		}
-		if err := index.Txf.CloseDB(); err != nil {
-			return errors.Wrap(err, "index.Txf.CloseDB()")
-		}
+	}
+	if err := h.txf.Close(); err != nil {
+		return errors.Wrap(err, "holder.Txf.Close()")
 	}
 
 	// Reset opened in case Holder needs to be reopened.
+	h.txf = nil
 	h.opened.mu.Lock()
 	h.opened.ch = make(chan struct{})
 	h.opened.mu.Unlock()
@@ -713,7 +767,7 @@ func (h *Holder) Close() error {
 func (h *Holder) NeedsSnapshot() bool {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
-	for _, idx := range h.indexes {
+	for _, idx := range h.Indexes() {
 		if idx.NeedsSnapshot() {
 			return true
 		}
@@ -727,7 +781,7 @@ func (h *Holder) NeedsSnapshot() bool {
 func (h *Holder) HasData() (bool, error) {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
-	if len(h.indexes) > 0 {
+	if len(h.Indexes()) > 0 {
 		return true, nil
 	}
 	// Open path to read all index directories.
@@ -771,7 +825,7 @@ func (h *Holder) hasV1TranslateKeysFile() (bool, error) {
 func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap {
 	m := make(map[string]*roaring.Bitmap)
 	for _, index := range h.Indexes() {
-		m[index.Name()] = index.AvailableShards()
+		m[index.Name()] = index.AvailableShards(includeRemote)
 	}
 	return m
 }
@@ -914,27 +968,26 @@ func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string {
 }
 
 // Index returns the index by name.
-func (h *Holder) Index(name string) *Index {
-	h.mu.RLock()
-	defer h.mu.RUnlock()
-	return h.index(name)
-}
-
-func (h *Holder) index(name string) *Index {
-	return h.indexes[name]
+func (h *Holder) Index(name string) (idx *Index) {
+	idx = h.bkgr.index(name)
+	return
 }
 
 // Indexes returns a list of all indexes in the holder.
 func (h *Holder) Indexes() []*Index {
-	h.mu.RLock()
-	a := make([]*Index, 0, len(h.indexes))
-	for _, index := range h.indexes {
-		a = append(a, index)
+	// utils_internal_test.go:345	func (t *ClusterCluster) Close() error
+	// wants to close an un-Open()-ed Holder.
+	// So I guess we won't panic here.
+	if h.bkgr == nil || h.bkgr.isClosed() {
+		return nil
 	}
-	h.mu.RUnlock()
+	req := newIndexReq("")
+	req.getAll = true
+	h.bkgr.getAllCh <- req
+	<-req.done
 
-	sort.Sort(indexSlice(a))
-	return a
+	sort.Sort(indexSlice(req.all))
+	return req.all
 }
 
 // CreateIndex creates an index.
@@ -944,7 +997,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 	defer h.mu.Unlock()
 
 	// Ensure index doesn't already exist.
-	if h.index(name) != nil {
+	if h.Index(name) != nil {
 		return nil, newConflictError(ErrIndexExists)
 	}
 	return h.createIndex(name, opt)
@@ -957,10 +1010,9 @@ func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index,
 	defer h.mu.Unlock()
 
 	// Return index if it exists.
-	if index := h.index(name); index != nil {
+	if index := h.Index(name); index != nil {
 		return index, nil
 	}
-
 	return h.createIndex(name, opt)
 }
 
@@ -978,7 +1030,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err = index.Open(true); err != nil {
+	if err = index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 	if err = index.saveMeta(); err != nil {
@@ -986,7 +1038,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	}
 
 	// Update options.
-	h.indexes[index.Name()] = index
+	h.addIndex(index)
 
 	// Since this is a new index, we need to kick off
 	// its translation sync.
@@ -1017,7 +1069,7 @@ func (h *Holder) DeleteIndex(name string) error {
 	defer h.mu.Unlock()
 
 	// Confirm index exists.
-	index := h.index(name)
+	index := h.Index(name)
 	if index == nil {
 		return newNotFoundError(ErrIndexNotFound, name)
 	}
@@ -1028,7 +1080,7 @@ func (h *Holder) DeleteIndex(name string) error {
 	}
 
 	// remove any backing store.
-	if err := index.Txf.DeleteIndex(name); err != nil {
+	if err := h.txf.DeleteIndex(name); err != nil {
 		return errors.Wrap(err, "index.Txf.DeleteIndex")
 	}
 
@@ -1038,7 +1090,7 @@ func (h *Holder) DeleteIndex(name string) error {
 	}
 
 	// Remove reference.
-	delete(h.indexes, name)
+	h.deleteIndex(name)
 
 	// I'm not sure if calling Reset() here is necessary
 	// since closing the index stops its translation
@@ -1046,6 +1098,10 @@ func (h *Holder) DeleteIndex(name string) error {
 	return h.translationSyncer.Reset()
 }
 
+func (h *Holder) deleteIndex(index string) {
+	h.bkgr.delIdxCh <- index
+}
+
 // Field returns the field for an index and name.
 func (h *Holder) Field(index, name string) *Field {
 	idx := h.Index(index)
@@ -1287,7 +1343,7 @@ func (s *holderSyncer) SyncHolder() error {
 					return nil
 				}
 
-				itr := s.Holder.Index(di.Name).AvailableShards().Iterator()
+				itr := s.Holder.Index(di.Name).AvailableShards(includeRemote).Iterator()
 				itr.Seek(0)
 				for shard, eof := itr.Next(); !eof; shard, eof = itr.Next() {
 					// Ignore shards that this host doesn't own.
@@ -1736,7 +1792,7 @@ func (c *holderCleaner) CleanHolder() error {
 		}
 
 		// Get the fragments that node is responsible for (based on hash(index, node)).
-		containedShards := c.Cluster.containsShards(index.Name(), index.AvailableShards(), c.Node)
+		containedShards := c.Cluster.containsShards(index.Name(), index.AvailableShards(includeRemote), c.Node)
 
 		// Get the fragments registered in memory.
 		for _, field := range index.Fields() {
@@ -1785,33 +1841,28 @@ func uint64InSlice(i uint64, s []uint64) bool {
 // Process loops through a holder based on the Check functions in op, calling
 // the Process functions in op when indicated.
 func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
-	var indexNames, fieldNames, viewNames []string
+	var fieldNames, viewNames []string
 	var fragNums []uint64
 
-	h.mu.Lock()
-	for indexName := range h.indexes {
-		indexNames = append(indexNames, indexName)
-	}
-	h.mu.Unlock()
-	for _, indexName := range indexNames {
+	indexes := h.Indexes()
+	for _, idx := range indexes {
 		if err = ctx.Err(); err != nil {
 			return err
 		}
+		if idx == nil {
+			continue
+		}
+		indexName := idx.name
 		process, recurse := op.CheckIndex(indexName)
 		if !process && !recurse {
 			continue
 		}
-		h.mu.Lock()
-		index := h.indexes[indexName]
-		h.mu.Unlock()
-		if index == nil {
-			continue
-		}
+
 		if err = ctx.Err(); err != nil {
 			return err
 		}
 		if process {
-			err = op.ProcessIndex(index)
+			err = op.ProcessIndex(idx)
 			if err != nil {
 				return err
 			}
@@ -1820,22 +1871,22 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
 			continue
 		}
 		fieldNames = fieldNames[:0]
-		index.mu.Lock()
-		for fieldName := range index.fields {
+		idx.mu.Lock()
+		for fieldName := range idx.fields {
 			fieldNames = append(fieldNames, fieldName)
 		}
-		index.mu.Unlock()
+		idx.mu.Unlock()
 		for _, fieldName := range fieldNames {
 			if err = ctx.Err(); err != nil {
 				return err
 			}
-			process, recurse := op.CheckField(indexName, fieldName)
+			process, recurse := op.CheckField(idx.name, fieldName)
 			if !process && !recurse {
 				continue
 			}
-			index.mu.Lock()
-			field := index.fields[fieldName]
-			index.mu.Unlock()
+			idx.mu.Lock()
+			field := idx.fields[fieldName]
+			idx.mu.Unlock()
 			if field == nil {
 				continue
 			}
@@ -1913,23 +1964,32 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
 
 // used by Index.openFields(), enabling Tx / Txf by telling
 // the holder about its own indexes.
-func (h *Holder) addIndexFromField(idx *Index) {
-	h.mu.Lock()
-	h.indexes[idx.Name()] = idx
-	h.mu.Unlock()
-}
-
-func (h *Holder) unprotectedAddIndexFromField(idx *Index) {
-	h.indexes[idx.Name()] = idx
+func (h *Holder) addIndex(idx *Index) {
+	if idx == nil {
+		panic("cannot pass nil to addIndex")
+	}
+	if h == nil {
+		panic("cannot call addIndex on nil Holder")
+	}
+	if h.bkgr == nil || h.bkgr.isClosed() {
+		// ugh. TestCluster_ResizeStates/Multiple_nodes,_with_data test
+		// from cluster_internal_test.go:799
+		// via utils_internal_test.go:450
+		// gets here, with the h.mu already held.
+		// so we cannot panic and complain or we mess up that test.
+		// But really, we should not be calling addIndex() on
+		// Holder that has not be Open()-ed.
+		h.mu.Lock()
+		h.bkgr = h.newHolderBackgroundGoro()
+		h.mu.Unlock()
+	}
+	h.bkgr.setIdxCh <- idx
 }
 
 func (h *Holder) DumpAllShards() {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
-	for index, idx := range h.indexes {
-		fmt.Printf("dump of index '%v'\n", index)
-		idx.Txf.dbPerShard.DumpAll()
-	}
+	h.txf.dbPerShard.DumpAll()
 }
 
 func (h *Holder) Txf() *TxFactory {
@@ -1941,5 +2001,38 @@ func (h *Holder) Txf() *TxFactory {
 // Begin starts a transaction on the holder. The index and shard
 // must be specified.
 func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) {
-	return idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
+	return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
+}
+
+func (h *Holder) NumContainersBlueGreenVerified() int {
+	return int(h.numCtBlueGreenVerified)
+}
+
+func (h *Holder) HasRoaringData() (has bool, err error) {
+
+	idxs := h.Indexes()
+	for _, idx := range idxs {
+		paths, err := listFilesUnderDir(idx.path, false, "", true)
+		if err != nil {
+			return false, errors.Wrap(err, "HasRoaringData listFilesUnderDir")
+		}
+		index := idx.name
+
+		for _, relpath := range paths {
+			field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
+			if err != nil {
+				continue // ignore .meta paths
+			}
+			abspath := idx.path + sep + relpath
+
+			hasData, err := roaringFragmentHasData(abspath, index, field, view, shard)
+			if err != nil {
+				return false, errors.Wrap(err, "HasRoaringData roaringFragmentHasData")
+			}
+			if hasData {
+				return true, nil
+			}
+		}
+	}
+	return
 }

holder_internal_test.go

@@ -16,12 +16,15 @@ package pilosa
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/testhook"
 )
 
+var _ = fmt.Printf
+
 type testHolderOperator struct {
 	indexSeen, indexProcessed       int
 	fieldSeen, fieldProcessed       int
@@ -79,7 +82,7 @@ func makeHolder(tb testing.TB) (*Holder, string, error) {
 		return nil, "", err
 	}
 	h := NewHolder(path, nil)
-	return h, path, nil
+	return h, path, h.Open()
 }
 
 func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
@@ -89,21 +92,81 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
 		t.Fatalf("creating index: %v", err)
 	}
 
-	shard := columnID / ShardWidth
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
-	defer tx.Rollback()
-
 	f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
 	if err != nil {
 		t.Fatalf("setting bit: %v", err)
 	}
-	_, err = f.SetBit(tx, rowID, columnID, nil)
+	_, err = f.SetBit(nil, rowID, columnID, nil)
 	if err != nil {
 		t.Fatalf("setting bit: %v", err)
 	}
-	if err := tx.Commit(); err != nil {
-		t.Fatal(err)
+}
+
+func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
+
+	//shard := columnID / ShardWidth
+
+	// hmm... if its a new holder, meta data isn't there, so ask for it.
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	panicOn(err)
+
+	f := idx.Field(field)
+	if f == nil {
+		t.Fatalf("no such field '%v'", field)
 	}
+
+	row, err := f.Row(nil, rowID)
+	if err != nil {
+		t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err)
+	}
+
+	cols := row.Columns()
+	if len(cols) == 0 {
+		t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID)
+	}
+
+	for _, c := range cols {
+		if c == columnID {
+			return // ok, found it.
+		}
+	}
+	t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID)
+}
+
+func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
+	if testHasBit(t, h, index, field, rowID, columnID) {
+		t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID)
+	}
+}
+
+func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool {
+
+	idx := h.Index(index)
+	if idx == nil {
+		return false // not even an index by this name. Obviously no hot bits either.
+	}
+
+	f := idx.Field(field)
+	if f == nil {
+		return false
+	}
+
+	row, err := f.Row(nil, rowID)
+	if err != nil {
+		return false
+	}
+
+	cols := row.Columns()
+	if len(cols) == 0 {
+		return false
+	}
+
+	for _, c := range cols {
+		if c == columnID {
+			return true // ok, found it.
+		}
+	}
+	return false
 }
 
 func TestHolderOperatorProcess(t *testing.T) {

holder_test.go

@@ -180,7 +180,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 
 		var shard uint64
-		tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
 		defer tx.Rollback()
 
 		if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
@@ -214,7 +214,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 
 		var shard uint64
-		tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -247,7 +247,7 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		}
 		var shard uint64
-		tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
 		defer tx.Rollback()
 
 		if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {

index.go

@@ -70,8 +70,6 @@ type Index struct {
 
 	// Instantiates new translation stores
 	OpenTranslateStore OpenTranslateStoreFunc
-
-	Txf *TxFactory
 }
 
 // NewIndex returns an existing (but possibly empty) instance of
@@ -105,19 +103,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
 		translationSyncer: NopTranslationSyncer,
 
 		OpenTranslateStore: OpenInMemTranslateStore,
-
-		// the Txf should be shared across all holder.
-		Txf: holder.txf,
 	}
 	return idx, nil
 }
 
 func (i *Index) NewTx(txo Txo) Tx {
-	return i.Txf.NewTx(txo)
+	return i.holder.txf.NewTx(txo)
 }
 
 func (i *Index) NeedsSnapshot() bool {
-	return i.Txf.NeedsSnapshot()
+	return i.holder.txf.NeedsSnapshot()
 }
 
 // CreatedAt is an timestamp for a specific version of an index.
@@ -174,12 +169,14 @@ func (i *Index) options() IndexOptions {
 }
 
 // Open opens and initializes the index.
-func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) }
+func (i *Index) Open() error {
+	return i.open(false)
+}
 
 // OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
-func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) }
+func (i *Index) OpenWithTimestamp() error { return i.open(true) }
 
-func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
+func (i *Index) open(withTimestamp bool) (err error) {
 	// Ensure the path exists.
 	i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
 	if err := os.MkdirAll(i.path, 0777); err != nil {
@@ -193,7 +190,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
 	}
 
 	i.holder.Logger.Debugf("open fields for index: %s", i.name)
-	if err := i.openFields(withTimestamp, haveHolderLock); err != nil {
+	if err := i.openFields(withTimestamp); err != nil {
 		return errors.Wrap(err, "opening fields")
 	}
 
@@ -240,7 +237,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
 var indexQueue = make(chan struct{}, 8)
 
 // openFields opens and initializes the fields inside the index.
-func (i *Index) openFields(withTimestamp, haveHolderLock bool) error {
+func (i *Index) openFields(withTimestamp bool) error {
 	f, err := os.Open(i.path)
 	if err != nil {
 		return errors.Wrap(err, "opening directory")
@@ -273,27 +270,8 @@ fileLoop:
 
 				mu.Lock()
 
-				// i.holder needs to know about its index i for the Txf to work.
-				//
-				// We face either a deadlock or a race here.
-				//
-				// We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK"
-				// if we call addIndexFromField, because in that test
-				// we get here while already holding i.holder.mu.
-				//
-				// On the other had, we get races on other tests
-				// such as TestExecutor_Execute_Existence/Row
-				// if we call unprotectedAddIndexFromField which does
-				// not lock i.holder.mu.
-				//
-				// The resolution was to have the goroutines that are holding
-				// the lock already tell us. That is the haveHolderLock
-				// argument.
-				if haveHolderLock {
-					i.holder.unprotectedAddIndexFromField(i)
-				} else {
-					i.holder.addIndexFromField(i)
-				}
+				// goroutine safe
+				i.holder.addIndex(i)
 
 				fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
 				if withTimestamp {
@@ -397,7 +375,7 @@ func (i *Index) Close() error {
 		_ = testhook.Closed(i.holder.Auditor, i, nil)
 	}()
 
-	err := i.Txf.CloseIndex(i)
+	err := i.holder.txf.CloseIndex(i)
 	if err != nil {
 		return errors.Wrap(err, "closing index")
 	}
@@ -424,8 +402,12 @@ func (i *Index) Close() error {
 	return nil
 }
 
+// make it clear what the Index.AvailableShards() calls are trying to obtain.
+const includeRemote = false
+const localOnly = true
+
 // AvailableShards returns a bitmap of all shards with data in the index.
-func (i *Index) AvailableShards() *roaring.Bitmap {
+func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
 	if i == nil {
 		return roaring.NewBitmap()
 	}
@@ -435,8 +417,8 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
 
 	b := roaring.NewBitmap()
 	for _, f := range i.fields {
-		//b.Union(f.AvailableShards())
-		b.UnionInPlace(f.AvailableShards())
+		//b.Union(f.AvailableShards(localOnly))
+		b.UnionInPlace(f.AvailableShards(localOnly))
 	}
 
 	i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0)
@@ -445,7 +427,7 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
 
 // Begin starts a transaction on a shard of the index.
 func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) {
-	return i.Txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil
+	return i.holder.txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil
 }
 
 // fieldPath returns the path to a field in the index.
@@ -621,7 +603,7 @@ func (i *Index) DeleteField(name string) error {
 		return errors.Wrap(err, "closing")
 	}
 
-	if err := i.Txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil {
+	if err := i.holder.txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil {
 		return errors.Wrap(err, "Txf.DeleteFieldFromStore")
 	}
 
@@ -705,7 +687,7 @@ func FormatQualifiedIndexName(index string) string {
 func (idx *Index) Dump(label string) {
 	//fileline := FileLine(2)
 	fmt.Printf("\nDump: %v\n\n", label)
-	idx.Txf.dbPerShard.DumpAll()
+	idx.holder.txf.dbPerShard.DumpAll()
 }
 
 func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []uint64, err error) {
@@ -843,3 +825,7 @@ func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) {
 		fmt.Fprintf(w, "empty index '%v'", idx.path)
 	}
 }
+
+func (idx *Index) Txf() *TxFactory {
+	return idx.holder.txf
+}

index_internal_test.go

@@ -47,7 +47,7 @@ func (i *Index) reopen() error {
 	if err := i.Close(); err != nil {
 		return err
 	}
-	if err := i.Open(false); err != nil {
+	if err := i.Open(); err != nil {
 		return err
 	}
 	return nil

license.exceptions

@@ -18,3 +18,4 @@
 ./gid.go
 ./cmd/pilosa-keydump/vprint.go
 ./cmd/pilosa-keydump/keydump.go
+./synthload/vprint.go

lmdb.go

@@ -69,6 +69,17 @@ type lmdbRegistrar struct {
 	path2db map[string]*LMDBWrapper
 }
 
+func (r *lmdbRegistrar) Size() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	nmp := len(r.mp)
+	npa := len(r.path2db)
+	if nmp != npa {
+		panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa))
+	}
+	return nmp
+}
+
 var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar()
 
 var globalNextTxSnLMDB int64
@@ -101,18 +112,19 @@ func (r *lmdbRegistrar) unregister(w *LMDBWrapper) {
 }
 
 func DumpAllLMDB() {
+	short := true
 	globalLMDBReg.mu.Lock()
 	defer globalLMDBReg.mu.Unlock()
 	for w := range globalLMDBReg.mp {
-		AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil))
+		AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil, short))
 	}
 }
 
 // lmdbPath is a helper for determining the full directory
 // in which the lmdb database will be stored.
 func lmdbPath(path string) string {
-	if !strings.HasSuffix(path, "-lmdb") {
-		return path + "-lmdb"
+	if !strings.HasSuffix(path, "-lmdb@") {
+		return path + "-lmdb@"
 	}
 	return path
 }
@@ -154,6 +166,9 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper
 
 	//flags := uint(lmdb.NoReadahead | lmdb.NoSubdir)
 	//flags := uint(lmdb.NoSubdir) // no difference without the No.Readahead on ./query.
+	// NoReadahead should be better for random workloads or those bigger than memory,
+	// as it avoids loading extra pages which then evict pages you are using.
+	//flags := uint(lmdb.NoReadahead)
 	flags := uint(0)
 
 	// unsafe, but get upper bound on performance.
@@ -162,7 +177,15 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper
 	//	NoSync      = C.MDB_NOSYNC     // Don't fsync after commit.
 	// flags = flags | lmdb.WriteMap | lmdb.NoMetaSync | lmdb.NoSync // about the same speed
 	// flags = flags | lmdb.NoMetaSync | lmdb.NoSync // slows things down
-	//flags = flags | lmdb.WriteMap // seems faster than without:
+
+	// TODO(jea): we got an odd segfault in the roaring/ pkg when we went to read-only mmap
+	// with the row-cache off; probably means we were trying to write to mmap-ed
+	// memory. Investigate at some point. reference: 2a77b25d ja/write_map_prevents_segfault
+	//
+	// Update: added COW to roaring/roaring.go:3282  func (c *Container) arrayRemove(),
+	// seems to fix the issue.
+	//
+	//flags = flags | lmdb.WriteMap // seems faster than without: or maybe not. not sure.
 	// kRemove  N=  710401   avg/op:      7.714µs   sd:      27.83µs  total: 5.480656859s
 	//    kAdd  N=  722835   avg/op:      9.096µs   sd:    105.787µs  total: 6.575497725s
 
@@ -225,6 +248,38 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper
 	return w, nil
 }
 
+func (w *LMDBWrapper) Path() string {
+	return w.path
+}
+
+func (w *LMDBWrapper) HasData() (has bool, err error) {
+
+	tx, err := w.NewTx(!writable, "", Txo{})
+	if err != nil {
+		return false, errors.Wrap(err, "HasData NewTx")
+	}
+	defer tx.Rollback()
+
+	bi := NewLMDBIterator(tx.(*LMDBTx), nil)
+	defer bi.Close()
+
+	for bi.Next() {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (w *LMDBWrapper) CleanupTx(tx Tx) {
+	// inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface.
+}
+
+func (tx *LMDBTx) IsDone() (done bool) {
+	tx.mu.Lock()
+	done = tx.unlocked
+	tx.mu.Unlock()
+	return
+}
+
 func (w *LMDBWrapper) OpenListString() (r string) {
 
 	list := w.listopen()
@@ -279,8 +334,6 @@ func (w *LMDBWrapper) DeleteIndex(indexName string) error {
 var _ Tx = (*LMDBTx)(nil)
 
 // LMDBWrapper provides the NewTx() method.
-// Execute lmdbJob's via LMDBWrapper.submit(); these must
-// be done by the lmdb goroutine worker pool.
 type LMDBWrapper struct {
 	env *lmdb.Env
 
@@ -310,6 +363,11 @@ type LMDBWrapper struct {
 	openTx map[*LMDBTx]bool
 }
 
+func (w *LMDBWrapper) SetHolder(h *Holder) {
+	// don't need it at the moment
+	//w.h = h
+}
+
 // NewTxWRITE lets us see in the callstack dumps where the WRITE tx are.
 // Can't have more than one active write per database, so the
 // 2nd one will block until the first finishes.
@@ -321,6 +379,9 @@ func (w *LMDBWrapper) NewTxWRITE() *lmdb.Txn {
 
 // NewTxREAD lets us see in the callstack dumps where the READ tx are.
 func (w *LMDBWrapper) NewTxREAD() *lmdb.Txn {
+	if w.env == nil {
+		panic("cannot call NewTxREAD() on closed LMDBWrapper() -- open a new Wrapper first")
+	}
 	lmdbTxn, err := w.env.BeginTxn(nil, lmdb.Readonly)
 	panicOn(err)
 	return lmdbTxn
@@ -342,7 +403,8 @@ func (w *LMDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx,
 
 	sn := atomic.AddInt64(&globalNextTxSnLMDB, 1)
 
-	//vv("lmdb new tx _sn_ %v; stack \n%v", sn, stack())
+	//vv("lmdb new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack())
+	//vv("lmdb new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString())
 
 	runtime.LockOSThread()
 
@@ -473,7 +535,7 @@ func (tx *LMDBTx) Type() string {
 }
 
 func (tx *LMDBTx) UseRowCache() bool {
-	return true
+	return false
 }
 
 // Pointer gives us a memory address for the underlying transaction for debugging.
@@ -502,6 +564,7 @@ func (tx *LMDBTx) Rollback() {
 	if !tx.unlocked {
 		runtime.UnlockOSThread()
 		tx.unlocked = true
+		tx.o.dbs.Cleanup(tx)
 	}
 }
 
@@ -527,6 +590,7 @@ func (tx *LMDBTx) Commit() error {
 	if !tx.unlocked {
 		runtime.UnlockOSThread()
 		tx.unlocked = true
+		tx.o.dbs.Cleanup(tx)
 	}
 	return err
 }
@@ -1244,6 +1308,7 @@ func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end
 // fragment.rowFromStorage() in fragment.go. For example:
 //
 //    data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard,
+//
 //                     f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
 //                     ^ offset            ^ start           ^ endx
 //
@@ -1252,6 +1317,10 @@ func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end
 // is done to conform to the roaring.OffsetRange() argument convention.
 //
 func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) {
+	//vv("top of LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx))
+	//defer func() {
+	//vv("returning from LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack())
+	//}()
 
 	// roaring does these three checks in its OffsetRange
 	if lowbits(offset) != 0 {
@@ -1469,11 +1538,11 @@ func ToContainer(typ byte, w []byte) (r *roaring.Container) {
 
 // StringifiedLMDBKeys returns a string with all the container
 // keys available in lmdb.
-func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
+func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx, short bool) (r string) {
 	if optionalUseThisTx == nil {
 		tx, _ := w.NewTx(!writable, "<StringifiedLMDBKeys>", Txo{})
 		defer tx.Rollback()
-		r = stringifiedLMDBKeysTx(tx.(*LMDBTx))
+		r = stringifiedLMDBKeysTx(tx.(*LMDBTx), short)
 		return
 	}
 
@@ -1481,7 +1550,7 @@ func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
 	if !ok {
 		return fmt.Sprintf("<not-a-LMDBTx-in-StringifiedLMDBKeys-was-%T>", optionalUseThisTx)
 	}
-	r = stringifiedLMDBKeysTx(btx)
+	r = stringifiedLMDBKeysTx(btx, short)
 	return
 }
 
@@ -1507,8 +1576,8 @@ func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
 	return
 }
 
-func (tx *LMDBTx) Dump() {
-	fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx))
+func (tx *LMDBTx) Dump(short bool) {
+	fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx, short))
 }
 
 // stringifiedLMDBKeysTx reports all the lmdb keys and a
@@ -1520,7 +1589,7 @@ func (tx *LMDBTx) Dump() {
 // By convention, we must return the empty string if there
 // are no keys present. The tests use this to confirm
 // an empty database.
-func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
+func stringifiedLMDBKeysTx(tx *LMDBTx, short bool) (r string) {
 
 	r = "allkeys:[\n"
 	it := NewLMDBIterator(tx, nil)
@@ -1547,7 +1616,9 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
 		srbm = BitmapAsString(rbm)
 
 		r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey))
-		r += "          ......." + srbm + "\n"
+		if !short {
+			r += "          ......." + srbm + "\n"
+		}
 	}
 	r += "]\n   all-in-blake3:" + hash.Blake3sum16([]byte(r))
 

lmdb_other.go

@@ -63,6 +63,10 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper
 	panic("lmdb only available on 64-bit arch")
 }
 
+func (r *lmdbRegistrar) Size() int {
+	panic("lmdb only available on 64-bit arch")
+}
+
 // register each lmdb created under tests, so we
 // can clean them up. This is called by openLMDBWrapper() while
 // holding the r.mu.Lock, since it needs to atomically
@@ -396,7 +400,11 @@ func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
 	panic("lmdb only available on 64-bit arch")
 }
 
-func (tx *LMDBTx) Dump() {
+func (tx *LMDBTx) IsDone() (done bool) {
+	panic("lmdb only available on 64-bit arch")
+}
+
+func (tx *LMDBTx) Dump(short bool) {
 	panic("lmdb only available on 64-bit arch")
 }
 

lmdb_test.go

@@ -94,7 +94,7 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) {
 	w = ww.(*LMDBWrapper)
 
 	// verify it is empty
-	allkeys := w.StringifiedLMDBKeys(nil)
+	allkeys := w.StringifiedLMDBKeys(nil, false)
 	if allkeys != "<empty lmdb database>" {
 		panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys))
 	}
@@ -1013,7 +1013,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
 	if n != 0 {
 		panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n))
 	}
-	allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
+	allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false)
 
 	// should have no keys
 	if allkeys != "<empty lmdb database>" {
@@ -1185,7 +1185,7 @@ func TestLMDB_DeleteIndex(t *testing.T) {
 		exists, err = tx.Contains(index, field, view, shard, v)
 		panicOn(err)
 		if exists {
-			allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
+			allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false)
 			panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))
 		}
 	}
@@ -1240,7 +1240,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) {
 		exists, err = tx.Contains(index, field, view, shard, v<<16)
 		panicOn(err)
 		if exists {
-			allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx))
+			allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false)
 			panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))
 		}
 	}
@@ -1268,3 +1268,33 @@ func TestLMDB_SliceOfShards(t *testing.T) {
 		}
 	}
 }
+
+func TestLMDB_HasData(t *testing.T) {
+
+	db, clean := mustOpenEmptyLMDBWrapper("TestLMDB_SliceOfShards")
+	defer clean()
+	defer db.Close()
+
+	// HasData should start out false.
+	hasAnything, err := db.HasData()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hasAnything {
+		t.Fatalf("HasData reported existing data on an empty database")
+	}
+
+	// check that HasData sees a committed record.
+
+	index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42)
+	LMDBMustSetBitvalue(db, index, field, view, shard, putme)
+
+	// HasData(false) should now report data
+	hasAnything, err = db.HasData()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hasAnything {
+		t.Fatalf("HasData() reported no data on a database that has bits written to it")
+	}
+}

mmap_test.go

@@ -36,7 +36,7 @@ func forceSnapshotsCheckMapping(t *testing.T) {
 	f.Logger = logger.NewLogfLogger(t)
 	defer f.Clean(t)
 
-	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
 	defer tx.Rollback()
 
 	for i := 0; i < f.MaxOpN; i++ {

rbf.go

@@ -34,7 +34,7 @@ import (
 
 // RbfDBWrapper wraps an *rbf.DB
 type RbfDBWrapper struct {
-	Path string
+	path string
 	db   *rbf.DB
 	reg  *rbfDBRegistrar
 	muDb sync.Mutex
@@ -49,6 +49,34 @@ type RbfDBWrapper struct {
 	doAllocZero bool
 }
 
+func (w *RbfDBWrapper) Path() string {
+	return w.path
+}
+
+func (w *RbfDBWrapper) SetHolder(h *Holder) {
+	// don't need it at the moment
+	//w.h = h
+}
+
+func (w *RbfDBWrapper) CleanupTx(tx Tx) {
+	r := tx.(*RBFTx)
+	r.mu.Lock()
+	if r.done {
+		r.mu.Unlock()
+		return
+	}
+	r.done = true
+	r.mu.Unlock()
+
+	// try not to old r.mu while locking w.muDb
+	w.muDb.Lock()
+
+	delete(w.openTx, r)
+	r.o.dbs.Cleanup(tx) // release the read/write lock.
+
+	w.muDb.Unlock()
+}
+
 // rbfDBRegistrar also allows opening the same path twice to
 // result in sharing the same open database handle, and
 // thus the same transactional guarantees.
@@ -60,6 +88,17 @@ type rbfDBRegistrar struct {
 	path2db map[string]*RbfDBWrapper
 }
 
+func (r *rbfDBRegistrar) Size() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	nmp := len(r.mp)
+	npa := len(r.path2db)
+	if nmp != npa {
+		panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa))
+	}
+	return nmp
+}
+
 var globalRbfDBReg *rbfDBRegistrar = newRbfDBRegistrar()
 
 func newRbfDBRegistrar() *rbfDBRegistrar {
@@ -77,29 +116,29 @@ func newRbfDBRegistrar() *rbfDBRegistrar {
 // return the existing instance.
 func (r *rbfDBRegistrar) unprotectedRegister(w *RbfDBWrapper) {
 	r.mp[w] = true
-	r.path2db[w.Path] = w
+	r.path2db[w.path] = w
 }
 
 // unregister removes w from r
 func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) {
 	r.mu.Lock()
 	delete(r.mp, w)
-	delete(r.path2db, w.Path)
+	delete(r.path2db, w.path)
 	r.mu.Unlock()
 }
 
 // rbfPath is a helper for determining the full directory
 // in which the RBF database will be stored.
 func rbfPath(path string) string {
-	if !strings.HasSuffix(path, "-rbfdb") {
-		return path + "-rbfdb"
+	if !strings.HasSuffix(path, "-rbfdb@") {
+		return path + "-rbfdb@"
 	}
 	return path
 }
 
 // OpenDBWrapper opens the database in the path directoy
 // without deleting any prior content. Any
-// database directory will have the "-rbfdb" suffix.
+// database directory will have the "-rbfdb@" suffix.
 //
 // OpenDBWrapper will check the registry and make a new instance only
 // if one does not exist for its path. Otherwise it returns
@@ -114,18 +153,21 @@ func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrappe
 		// creates the effect of having only one DB open per pilosa node.
 		return w, nil
 	}
-	db := rbf.NewDB(path)
+	var db *rbf.DB
+	if doAllocZero {
+		db = rbf.NewDBWithAllocZero(path)
+	} else {
+		db = rbf.NewDB(path)
+	}
 
 	w = &RbfDBWrapper{
 		reg:         r,
-		Path:        path,
+		path:        path,
 		db:          db,
 		doAllocZero: doAllocZero,
 		openTx:      make(map[*RBFTx]bool),
 	}
-
 	r.unprotectedRegister(w)
-	rbf.DoAllocZero = doAllocZero
 
 	err := db.Open()
 	if err != nil {
@@ -143,6 +185,16 @@ type RBFTx struct {
 	o            Txo
 	sn           int64 // serial number
 	Db           *RbfDBWrapper
+
+	done bool
+	mu   sync.Mutex // protect done as it changes state
+}
+
+func (tx *RBFTx) IsDone() (done bool) {
+	tx.mu.Lock()
+	done = tx.done
+	tx.mu.Unlock()
+	return
 }
 
 func (tx *RBFTx) DBPath() string {
@@ -154,19 +206,18 @@ func (tx *RBFTx) Type() string {
 }
 
 func (tx *RBFTx) Rollback() {
-	tx.Db.muDb.Lock()
-	delete(tx.Db.openTx, tx)
-	tx.Db.muDb.Unlock()
-
 	tx.tx.Rollback()
+
+	// must happen after actual rollback
+	tx.Db.CleanupTx(tx)
 }
 
-func (tx *RBFTx) Commit() error {
-	tx.Db.muDb.Lock()
-	delete(tx.Db.openTx, tx)
-	tx.Db.muDb.Unlock()
+func (tx *RBFTx) Commit() (err error) {
+	err = tx.tx.Commit()
 
-	return tx.tx.Commit()
+	// must happen after actual commit
+	tx.Db.CleanupTx(tx)
+	return
 }
 
 func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
@@ -375,8 +426,8 @@ func (tx *RBFTx) Pointer() string {
 	return fmt.Sprintf("%p", tx)
 }
 
-func (tx *RBFTx) Dump() {
-	tx.tx.Dump()
+func (tx *RBFTx) Dump(short bool) {
+	tx.tx.Dump(short)
 }
 
 // Readonly is true if the transaction is not read-and-write, but only doing reads.
@@ -416,6 +467,12 @@ func rbfFieldPrefix(index, field string) string {
 	return string(txkey.FieldPrefix(index, field))
 }
 
+func (w *RbfDBWrapper) HasData() (has bool, err error) {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	return w.db.HasData(false) // false => any prior attempt at write means we "have data"
+}
+
 func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error {
 	w.muDb.Lock()
 	defer w.muDb.Unlock()

rbf/cursorx.go

@@ -26,10 +26,13 @@ import (
 
 // if enableRowCache, then we must not return mmap-ed memory
 // directly, but only a copy.
-const EnableRowCache = true
+const EnableRowCache = false
 
-// makes a copy, BUT doesn't do the zero out for now TODO(jea) zero out actually to detect
-// access past tx.
+// DoAllocZero means we copy mmap read data and
+// wipe it afterwards to catch retention of data
+// past Tx.Rollback which was a big problem.
+// This should be set by NewDBWithAllocZero and never changed
+// afterwards in order to avoid a data race.
 var DoAllocZero bool
 
 //probably should just implement the container interface

rbf/db.go

@@ -58,6 +58,14 @@ type DB struct {
 	MaxSize int64
 }
 
+// NewDBWithAllocZero sets DoAllocZero true and
+// returns a new instance of DB. We set it here
+// to avoid a data race afterwards.
+func NewDBWithAllocZero(path string) *DB {
+	DoAllocZero = true
+	return NewDB(path)
+}
+
 // NewDB returns a new instance of DB.
 func NewDB(path string) *DB {
 	db := &DB{
@@ -544,6 +552,68 @@ func (db *DB) closeWALSegments() (err error) {
 	return err
 }
 
+// HasData with requireOneHotBit=false returns
+// hasAnyRecords true if any record has been stored,
+// even if the value for that bitmap record turned out to have
+// no bits hot (be all zeroes).
+//
+// In this case, we are taking the attempted storage
+// of any named bitmap into the database as evidence
+// that the db is in use, and we return hasAnyRecords true.
+//
+// Conversely, if requireOneHotBit is true, then a
+// database consisting of only a named bitmap with
+// an all zeroes (no bits hot)
+// will return hasAnyRecords false. We must find at
+// least a single hot bit inside the db
+// in order to return hasAnyRecords true.
+//
+// HasData is used by backend migration and blue/green checks.
+//
+// If there is a disk error we return (false, error), so always
+// check the error before deciding if hasAnyRecords is valid.
+//
+// We will internally create and rollback a read-only
+// transaction to answer this query.
+func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) {
+
+	// Read a list of all bitmaps in Tx.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return false, err
+	}
+	defer tx.Rollback()
+
+	records, err := tx.RootRecords()
+	if err != nil {
+		return false, err
+	}
+	// Loop over each bitmap and attempt to move to the first cell.
+	// If we can move to a cell then we have at least one record.
+	for _, record := range records {
+		// Fetch cursor for bitmap.
+		cur, err := tx.Cursor(record.Name)
+		if err != nil {
+			return false, err
+		} else if cur == nil {
+			continue // no bitmap
+		}
+		if !requireOneHotBit {
+			return true, nil
+		}
+		// INVAR: requireOneHotBit true
+
+		// Check if we can move to the first cell.
+		if err := cur.First(); err == io.EOF {
+			continue // no data in bitmap
+		} else if err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
 // Size returns the size of the database & WAL, in bytes.
 func (db *DB) Size() (int64, error) {
 

rbf/db_test.go

@@ -191,3 +191,83 @@ func TestDB_BeginWithExclusiveLock(t *testing.T) {
 		}
 	})
 }
+
+func TestDB_HasData(t *testing.T) {
+
+	db := MustOpenDB(t)
+	defer MustCloseDB(t, db)
+
+	// HasData should start out false.
+	const requireOneHotBit = true
+	hasAnything, err := db.HasData(!requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hasAnything {
+		t.Fatalf("HasData reported existing data on an empty database")
+	}
+
+	hasAnything, err = db.HasData(requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hasAnything {
+		t.Fatalf("HasData reported existing data on an empty database")
+	}
+
+	// check that HasData sees a committed record.
+
+	// Create bitmap with no hot bits.
+	if tx, err := db.Begin(true); err != nil {
+		t.Fatal(err)
+	} else if err := tx.CreateBitmap("x"); err != nil {
+		t.Fatal(err)
+	} else if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	// HasData(false) should now report seeing the 'x' record, even though
+	// the value is an empty bitmap, since !requireOneHotBit.
+	hasAnything, err = db.HasData(!requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hasAnything {
+		t.Fatalf("HasData(!requireOneHotBit) reported no data on a database that has 'x' written to it")
+	}
+
+	// HasData(requireOneHotBit) should report false, since we have no hot bits.
+	hasAnything, err = db.HasData(requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hasAnything {
+		t.Fatalf("HasData(requireOneHotBit) reported data on a database that has 'x' -> empty bitmap")
+	}
+
+	// hot up a bit.
+	if tx, err := db.Begin(true); err != nil {
+		t.Fatal(err)
+	} else if _, err := tx.Add("x", rand.Uint64()); err != nil {
+		t.Fatal(err)
+	} else if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Now it shouldn't matter, we should get HasData true either way
+	// HasData(requireOneHotBit) should report false, since we have no hot bits.
+	hasAnything, err = db.HasData(requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hasAnything {
+		t.Fatalf("HasData should have seen the hot bit")
+	}
+	hasAnything, err = db.HasData(!requireOneHotBit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hasAnything {
+		t.Fatalf("HasData should have seen the hot bit")
+	}
+}

rbf/tx.go

@@ -1334,10 +1334,10 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
 	panic("emptyContainerIterator never has any Values")
 }
 
-func (tx *Tx) Dump() {
-	fmt.Println(tx.DumpString())
+func (tx *Tx) Dump(short bool) {
+	fmt.Println(tx.DumpString(short))
 }
-func (tx *Tx) DumpString() (r string) {
+func (tx *Tx) DumpString(short bool) (r string) {
 
 	r = "allkeys:[\n"
 
@@ -1365,7 +1365,7 @@ func (tx *Tx) DumpString() (r string) {
 			ckey := cell.Key
 			ct := toContainer(cell, tx)
 
-			s := stringOfCkeyCt(ckey, ct, rr.Name)
+			s := stringOfCkeyCt(ckey, ct, rr.Name, short)
 			r += s
 			n++
 		}
@@ -1419,7 +1419,7 @@ func bitmapAsString(rbm *roaring.Bitmap) (r string) {
 	return r + ")"
 }
 
-func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string) {
+func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short bool) (s string) {
 
 	by := containerToBytes(ct)
 	hash := hash.Blake3sum16(by)
@@ -1433,7 +1433,9 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string
 	bkey := pre + fmt.Sprintf("ckey@%020d", ckey)
 
 	s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
-	s += "          ......." + srbm + "\n"
+	if !short {
+		s += "          ......." + srbm + "\n"
+	}
 	return
 }
 

rbf/tx_test.go

@@ -483,7 +483,7 @@ func TestTx_Dump(t *testing.T) {
 	}
 
 	// test that we don't crash, and get *something* back
-	s := tx.DumpString()
+	s := tx.DumpString(true)
 	if s == "" {
 		panic("should have had 3 containers!")
 	}

roaring/fuzzer.go

@@ -191,14 +191,14 @@ func FuzzRoaringOps(data []byte) int {
 	expected = make([]uint64, 0)
 	actual = make([]uint64, 0)
 	forEachInSlice(s1, func(v uint64) { expected = append(expected, v) })
-	bm1.ForEach(func(v uint64) { actual = append(actual, v) })
+	bm1.ForEach(func(v uint64) error { actual = append(actual, v); return nil })
 	if !reflect.DeepEqual(expected, actual) {
 		panic(fmt.Sprintf("for each:\n expected %v\n got %v", expected, actual))
 	}
 	expected = make([]uint64, 0)
 	actual = make([]uint64, 0)
 	forEachInSlice(s2, func(v uint64) { expected = append(expected, v) })
-	bm2.ForEach(func(v uint64) { actual = append(actual, v) })
+	bm2.ForEach(func(v uint64) error { actual = append(actual, v); return nil })
 	if !reflect.DeepEqual(expected, actual) {
 		panic(fmt.Sprintf("for each:\n expected %v\n got %v", expected, actual))
 	}
@@ -206,14 +206,14 @@ func FuzzRoaringOps(data []byte) int {
 	expected = make([]uint64, 0)
 	actual = make([]uint64, 0)
 	forEachInRangeSlice(s1, start, end, func(v uint64) { expected = append(expected, v) })
-	bm1.ForEachRange(start, end, func(v uint64) { actual = append(actual, v) })
+	bm1.ForEachRange(start, end, func(v uint64) error { actual = append(actual, v); return nil })
 	if !reflect.DeepEqual(expected, actual) {
 		panic(fmt.Sprintf("for each in range:\n expected %v\n got %v", expected, actual))
 	}
 	expected = make([]uint64, 0)
 	actual = make([]uint64, 0)
 	forEachInRangeSlice(s2, start, end, func(v uint64) { expected = append(expected, v) })
-	bm2.ForEachRange(start, end, func(v uint64) { actual = append(actual, v) })
+	bm2.ForEachRange(start, end, func(v uint64) error { actual = append(actual, v); return nil })
 	if !reflect.DeepEqual(expected, actual) {
 		panic(fmt.Sprintf("for each in range:\n expected %v\n got %v", expected, actual))
 	}

roaring/roaring.go

@@ -3301,8 +3301,32 @@ func (c *Container) arrayRemove(v uint16) (*Container, bool) {
 	c = c.Thaw()
 	array = c.array()
 
-	array = append(array[:i], array[i+1:]...)
-	c.setArray(array)
+	const needCopyOnWriteDueToReadOnlyMmap = true
+
+	// TODO(jea) quick benchmarks don't show less performance if needCopyOnWriteDueToReadOnlyMmap
+	// is true, but we may need more rigorous measurement.
+	//
+	// Don't have a COW:
+	// all benchmarks in roaring/
+	// ok  	github.com/pilosa/pilosa/v2/roaring	217.138s
+	//
+	// ok, have a COW:
+	// all benchmarks in roaring/
+	// ok  	github.com/pilosa/pilosa/v2/roaring	214.295s
+
+	if needCopyOnWriteDueToReadOnlyMmap {
+		n := len(array)
+		array2 := make([]uint16, n-1)
+		copy(array2, array[:i])
+		copy(array2[i:], array[i+1:])
+		c.setArray(array2)
+	} else {
+		// seg fault here with read-only mmap; go 1.14.7 linux.
+		// example: cap =  7  i =  0  len = 7
+		// the append tries to write read-only memory?
+		array = append(array[:i], array[i+1:]...)
+		c.setArray(array)
+	}
 	return c, true
 }
 

rrtx.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"sync"
 	"sync/atomic"
 
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -35,18 +36,30 @@ type RoaringTx struct {
 	fragment *fragment
 	o        Txo
 	sn       int64 // serial number
+
+	done bool
+	mu   sync.Mutex // protect done as it changes state
+
+	w *RoaringWrapper
+}
+
+func (tx *RoaringTx) IsDone() (done bool) {
+	tx.mu.Lock()
+	done = tx.done
+	tx.mu.Unlock()
+	return
 }
 
 func (tx *RoaringTx) Type() string {
 	return RoaringTxn
 }
 
-func (tx *RoaringTx) Dump() {
-	fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(false, false))
+func (tx *RoaringTx) Dump(short bool) {
+	fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, tx.o))
 }
 
 func (tx *RoaringTx) UseRowCache() bool {
-	return true
+	return false
 }
 
 func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
@@ -125,11 +138,14 @@ func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, chang
 	frag.incrementOpN(changedN)
 }
 
-// Rollback is a no-op as Roaring does not support transactions.
-func (tx *RoaringTx) Rollback() {}
+// Rollback
+func (tx *RoaringTx) Rollback() {
+	tx.w.CleanupTx(tx)
+}
 
-// Commit is a no-op as Roaring does not support transactions.
+// Commit
 func (tx *RoaringTx) Commit() error {
+	tx.w.CleanupTx(tx)
 	return nil
 }
 
@@ -359,80 +375,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
 	return frag.storage, nil
 }
 
-var globalRoaringReg = &RoaringStore{}
-
-func (r *RoaringStore) OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error) {
-	return r, nil
-}
-
-func (w *RoaringStore) DeleteDBPath(dbs *DBShard) error {
-	return os.RemoveAll(dbs.Path)
-}
-
-func (r *RoaringStore) Close() error {
-	return nil
-}
-
-func (w *RoaringStore) OpenListString() (r string) {
-	return "RoaringStore.OpenListString() not yet implemented"
-}
-
-func (w *RoaringStore) OpenSnList() (sns []int64) {
-	return nil
-}
-
-type RoaringStore struct{}
-
-func NewRoaringStore() *RoaringStore {
-	return &RoaringStore{}
-}
-
-var globalNextTxSnRoaring int64
-
-func (db *RoaringStore) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
-	sn := atomic.AddInt64(&globalNextTxSnRoaring, 1)
-	return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment, o: o, sn: sn}, nil
-}
-
-func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
-
-	// match txn sn count vs lmdb/etc.
-	atomic.AddInt64(&globalNextTxSnRoaring, 1)
-
-	// under blue-green badger_roaring, the directory will not be found, b/c badger will have
-	// already done the os.RemoveAll().	BUT, RemoveAll returns nil error in this case. Docs:
-	// "If the path does not exist, RemoveAll returns nil (no error)"
-	err := os.RemoveAll(fieldPath)
-	if err != nil {
-		return errors.Wrap(err, "removing directory")
-	}
-	return nil
-}
-
-// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
-// The fragment should be closed before this.
-func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
-
-	// match txn sn count vs lmdb/etc.
-	atomic.AddInt64(&globalNextTxSnRoaring, 1)
-
-	fragment, ok := frag.(*fragment)
-	if !ok {
-		return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
-	}
-
-	// Delete fragment file.
-	if err := os.Remove(fragment.path); err != nil {
-		return errors.Wrap(err, "deleting fragment file")
-	}
-
-	// Delete fragment cache file.
-	if err := os.Remove(fragment.cachePath()); err != nil {
-		return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
-	}
-	return nil
-}
-
 func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
 	file, err := os.Open(fragmentPathForRoaring) // open the fragment file
 	if err != nil {
@@ -459,3 +401,193 @@ func (tx *RoaringTx) Options() Txo {
 func (tx *RoaringTx) Sn() int64 {
 	return tx.sn
 }
+
+//////// registrar and wrapper machinery
+
+// roaringRegistrar mirrors the machinery expected
+// for all backends for the roaring files approach.
+//
+type roaringRegistrar struct {
+	mu sync.Mutex
+	mp map[*RoaringWrapper]bool
+
+	path2db map[string]*RoaringWrapper
+}
+
+func (r *roaringRegistrar) Size() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	nmp := len(r.mp)
+	npa := len(r.path2db)
+	if nmp != npa {
+		panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa))
+	}
+	return nmp
+}
+
+var globalRoaringReg *roaringRegistrar = newRoaringRegistrar()
+
+func newRoaringRegistrar() *roaringRegistrar {
+	return &roaringRegistrar{
+		mp:      make(map[*RoaringWrapper]bool),
+		path2db: make(map[string]*RoaringWrapper),
+	}
+}
+
+func (r *roaringRegistrar) unprotectedRegister(w *RoaringWrapper) {
+	r.mp[w] = true
+	r.path2db[w.path] = w
+}
+
+// unregister removes w from r
+func (r *roaringRegistrar) unregister(w *RoaringWrapper) {
+	r.mu.Lock()
+	delete(r.mp, w)
+	delete(r.path2db, w.path)
+	r.mu.Unlock()
+}
+
+// openRoaringDB will check the registry and make a new instance only
+// if one does not exist for its path0. Otherwise it returns
+// the existing instance.
+func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error) {
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	w, ok := r.path2db[path]
+	if ok {
+		return w, nil
+	}
+	// otherwise, make a new roaring and store it in globalRoaringReg
+	w = &RoaringWrapper{
+		reg:  r,
+		path: path,
+	}
+	r.unprotectedRegister(w)
+
+	return w, nil
+}
+
+func (w *RoaringWrapper) SetHolder(h *Holder) {
+	w.h = h
+}
+
+func (w *RoaringWrapper) Path() string {
+	return w.path
+}
+
+func (w *RoaringWrapper) HasData() (has bool, err error) {
+	return w.h.HasRoaringData()
+}
+
+func (w *RoaringWrapper) CleanupTx(tx Tx) {
+	r := tx.(*RoaringTx)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.done {
+		return
+	}
+	r.done = true
+
+	r.o.dbs.Cleanup(tx) // release the read/write lock.
+}
+
+func (w *RoaringWrapper) OpenListString() (r string) {
+	return "RoaringWrapper.OpenListString() not yet implemented"
+}
+
+func (w *RoaringWrapper) OpenSnList() (slc []int64) {
+	return nil
+}
+
+// statically confirm that RoaringTx satisfies the Tx interface.
+var _ Tx = (*RoaringTx)(nil)
+
+// RoaringWrapper provides the NewTx() method.
+type RoaringWrapper struct {
+	muDb sync.Mutex
+
+	path string
+
+	h *Holder
+
+	reg *roaringRegistrar
+
+	// make RoaringWrapper.Close() idempotent, avoiding panic on double Close()
+	closed bool
+}
+
+var globalNextTxSnRoaring int64
+
+func (w *RoaringWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
+
+	sn := atomic.AddInt64(&globalNextTxSnRoaring, 1)
+	return &RoaringTx{
+		write:    o.Write,
+		Field:    o.Field,
+		Index:    o.Index,
+		fragment: o.Fragment,
+		o:        o,
+		sn:       sn,
+		w:        w,
+	}, nil
+}
+
+// Close shuts down the Roaring database.
+func (w *RoaringWrapper) Close() (err error) {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+	if !w.closed {
+		w.reg.unregister(w)
+		w.closed = true
+	}
+	return nil
+}
+
+func (w *RoaringWrapper) IsClosed() (closed bool) {
+	w.muDb.Lock()
+	closed = w.closed
+	w.muDb.Unlock()
+	return
+}
+
+func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) {
+	return os.RemoveAll(dbs.Path)
+}
+
+func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error {
+
+	// match txn sn count vs lmdb/etc.
+	atomic.AddInt64(&globalNextTxSnRoaring, 1)
+
+	// under blue-green badger_roaring, the directory will not be found, b/c badger will have
+	// already done the os.RemoveAll().	BUT, RemoveAll returns nil error in this case. Docs:
+	// "If the path does not exist, RemoveAll returns nil (no error)"
+	err := os.RemoveAll(fieldPath)
+	if err != nil {
+		return errors.Wrap(err, "removing directory")
+	}
+	return nil
+}
+
+func (w *RoaringWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
+
+	// match txn sn count vs lmdb/etc.
+	atomic.AddInt64(&globalNextTxSnRoaring, 1)
+
+	fragment, ok := frag.(*fragment)
+	if !ok {
+		return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
+	}
+
+	// Delete fragment file.
+	if err := os.Remove(fragment.path); err != nil {
+		return errors.Wrap(err, "deleting fragment file")
+	}
+
+	// Delete fragment cache file.
+	if err := os.Remove(fragment.cachePath()); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
+	}
+	return nil
+}

rrtx_internal_test.go

@@ -0,0 +1,63 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"os"
+	"testing"
+)
+
+func TestRoaring_HasData(t *testing.T) {
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+	os.Setenv("PILOSA_TXSRC", "roaring")
+
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
+	db, err := globalRoaringReg.OpenDBWrapper(idx.path, false)
+	panicOn(err)
+	db.SetHolder(idx.holder)
+
+	// HasData should start out false.
+	hasAnything, err := db.HasData()
+	panicOn(err)
+
+	if hasAnything {
+		t.Fatalf("HasData reported existing data on an empty database")
+	}
+
+	// check that HasData sees a committed record.
+
+	field, shard := "f", uint64(123)
+
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	f, err := idx.CreateField(field)
+	panicOn(err)
+	_, err = f.SetBit(tx, 1, 1, nil)
+	panicOn(err)
+	panicOn(tx.Commit())
+
+	hasAnything, err = db.HasData()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hasAnything {
+		t.Fatalf("HasData() reported no data on a database that has 'x' written to it")
+	}
+}

server/server.go

@@ -290,6 +290,9 @@ func (m *Command) SetupServer() error {
 	txty := pilosa.MustTxsrcToTxtype(m.Config.Txsrc) // will panic on unknown Txsrc.
 	os.Setenv("PILOSA_TXSRC", m.Config.Txsrc)
 	m.logger.Printf("using Txsrc '%v'/%v", m.Config.Txsrc, txty)
+	if len(txty) == 2 {
+		m.logger.Printf("blue='%v' / green='%v'", txty[0], txty[1])
+	}
 
 	// validateAddrs sets the appropriate values for Bind and Advertise
 	// based on the inputs. It is not responsible for applying defaults, although

stattx.go

@@ -310,8 +310,8 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit
 	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 }
 
-func (c *statTx) Dump() {
-	c.b.Dump()
+func (c *statTx) Dump(short bool) {
+	c.b.Dump(short)
 }
 
 func (c *statTx) Readonly() bool {
@@ -433,6 +433,10 @@ func (c *statTx) UseRowCache() bool {
 	return c.b.UseRowCache()
 }
 
+func (c *statTx) IsDone() (done bool) {
+	return c.b.IsDone()
+}
+
 func (c *statTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
 	me := kAdd
 

synthload/synthload_test.go

@@ -0,0 +1,361 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package synthload
+
+/* // still work in progress, comment out for now, since we
+   // are getting a hang on go1.13 in CI
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_SynthLoad_ImportSchema(t *testing.T) {
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+
+	tarball := "testindex.tar.gz"
+
+	// get the holder.Path to write to
+	h := m0.API.Holder()
+	target := h.Path()
+	panicOn(h.Close())
+
+	panicOn(unpackTarball(tarball, target))
+
+	// reopen
+	panicOn(h.Open())
+
+	qs := strings.Split(pql, "\n\n")
+	//vv("qs = '%#v'", qs)
+	for _, q := range qs {
+		req := &pilosa.QueryRequest{
+			// Index to execute query against.
+			Index: "testindex",
+
+			// The query string to parse and execute.
+			Query: q,
+
+			// // The shards to include in the query execution.
+			// // If empty, all shards are included.
+			// Shards []uint64
+
+			// // Return column attributes, if true.
+			// ColumnAttrs bool
+
+			// // Do not return row attributes, if true.
+			// ExcludeRowAttrs bool
+
+			// // Do not return columns, if true.
+			// ExcludeColumns bool
+
+			// // If true, indicates that query is part of a larger distributed query.
+			// // If false, this request is on the originating node.
+			// Remote bool
+
+			// // Should we profile this query?
+			// Profile bool
+
+			// // Additional data associated with the query, in cases where there's
+			// // row-style inputs for precomputed values.
+			// EmbeddedData []*Row
+		}
+
+		qr, err := m0.API.Query(context.Background(), req)
+		panicOn(err)
+		vv("qr = '%#v'", qr)
+	}
+}
+
+var _ = applySchema
+
+func applySchema(m0 *test.Command, schemaStr string) {
+	// don't need schema now that we import the tarball, it has it all.
+	schema := &pilosa.Schema{}
+	err := json.NewDecoder(bytes.NewBufferString(schemaStr)).Decode(schema)
+	panicOn(err)
+
+	ctx := context.Background()
+	remote := false
+	err = m0.API.ApplySchema(ctx, schema, remote)
+	panicOn(err)
+}
+
+func unpackTarball(tarball, target string) error {
+
+	vv("target = '%v'", target)
+	fd, err := os.Open(tarball)
+	panicOn(err)
+	defer fd.Close()
+	gz, err := gzip.NewReader(fd)
+	panicOn(err)
+	defer gz.Close()
+
+	tarReader := tar.NewReader(gz)
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			panic(err)
+		}
+
+		path := filepath.Join(target, header.Name)
+		info := header.FileInfo()
+		if info.IsDir() {
+			if err = os.MkdirAll(path, info.Mode()); err != nil {
+				panicOn(err)
+			}
+			continue
+		}
+
+		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
+		panicOn(err)
+
+		_, err = io.Copy(file, tarReader)
+		panicOn(err)
+		file.Close()
+	}
+	return nil
+}
+
+// mkdir o; bangbang schemator -o o
+var pql = `
+Count(All())
+
+Rows(field='bools',limit=10)
+Count(Row(bools='bool'))
+Count(Not(Row(bools='bool')))
+Count(Intersect(Row(bools='bool')))
+Count(Difference(Union(Row(bools='bool')),Intersect(Row(bools='bool'))))
+GroupBy(Rows(field='bools'),limit=10)
+
+Rows(field='bools-exists',limit=10)
+Count(Row(bools-exists='bool'))
+Count(Not(Row(bools-exists='bool')))
+Count(Intersect(Row(bools-exists='bool')))
+Count(Difference(Union(Row(bools-exists='bool')),Intersect(Row(bools-exists='bool'))))
+GroupBy(Rows(field='bools-exists'),limit=10)
+
+GroupBy(Rows(field='int'),limit=10)
+Min(field='int')
+Max(field='int')
+Sum(field='int')
+
+Rows(field='mutex',limit=10)
+Count(Row(mutex='12'))
+Count(Not(Row(mutex='12')))
+Count(Row(mutex='13'))
+Count(Not(Row(mutex='13')))
+Count(Row(mutex='0'))
+Count(Not(Row(mutex='0')))
+Count(Row(mutex='5'))
+Count(Not(Row(mutex='5')))
+Count(Row(mutex='2'))
+Count(Not(Row(mutex='2')))
+Count(Row(mutex='14'))
+Count(Not(Row(mutex='14')))
+Count(Row(mutex='3'))
+Count(Not(Row(mutex='3')))
+Count(Row(mutex='18'))
+Count(Not(Row(mutex='18')))
+Count(Row(mutex='7'))
+Count(Not(Row(mutex='7')))
+Count(Row(mutex='19'))
+Count(Not(Row(mutex='19')))
+Count(Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')))
+Count(Difference(Union(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')),Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19'))))
+GroupBy(Rows(field='mutex'),limit=10)
+
+Rows(field='set',limit=10)
+Count(Row(set='v621'))
+Count(Not(Row(set='v621')))
+Count(Row(set='v997'))
+Count(Not(Row(set='v997')))
+Count(Row(set='v772'))
+Count(Not(Row(set='v772')))
+Count(Row(set='v340'))
+Count(Not(Row(set='v340')))
+Count(Row(set='v766'))
+Count(Not(Row(set='v766')))
+Count(Row(set='v416'))
+Count(Not(Row(set='v416')))
+Count(Row(set='v481'))
+Count(Not(Row(set='v481')))
+Count(Row(set='v581'))
+Count(Not(Row(set='v581')))
+Count(Row(set='v591'))
+Count(Not(Row(set='v591')))
+Count(Row(set='v675'))
+Count(Not(Row(set='v675')))
+Count(Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')))
+Count(Difference(Union(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')),Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675'))))
+GroupBy(Rows(field='set'),limit=10)
+
+Rows(field='string',limit=10)
+Count(Row(string='KPFGWOYUGTCF'))
+Count(Not(Row(string='KPFGWOYUGTCF')))
+Count(Row(string='VNSHWDTNELDA'))
+Count(Not(Row(string='VNSHWDTNELDA')))
+Count(Row(string='LTDOGKKFGZPW'))
+Count(Not(Row(string='LTDOGKKFGZPW')))
+Count(Row(string='EETNIIDCDZHB'))
+Count(Not(Row(string='EETNIIDCDZHB')))
+Count(Row(string='ATWOMTRASGHP'))
+Count(Not(Row(string='ATWOMTRASGHP')))
+Count(Row(string='KOJUBJQVMXZL'))
+Count(Not(Row(string='KOJUBJQVMXZL')))
+Count(Row(string='NOXDEYPTGZBH'))
+Count(Not(Row(string='NOXDEYPTGZBH')))
+Count(Row(string='DLSRPRADBPKX'))
+Count(Not(Row(string='DLSRPRADBPKX')))
+Count(Row(string='WJUEQABAEEUX'))
+Count(Not(Row(string='WJUEQABAEEUX')))
+Count(Row(string='PFRGBFDLHHPK'))
+Count(Not(Row(string='PFRGBFDLHHPK')))
+Count(Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')))
+Count(Difference(Union(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')),Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK'))))
+GroupBy(Rows(field='string'),limit=10)
+
+GroupBy(Rows(field='time'),limit=10)
+`
+
+//  datagen --source=kitchensink --pilosa.index=testindex --start-from=1 --end-at=1000 --pilosa.batch-size=10000 --pilosa.hosts localhost:10101
+// curl localhost:10101/schema
+
+var _ = schemaString
+var schemaString = `
+{
+  "indexes": [
+    {
+      "name": "testindex",
+      "createdAt": 1599601704641744600,
+      "options": {
+        "keys": false,
+        "trackExistence": true
+      },
+      "fields": [
+        {
+          "name": "bools",
+          "createdAt": 1599601704647053000,
+          "options": {
+            "type": "set",
+            "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": true
+          }
+        },
+        {
+          "name": "bools-exists",
+          "createdAt": 1599601704643225900,
+          "options": {
+            "type": "set",
+            "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": true
+          }
+        },
+        {
+          "name": "decimal",
+          "createdAt": 1599601704642064400,
+          "options": {
+            "type": "decimal",
+            "base": 0,
+            "scale": 2,
+            "bitDepth": 63,
+            "min": -92233720368547760,
+            "max": 92233720368547760,
+            "keys": false
+          }
+        },
+        {
+          "name": "int",
+          "createdAt": 1599601704645510100,
+          "options": {
+            "type": "int",
+            "base": 0,
+            "bitDepth": 63,
+            "min": -9223372036854776000,
+            "max": 9223372036854776000,
+            "keys": false,
+            "foreignIndex": ""
+          }
+        },
+        {
+          "name": "mutex",
+          "createdAt": 1599601704647187200,
+          "options": {
+            "type": "mutex",
+            "cacheType": "ranked",
+            "cacheSize": 500,
+            "keys": true
+          }
+        },
+        {
+          "name": "set",
+          "createdAt": 1599601704643664400,
+          "options": {
+            "type": "set",
+            "cacheType": "lru",
+            "cacheSize": 1,
+            "keys": true
+          }
+        },
+        {
+          "name": "string",
+          "createdAt": 1599601704644342000,
+          "options": {
+            "type": "set",
+            "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": true
+          }
+        },
+        {
+          "name": "time",
+          "createdAt": 1599601704643013400,
+          "options": {
+            "type": "time",
+            "timeQuantum": "YMD",
+            "keys": true,
+            "noStandardView": false
+          }
+        }
+      ],
+      "shardWidth": 1048576
+    }
+  ]
+}
+`
+
+*/

synthload/vprint.go

@@ -0,0 +1,170 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package synthload
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+var _ = panicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+var _ = stack
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}

test/holder.go

@@ -160,7 +160,7 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time
 	}
 
 	shard := columnID / pilosa.ShardWidth
-	tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
+	tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = f.SetBit(tx, rowID, columnID, t)
@@ -179,7 +179,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
 	}
 
 	shard := columnID / pilosa.ShardWidth
-	tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
+	tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = f.ClearBit(tx, rowID, columnID)
@@ -205,7 +205,7 @@ func (h *Holder) SetValue(index, field string, columnID uint64, value int64) *In
 		panic(err)
 	}
 	shard := columnID / pilosa.ShardWidth
-	tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
+	tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard})
 	defer tx.Rollback()
 
 	_, err = f.SetValue(tx, columnID, value)
@@ -226,7 +226,7 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) {
 		panic(err)
 	}
 	shard := columnID / pilosa.ShardWidth
-	tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index, Shard: shard})
+	tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: false, Index: idx.Index, Shard: shard})
 	defer tx.Rollback()
 
 	val, exists, err := f.Value(tx, columnID)

test/index.go

@@ -59,7 +59,7 @@ func (i *Index) Reopen() error {
 	if err := i.Index.Close(); err != nil {
 		return err
 	}
-	return i.Index.Open(false)
+	return i.Index.Open()
 }
 
 // CreateField creates a field with the given options.

tx.go

@@ -68,6 +68,11 @@ type Tx interface {
 	// Commit makes the updates in the Tx visible to subsequent transactions.
 	Commit() error
 
+	// IsDone must return true if Rollback() or Commit() has already
+	// been called. Otherwise it must return false. This allows
+	// DBWrapper.CleanupTx(tx Tx) to be idempotent.
+	IsDone() bool
+
 	// Readonly returns the flag this transaction was created with
 	// during NewTx. If the transaction is writable, it will return false.
 	Readonly() bool
@@ -197,7 +202,7 @@ type Tx interface {
 	Group() *TxGroup
 
 	// Dump is for debugging, what does this Tx see as its database?
-	Dump()
+	Dump(short bool)
 
 	// Options returns the options used to create this Tx. This
 	// can be implementd by embedding Txo, and Txo provides the

txfactory.go

@@ -37,6 +37,7 @@ const (
 	RoaringTxn string = "roaring"
 	LmdbTxn    string = "lmdb"
 	RBFTxn     string = "rbf"
+	BadgerTxn  string = "badger"
 )
 
 // DefaultTxsrc is set here. pilosa/server/config.go references it
@@ -389,6 +390,10 @@ type TxFactory struct {
 	holder *Holder
 
 	blueGreenReg *blueGreenRegistry
+
+	// allow holder to activate blue-green checking only
+	// once we have synced both sides at start up time.
+	blueGreenOff bool
 }
 
 func (f *TxFactory) Types() []txtype {
@@ -403,20 +408,41 @@ const (
 	roaringTxn txtype = 1 // these don't really have any transactions
 	rbfTxn     txtype = 2
 	lmdbTxn    txtype = 3
+	badgerTxn  txtype = 4
 )
 
+// these need to be skipped by the holder.go field scanner that
+// calls IsTxDatabasePath
+var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, badgerTxn}
+
+// FileSuffix is used to determine backend directory names.
+// We append '@' to be sure we never collide with a field name
+// inside the index directory. In the future for different
+// versions of the same backend, there might be version
+// identifier tacked on too.
 func (ty txtype) FileSuffix() string {
 	switch ty {
 	case roaringTxn:
 		return ""
 	case rbfTxn:
-		return "-rbfdb"
+		return "-rbfdb@"
 	case lmdbTxn:
-		return "-lmdb"
+		return "-lmdb@"
+	case badgerTxn:
+		return "-badgerdb@"
 	}
 	panic(fmt.Sprintf("unkown txtype %v", int(ty)))
 }
 
+func (txf *TxFactory) IsTxDatabasePath(path string) bool {
+	for _, ty := range allTypesWithSuffixes {
+		if strings.HasSuffix(path, ty.FileSuffix()) {
+			return true
+		}
+	}
+	return false
+}
+
 func (txf *TxFactory) NeedsSnapshot() (b bool) {
 	for _, ty := range txf.types {
 		switch ty {
@@ -447,6 +473,8 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
 			types = append(types, rbfTxn)
 		case LmdbTxn: // "lmdb"
 			types = append(types, lmdbTxn)
+		case BadgerTxn: // "badger"
+			types = append(types, badgerTxn)
 		default:
 			panic(fmt.Sprintf("unknown txsrc '%v'", s))
 		}
@@ -467,12 +495,15 @@ func NewTxFactory(txsrc string, holderDir string, holder *Holder) (f *TxFactory,
 	types := MustTxsrcToTxtype(txsrc)
 
 	f = &TxFactory{
-		types:        types,
-		typeOfTx:     txsrc,
-		holder:       holder,
-		blueGreenReg: newBlueGreenReg(),
+		types:    types,
+		typeOfTx: txsrc,
+		holder:   holder,
+	}
+	if len(types) == 2 {
+		f.blueGreenReg = newBlueGreenReg(types)
 	}
 	f.dbPerShard = f.NewDBPerShard(types, holderDir)
+
 	return f, err
 }
 
@@ -489,6 +520,8 @@ type Txo struct {
 	per *DBPerShard
 
 	Group *TxGroup
+
+	blueGreenOff bool
 }
 
 func (o Txo) String() string {
@@ -528,7 +561,7 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 	return nil
 }
 
-func (f *TxFactory) CloseDB() (err error) {
+func (f *TxFactory) Close() (err error) {
 	if f.dbsClosed {
 		return nil
 	}
@@ -684,7 +717,7 @@ func (g *TxGroup) AbortGroup() {
 }
 
 func (f *TxFactory) NewTx(o Txo) (txn Tx) {
-	f.mu.Lock() // deadlock here
+	f.mu.Lock()
 	defer f.mu.Unlock()
 
 	defer func() {
@@ -693,6 +726,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
 		}
 	}()
 
+	o.blueGreenOff = f.blueGreenOff
+
 	indexName := ""
 	if o.Index != nil {
 		indexName = o.Index.name
@@ -711,8 +746,15 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
 	// per-shard database. Opens a new one if needed.
 	dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index)
 	panicOn(err)
+
+	if dbs.Shard != o.Shard {
+		panic(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard)))
+	}
+	//vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W)
+
 	o.dbs = dbs          // our specific database per shard.
 	o.per = f.dbPerShard // for top level debug Dumps
+
 	tx, err := dbs.NewTx(o.Write, indexName, o)
 	if err != nil {
 		panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
@@ -730,6 +772,8 @@ func (ty txtype) String() string {
 		return "rbfTxn"
 	case lmdbTxn:
 		return "lmdbTxn"
+	case badgerTxn:
+		return "badgerTxn"
 	}
 	panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
 }
@@ -767,7 +811,7 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
 
 // hashOnly means only show the value hash, not the content bits.
 // showOps means display the ops log.
-func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) {
+func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) {
 	paths, err := listFilesUnderDir(idx.path, false, "", true)
 	panicOn(err)
 	index := idx.name
@@ -784,9 +828,9 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) {
 		s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout)
 		panicOn(err)
 		//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
-		if s == "" {
-			s = "<empty bitmap>"
-		}
+		//if s == "" {
+		//s = "<empty bitmap>"
+		//}
 		r += s
 		n++
 	}
@@ -1073,3 +1117,154 @@ func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) {
 	}
 	tw.Flush()
 }
+
+var _ = anyGlobalDBWrappersStillOpen // happy linter
+
+func anyGlobalDBWrappersStillOpen() bool {
+	if globalRoaringReg.Size() != 0 {
+		return true
+	}
+	if globalRbfDBReg.Size() != 0 {
+		return true
+	}
+	if globalLMDBReg.Size() != 0 {
+		return true
+	}
+	if globalBadgerReg.Size() != 0 {
+		return true
+	}
+	return false
+}
+
+func (f *TxFactory) blueGreenOnIfRunningBlueGreen() {
+	if len(f.types) == 2 {
+		f.blueGreenOff = false
+	}
+}
+
+func (f *TxFactory) blueGreenOffIfRunningBlueGreen() {
+	if len(f.types) == 2 {
+		f.blueGreenOff = true
+	}
+}
+
+func (f *TxFactory) hasRoaring() bool {
+	return f.types[0] == roaringTxn || f.types[1] == roaringTxn
+}
+
+var _ = (&TxFactory{}).hasRoaring // happy linter
+
+func (f *TxFactory) blueHasData() (hasData bool, err error) {
+	if len(f.types) != 2 {
+		return false, nil
+	}
+	return f.dbPerShard.HasData(0)
+}
+
+// green2blue is called at the very end of Holder.Open(), so
+// we know that the holder is ready to go, knowing its holder.Indexes(), fields,
+// view, shards, and other metadata if any.
+//
+// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in
+// txfactory_internal_test.go as well.
+//
+func (f *TxFactory) green2blue(holder *Holder) (err error) {
+
+	// Holder.Open will always call us, even without blue_green. Which is fine.
+	// We are just a no-op in that case.
+	if len(f.types) != 2 {
+		return nil
+	}
+
+	blueDest := f.types[0]
+	greenSrc := f.types[1]
+	idxs := holder.Indexes()
+
+	verifyInsteadOfCopy := false
+
+	hasData, err := f.blueHasData()
+	if err != nil {
+		return errors.Wrap(err, "TxFactory.green2blue DataSize(0)")
+	}
+
+	if hasData {
+		verifyInsteadOfCopy = true
+	}
+
+	for _, idx := range idxs {
+
+		blueShards, err := TypedDBPerShardGetLocalShardsForIndex(blueDest, idx, "")
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name))
+		}
+		greenShards, err := TypedDBPerShardGetLocalShardsForIndex(greenSrc, idx, "")
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name))
+		}
+
+		diff := f.shardSliceDiff(blueShards, greenShards)
+		if diff != "" {
+			return fmt.Errorf("blue[%v] and green[%v] have different shards for index '%v': %v", blueDest, greenSrc, idx.name, diff)
+		}
+
+		shards := idx.AvailableShards(localOnly).Slice()
+
+		diff2 := f.shardSliceDiff(blueShards, shards)
+		if diff2 != "" {
+			return fmt.Errorf("blue[%v] and meta data (from green[%v]?)have different shards for index '%v': %v", blueDest, greenSrc, idx.name, diff2)
+		}
+
+		for _, shard := range shards {
+
+			dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard)))
+			}
+
+			if verifyInsteadOfCopy {
+				// verify all containers
+				err = dbs.verifyBlueEqualsGreen(&holder.numCtBlueGreenVerified)
+				if err != nil {
+					return errors.Wrap(err,
+						fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+
+							"green='%v') for index='%v', shard='%v'",
+							blueDest, greenSrc, idx.name, int(shard)))
+				}
+			} else {
+				// the main copy work
+				err = dbs.populateBlueFromGreen()
+				if err != nil {
+					return errors.Wrap(err,
+						fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+
+							"green='%v') for index='%v', shard='%v'",
+							blueDest, greenSrc, idx.name, int(shard)))
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (f *TxFactory) shardSliceDiff(blueShards, greenShards []uint64) (diff string) {
+	nb := len(blueShards)
+	ng := len(greenShards)
+	if nb != ng {
+		diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng)
+	}
+	b := make(map[uint64]bool)
+	g := make(map[uint64]bool)
+	for _, bs := range blueShards {
+		b[bs] = true
+	}
+	for _, gs := range greenShards {
+		g[gs] = true
+	}
+	bmg := mapDiff(b, g) // get blue - green
+	gmb := mapDiff(g, b) // get green - blue
+
+	if len(bmg) == 0 && len(gmb) == 0 {
+		return ""
+	}
+	diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb)
+	return
+}

txfactory_internal_test.go

@@ -15,6 +15,7 @@
 package pilosa
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -25,10 +26,10 @@ import (
 
 func Test_TxFactory_Qcx_query_context(t *testing.T) {
 	src := os.Getenv("PILOSA_TXSRC")
-	if src == "rbf" || src == "lmdb" {
+	if src == "rbf" || src == "lmdb" || src == "badger" {
 		// ok
 	} else {
-		t.Skip("this test only for lmdb and rbf")
+		t.Skip("this test only for lmdb and rbf and badger")
 	}
 
 	shard := uint64(0)
@@ -50,7 +51,7 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) {
 			default:
 			}
 			// add to the group txn on the txf.
-			qcx := idx.Txf.NewQcx()
+			qcx := idx.holder.txf.NewQcx()
 
 			tx, finisher := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard})
 
@@ -95,8 +96,256 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) {
 	// we will crash as the goroutines making Tx will try to use the env
 	// after it is closed. It can take quite a while.
 	// one writer might be blocking the other... so ask for only N-1 at first.
-	//barrier.BlockUntil(N - 1)
-	barrier.BlockUntil(N)
-	//barrier.UnblockReaders()
-	//time.Sleep(1 * time.Second)
+	barrier.BlockUntil(N - 1)
+	//barrier.BlockUntil(N)
+	barrier.UnblockReaders()
+	time.Sleep(1 * time.Second)
+}
+
+// test TxFactory.green2blue
+//
+// blue_green starting with an empty or full blue database
+//  should copy all of green (if blue is empty); or if blue is ull,
+//  verify that blue has all the same bits as green.
+//
+// Benefits: a) we start with known identical state so our testing/comparisons can be valid;
+//       and b) we have an easy migration mechanism, to go from one storage format to another.
+//
+func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+
+	checked := []string{"lmdb", "roaring", "badger", "rbf"}
+
+	for _, blue := range checked {
+		for _, green := range checked {
+			if blue == green {
+				continue
+			}
+			blue_green := blue + "_" + green
+
+			// =============================
+			// Begin setup.
+			//
+			// Setup happens with green only.
+
+			os.Setenv("PILOSA_TXSRC", green)
+			h, path, err := makeHolder(t)
+			if err != nil {
+				t.Fatalf("creating holder: %v", err)
+			}
+			defer os.RemoveAll(path)
+
+			// we will manually h.Close() below
+
+			// Write bits to separate indexes.
+			testSetBit(t, h, "i0", "f", 100, 200)
+			testSetBit(t, h, "i1", "f", 100, 200)
+			testSetBit(t, h, "i1", "f", 100, 12345678)
+
+			testOp := testHolderOperator{}
+			ctx := context.Background()
+			err = h.Process(ctx, &testOp)
+			if err != nil {
+				t.Fatalf("processing holder: %v", err)
+			}
+			expected := testHolderOperator{
+				indexSeen: 2, indexProcessed: 2,
+				fieldSeen: 2, fieldProcessed: 2,
+				viewSeen: 2, viewProcessed: 2,
+				fragmentSeen: 3, fragmentProcessed: 3,
+			}
+			if testOp != expected {
+				t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
+			}
+
+			// verify data is there
+			rowID := uint64(100)
+			colID := uint64(200)
+			_, _ = rowID, colID
+			testMustHaveBit(t, h, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h, "i1", "f", 100, 200)
+			testMustHaveBit(t, h, "i1", "f", 100, 12345678)
+
+			h.Close()
+
+			// can we re.Open the same holder h? hopefully without a problem.
+			panicOn(h.Open())
+
+			testMustHaveBit(t, h, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h, "i1", "f", 100, 200)
+			testMustHaveBit(t, h, "i1", "f", 100, 12345678)
+			h.Close()
+
+			// successful re-open and then Close again of h.
+
+			// check that we can open a NewHolder on green, on same path, and still see our bits.
+			// Because the NewHolder is the code that creates and configures TxFactory as blue_green.
+			h2 := NewHolder(path, nil)
+			panicOn(h2.Open())
+
+			testMustHaveBit(t, h2, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h2, "i1", "f", 100, 200)
+			testMustHaveBit(t, h2, "i1", "f", 100, 12345678)
+			h2.Close()
+
+			// verify that blue does not have it.
+			// open a new holder on path, just looking at blue.
+			os.Setenv("PILOSA_TXSRC", blue)
+
+			h3 := NewHolder(path, nil)
+			panicOn(h3.Open())
+
+			testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
+			testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
+			testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678)
+
+			h3.Close()
+
+			// =============================
+			// Setup done. On to actual test.
+
+			// Opening in blue_green mode means that once Holder.Open()
+			// returns without error, the blue and green databases are
+			// identical.
+			// Since blue is empty, the blue database will get synched up
+			// with the green during Holder.Open().
+
+			os.Setenv("PILOSA_TXSRC", blue_green)
+
+			// open a holder with path again, now looking at both blue and green.
+			// The Holder.Open should do the migration from green, populating blue.
+			h4 := NewHolder(path, nil)
+			panicOn(h4.Open())
+			defer h4.Close()
+
+			testMustHaveBit(t, h4, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h4, "i1", "f", 100, 200)
+			testMustHaveBit(t, h4, "i1", "f", 100, 12345678)
+		}
+	}
+}
+
+// test the situation where we startup blue_green with existing data and
+// go to verify it but blue has more data than green.
+// That will also cause query divergence.
+func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+
+	checked := []string{"lmdb", "roaring", "badger", "rbf"}
+
+	for _, blue := range checked {
+		for _, green := range checked {
+			if blue == green {
+				continue
+			}
+			blue_green := blue + "_" + green
+
+			// =============================
+			// Begin setup.
+			//
+			// Setup happens with green only.
+
+			os.Setenv("PILOSA_TXSRC", green)
+			h, path, err := makeHolder(t)
+			if err != nil {
+				t.Fatalf("creating holder: %v", err)
+			}
+			defer os.RemoveAll(path)
+
+			// we will manually h.Close() below
+
+			// Write bits to separate indexes.
+			testSetBit(t, h, "i0", "f", 100, 200)
+			testSetBit(t, h, "i1", "f", 100, 200)
+			testSetBit(t, h, "i1", "f", 100, 12345678)
+
+			testOp := testHolderOperator{}
+			ctx := context.Background()
+			err = h.Process(ctx, &testOp)
+			if err != nil {
+				t.Fatalf("processing holder: %v", err)
+			}
+			expected := testHolderOperator{
+				indexSeen: 2, indexProcessed: 2,
+				fieldSeen: 2, fieldProcessed: 2,
+				viewSeen: 2, viewProcessed: 2,
+				fragmentSeen: 3, fragmentProcessed: 3,
+			}
+			if testOp != expected {
+				t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
+			}
+
+			// verify data is there
+			rowID := uint64(100)
+			colID := uint64(200)
+			_, _ = rowID, colID
+			testMustHaveBit(t, h, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h, "i1", "f", 100, 200)
+			testMustHaveBit(t, h, "i1", "f", 100, 12345678)
+
+			h.Close()
+
+			// verify that blue does not have it.
+			// open a new holder on path, just looking at blue.
+			os.Setenv("PILOSA_TXSRC", blue)
+
+			h3 := NewHolder(path, nil)
+			panicOn(h3.Open())
+
+			testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
+			testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
+			testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678)
+
+			h3.Close()
+
+			// =============================
+			// Setup done. On to actual test.
+
+			// Opening in blue_green mode means that once Holder.Open()
+			// returns without error, the blue and green databases are
+			// identical.
+			// Since blue is empty, the blue database will get synched up
+			// with the green during Holder.Open().
+
+			os.Setenv("PILOSA_TXSRC", blue_green)
+
+			// open a holder with path again, now looking at both blue and green.
+			// The Holder.Open should do the migration from green, populating blue.
+			h4 := NewHolder(path, nil)
+			panicOn(h4.Open())
+
+			testMustHaveBit(t, h4, "i0", "f", rowID, colID)
+			testMustHaveBit(t, h4, "i1", "f", 100, 200)
+			testMustHaveBit(t, h4, "i1", "f", 100, 12345678)
+			h4.Close()
+
+			// now open just blue, and add a bit to a new index, i2.
+			os.Setenv("PILOSA_TXSRC", blue)
+
+			h5 := NewHolder(path, nil)
+			panicOn(h5.Open())
+			testSetBit(t, h5, "i2", "f", 500, 777)
+			h5.Close()
+
+			// now open blue_green. should get a verification failure
+			// due to the extra bit in blue.
+			os.Setenv("PILOSA_TXSRC", blue_green)
+
+			// BEGIN verficiation that should ERROR out b/c blue has more data.
+
+			// open a holder with path again, now looking at both blue and green.
+			// The Holder.Open should verify blue against green and notice the extra bit.
+			h6 := NewHolder(path, nil)
+			err = h6.Open()
+			if err == nil {
+				h6.Close()
+				t.Fatalf("should have had blue-green verification fail on Holder.Open")
+			}
+			h6.Close()
+		}
+	}
 }

util.go

@@ -20,8 +20,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
+	"syscall"
 	"unsafe"
 
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -237,3 +241,119 @@ func fromInterval16(a []roaring.Interval16) []byte {
 	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
 }
+
+// DiskUse reports the total bytes uses by all files under root
+// that match requiredSuffix. requiredSuffix can be empty string.
+// Space used by directories is not counted.
+func DiskUse(root string, requiredSuffix string) (tot int, err error) {
+	if !DirExists(root) {
+		return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
+	}
+
+	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if info == nil {
+			panic(fmt.Sprintf("info was nil for path = '%v'", path))
+		}
+		if info.IsDir() {
+			// skip the size of directories themselves, only summing files.
+		} else {
+			sz := info.Size()
+			if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
+				tot += int(sz)
+			}
+		}
+		return nil
+	})
+	return
+}
+
+// rootDir must exist. Return the size in bytes of the largest sub-directory
+// that has the required suffix. The largestSize is from DiskUse() called
+// on the sub-dir. DiskUse only counts file size, nothing for directory inodes.
+func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) {
+	if !DirExists(rootDir) {
+		return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir)
+	}
+
+	err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+		if info == nil {
+			panic(fmt.Sprintf("info was nil for path = '%v'", path))
+		}
+
+		if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) {
+			exists = true
+			size, err := DiskUse(path, "")
+			if err != nil {
+				// disk error? report it
+				return err
+			}
+			if size > largestSize {
+				largestSize = size
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return exists, -1, err
+	}
+	return
+}
+
+// called by Holder.hasRoaringData()
+func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) {
+
+	var info roaring.BitmapInfo
+	_ = info
+	var f *os.File
+	f, err = os.Open(path)
+	if err != nil {
+		return
+	}
+
+	var fi os.FileInfo
+	fi, err = f.Stat()
+	if err != nil {
+		return
+	}
+
+	// Memory map the file.
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		err = errors.Wrap(err, "mmapping")
+		return
+	}
+	defer func() {
+		err = syscall.Munmap(data)
+		if err != nil {
+			err = errors.Wrap(err, "roaringFragmentHasData: munmap failed")
+		}
+		err = f.Close()
+		if err != nil {
+			err = errors.Wrap(err, "roaringFragmentHasData f.Close() in defer")
+		}
+	}()
+
+	// Attach the mmap file to the bitmap.
+	var rbm *roaring.Bitmap
+	rbm, _, err = roaring.InspectBinary(data, true, &info)
+	if err != nil {
+		err = errors.Wrap(err, "inspecting")
+		return
+	}
+
+	if info.ContainerCount > 0 {
+		return true, nil
+	}
+	if info.Ops > 0 {
+		return true, nil
+	}
+
+	citer, found := rbm.Containers.Iterator(0)
+	_ = found
+
+	for citer.Next() {
+		return true, nil
+	}
+
+	return
+}

utils_internal_test.go

@@ -167,9 +167,9 @@ func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *tim
 		}
 
 		if err := func() error {
-			idx := c.holder.indexes[f.index]
+			idx := c.holder.Index(f.index)
 			shard := colID / ShardWidth
-			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+			tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
 			if tx != nil {
 				defer tx.Rollback()
 			}
@@ -488,11 +488,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 			// there will be two -badgerdb directories/databases, we need to copy
 			// from src to dest the fragment. This simulates sending the fragment over the network.
 			srcIdx := srcCluster.holder.Index(src.Index)
-			srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard})
+			srctx := srcIdx.holder.txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard})
 
 			destIdx := destCluster.holder.Index(src.Index)
 
-			desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment, Shard: destFragment.shard})
+			desttx := destIdx.holder.txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment, Shard: destFragment.shard})
 
 			citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
 			panicOn(err)

view.go

@@ -173,7 +173,6 @@ var workQueue = make(chan struct{}, runtime.NumCPU()*2)
 // replaces v.openFragments() with Tx generic code.
 func (v *view) openFragmentsInTx() error {
 
-	// we think this is correct for dbpershard, but might be slower. TODO
 	shards, err := DBPerShardGetShardsForIndex(v.idx, v.path)
 	if err != nil {
 		return errors.Wrap(err, "DBPerShardGetShardsForIndex()")
@@ -368,15 +367,6 @@ func (v *view) notifyIfNewShard(shard uint64) {
 
 func (v *view) newFragment(path string, shard uint64) *fragment {
 
-	if v.holder != nil && v.idx != nil {
-		// A view must have its v.idx *Index registered with its holder.
-		// Otherwise TestField_AvailableShards crashes, as one example.
-		hIdx := v.holder.Index(v.idx.name)
-		if hIdx == nil && v.idx != nil {
-			v.holder.addIndexFromField(v.idx)
-		}
-	}
-
 	frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags())
 	frag.CacheType = v.cacheType
 	frag.CacheSize = v.cacheSize
@@ -402,7 +392,7 @@ func (v *view) deleteFragment(shard uint64) error {
 
 	idx := f.holder.Index(v.index)
 	f.Close()
-	if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
+	if err := idx.holder.txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
 		return errors.Wrap(err, "DeleteFragment")
 	}
 	delete(v.fragments, shard)
@@ -418,7 +408,7 @@ func (v *view) row(txOrig Tx, rowID uint64) (*Row, error) {
 
 		tx := txOrig
 		if NilInside(tx) {
-			tx = v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard})
+			tx = v.idx.holder.txf.NewTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard})
 			defer tx.Rollback()
 		}
 
@@ -445,7 +435,7 @@ func (v *view) setBit(txOrig Tx, rowID, columnID uint64) (changed bool, err erro
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
+		tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
 		defer func() {
 			if err == nil {
 				panicOn(tx.Commit())
@@ -467,7 +457,7 @@ func (v *view) clearBit(txOrig Tx, rowID, columnID uint64) (changed bool, err er
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
+		tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
 		defer func() {
 			if err == nil {
 				panicOn(tx.Commit())
@@ -490,7 +480,7 @@ func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint) (value int64, ex
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = frag.idx.Txf.NewTx(Txo{Write: !writable, Index: frag.idx, Fragment: frag, Shard: frag.shard})
+		tx = frag.idx.holder.txf.NewTx(Txo{Write: !writable, Index: frag.idx, Fragment: frag, Shard: frag.shard})
 		defer tx.Rollback()
 	}
 
@@ -507,7 +497,7 @@ func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint, value int64)
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
+		tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
 		defer func() {
 			if err == nil {
 				panicOn(tx.Commit())
@@ -530,7 +520,7 @@ func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint, value int64
 
 	tx := txOrig
 	if NilInside(tx) {
-		tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
+		tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
 		defer func() {
 			if err == nil {
 				panicOn(tx.Commit())

view_internal_test.go

@@ -36,6 +36,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
 
 	h := NewHolder(path, nil)
 	// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
+
 	idx, err := h.createIndex(index, IndexOptions{})
 	testhook.Cleanup(tb, func() {
 		h.Close()