Merge branch 'master' into appendSemantics

.circleci/config.yml

@@ -198,12 +198,24 @@ workflows:
           resource_class: xlarge
           requires:
             - setup
+      - test:
+          name: test-txstore-rbf
+          test_make_target: test-txstore-rbf
+          resource_class: xlarge
+          requires:
+            - setup
+      - test:
+          name: test-txstore-rbf_lmdb
+          test_make_target: test-txstore-rbf_lmdb
+          resource_class: xlarge
+          requires:
+            - setup            
       - test:
           name: test-shardwidth-22
           shard_width: "22"
           resource_class: large
           requires:
-            - setup
+            - setup            
       - cluster-tests:
           requires:
             - setup

.gitignore

@@ -6,3 +6,6 @@ vendor
 build
 *~
 lattice
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf_lmdb test-txstore-rbf
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -55,7 +55,7 @@ testv-race: topt-race testvsub-race
 #            find which test is hung/deadlocked.
 #
 testvsub:
-	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
            echo; echo "___ testing subpkg $$i"; \
            cd $$i; pwd; \
            go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
@@ -64,7 +64,7 @@ testvsub:
         done
 
 testvsub-race:
-	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
            echo; echo "___ testing subpkg $$i -race"; \
            cd $$i; pwd; \
            go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
@@ -166,9 +166,14 @@ generate-stringer:
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
-# dunno if protoc-gen-gofast is actually needed here
-generate-proto-grpc: require-protoc require-protoc-gen-gofast
+generate-proto-grpc: require-protoc require-protoc-gen-go
 	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
+	# TODO: Modify above commands and remove the below mv if possible.
+	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
+	# I couldn't get it to work during development - Cody
+	cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/
+	rm -rf proto/github.com
 
 # `go generate` all needed packages
 generate: generate-protoc generate-statik generate-stringer generate-pql
@@ -196,6 +201,9 @@ pilosa-keydump:
 pilosa-chk:
 	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
 
+pilosa-fsck:
+	cd ./cmd/pilosa-fsck && make install && make release
+
 # Run Pilosa tests inside Docker container
 docker-test:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
@@ -386,6 +394,9 @@ install-stringer:
 install-protoc-gen-gofast:
 	GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
 
+install-protoc-gen-go:
+	GO111MODULE=off go get -u github.com/golang/protobuf/protoc-gen-go
+
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 
@@ -399,3 +410,10 @@ install-gometalinter:
 	GO111MODULE=off go get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
 	GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
+
+test-txstore-rbf:
+	PILOSA_TXSRC=rbf $(MAKE) testv-race
+
+test-txstore-rbf_lmdb:
+	PILOSA_TXSRC=rbf_lmdb $(MAKE) testv-race
+

api.go

@@ -25,6 +25,8 @@ import (
 	"io/ioutil"
 	"math"
 	"net/url"
+	"os"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -376,37 +378,42 @@ func importWorker(importWork chan importJob) {
 					}
 				}
 
-				tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard})
-				defer finisher(&err0)
+				if err := func() (err1 error) {
+					tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard})
+					defer finisher(&err1)
 
-				var doClear bool
-				switch doAction {
-				case RequestActionOverwrite:
-					err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block)
-					if err != nil {
-						return errors.Wrap(err, "importing roaring as overwrite")
-					}
-				case RequestActionClear:
-					doClear = true
-					fallthrough
-				case RequestActionSet:
-					fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
-					if fileMagic == roaring.MagicNumber { // if pilosa roaring format
-						err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear)
+					var doClear bool
+					switch doAction {
+					case RequestActionOverwrite:
+						err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block)
 						if err != nil {
-							return errors.Wrap(err, "importing pilosa roaring")
+							return errors.Wrap(err, "importing roaring as overwrite")
 						}
-					} else {
-						// must make a copy of data to operate on locally on standard roaring format.
-						// field.importRoaring changes the standard roaring run format to pilosa roaring
-						data := make([]byte, len(viewData))
-						copy(data, viewData)
-						err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
+					case RequestActionClear:
+						doClear = true
+						fallthrough
+					case RequestActionSet:
+						fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
+						if fileMagic == roaring.MagicNumber { // if pilosa roaring format
+							err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear)
+							if err != nil {
+								return errors.Wrap(err, "importing pilosa roaring")
+							}
+						} else {
+							// must make a copy of data to operate on locally on standard roaring format.
+							// field.importRoaring changes the standard roaring run format to pilosa roaring
+							data := make([]byte, len(viewData))
+							copy(data, viewData)
+							err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
 
-						if err != nil {
-							return errors.Wrap(err, "importing standard roaring")
+							if err != nil {
+								return errors.Wrap(err, "importing standard roaring")
+							}
 						}
 					}
+					return nil
+				}(); err != nil {
+					return err
 				}
 			}
 			return nil
@@ -774,12 +781,85 @@ func (api *API) Hosts(ctx context.Context) []*Node {
 	return api.cluster.Nodes()
 }
 
+func (api *API) HostStates(ctx context.Context) map[string]string {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.HostStates")
+	defer span.Finish()
+	return api.cluster.AllNodeStates()
+}
+
 // Node gets the ID, URI and coordinator status for this particular node.
 func (api *API) Node() *Node {
 	node := api.server.node()
 	return &node
 }
 
+// Usage gets the disk usage per index
+func (api *API) Usage() (map[string]int64, int64, error) {
+	indexSizes := make(map[string]int64)
+	var totalSize int64
+
+	dirName, err := expandDirName(api.server.dataDir)
+	if err != nil {
+		return indexSizes, totalSize, errors.Wrap(err, "expanding data directory")
+	}
+	dir, err := os.Open(dirName)
+	if err != nil {
+		return indexSizes, totalSize, errors.Wrap(err, "opening data directory")
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return indexSizes, totalSize, errors.Wrap(err, "reading data directory")
+	}
+
+	for _, file := range files {
+		if !file.IsDir() {
+			continue
+		}
+		if api.holder.Txf().IsTxDatabasePath(file.Name()) {
+			continue
+		}
+		fullName := path.Join(dirName, file.Name())
+		indexSizes[file.Name()], err = diskUsage(fullName)
+		if err != nil {
+			break
+		}
+		totalSize += indexSizes[file.Name()]
+	}
+
+	return indexSizes, totalSize, nil
+}
+
+func diskUsage(fname string) (int64, error) {
+	var size int64
+
+	dir, err := os.Open(fname)
+	if err != nil {
+		return 0, errors.Wrap(err, "opening data subdirectory")
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return 0, errors.Wrap(err, "reading data subdirectory")
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			sz, err := diskUsage(path.Join(fname, file.Name()))
+			if err != nil {
+				return 0, err
+			}
+			size += sz
+		} else {
+			size += file.Size()
+		}
+	}
+
+	return size, nil
+}
+
 // RecalculateCaches forces all TopN caches to be updated.
 // This is done internally within a TopN query, but a user may want to do it ahead of time?
 func (api *API) RecalculateCaches(ctx context.Context) error {
@@ -1667,6 +1747,14 @@ func (api *API) State() string {
 	return api.cluster.State()
 }
 
+// ClusterName returns the cluster name.
+func (api *API) ClusterName() string {
+	if api.cluster.Name == "" {
+		return api.cluster.id
+	}
+	return api.cluster.Name
+}
+
 // Version returns the Pilosa version.
 func (api *API) Version() string {
 	return strings.TrimPrefix(Version, "v")
@@ -1692,6 +1780,7 @@ func (api *API) Info() serverInfo {
 		CPUType:          si.CPUModel(),
 		Memory:           mem,
 		TxSrc:            api.holder.txf.TxType(),
+		ReplicaN:         api.cluster.ReplicaN,
 	}
 }
 
@@ -1938,6 +2027,7 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin
 
 type serverInfo struct {
 	ShardWidth       uint64 `json:"shardWidth"`
+	ReplicaN         int    `json:"replicaN"`
 	Memory           uint64 `json:"memory"`
 	CPUType          string `json:"cpuType"`
 	CPUPhysicalCores int    `json:"cpuPhysicalCores"`

boltdb/translate.go

@@ -17,9 +17,12 @@ package boltdb
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 	"time"
 
@@ -27,8 +30,13 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
 	"github.com/zeebo/blake3"
+
+	"runtime/pprof"
 )
 
+var _ = ioutil.TempFile
+var _ = pprof.StartCPUProfile
+
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
@@ -90,6 +98,10 @@ type TranslateStore struct {
 	Path string
 }
 
+func (s *TranslateStore) GetStorePath() string {
+	return s.Path
+}
+
 // NewTranslateStore returns a new instance of TranslateStore.
 func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
@@ -104,6 +116,15 @@ func NewTranslateStore(index, field string, partitionID, partitionN int) *Transl
 
 // Open opens the translate file.
 func (s *TranslateStore) Open() (err error) {
+
+	// add the path to the problem database if we panic handling it.
+	defer func() {
+		r := recover()
+		if r != nil {
+			panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r))
+		}
+	}()
+
 	if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
 	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
@@ -485,7 +506,7 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string {
 	return string(boltKey)
 }
 
-func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) {
+func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) {
 	sum = &pilosa.TranslatorSummary{}
 	hasher := blake3.New()
 
@@ -524,3 +545,787 @@ func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSumma
 	sum.Checksum = string(buf[:])
 	return sum, nil
 }
+
+func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) {
+	sum = &pilosa.TranslatorSummary{}
+	hasher := blake3.New()
+
+	if partitionID != s.partitionID {
+		panic(fmt.Sprintf("inconsistent partitionID arg %v with TranslateStore.paritionID %v", partitionID, s.partitionID))
+	}
+	firstPrimary := topo.PrimaryNodeIndex(partitionID)
+
+	err = s.db.View(func(tx *bolt.Tx) error {
+
+		bkt := tx.Bucket(bucketKeys) // key -> id
+		if bkt == nil {
+			panic("bucketKeys not found")
+		}
+
+		cur := bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			input := append(k, v...)
+			//vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), k='%v', v=%x", partitionID, s.Path, string(k), v)
+			_, _ = hasher.Write(input)
+			sum.KeyCount++
+		}
+
+		bkt = tx.Bucket(bucketIDs) // id -> key
+		if bkt == nil {
+			panic("bucketIDs not found")
+		}
+
+		cur = bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+
+			// should the primary be the same for each key in this partition?
+			id := btou64(k)
+			shard := id / pilosa.ShardWidth
+
+			ks := string(v)
+			primary := topo.GetPrimaryForColKeyTranslation(s.index, ks)
+			if firstPrimary < 0 {
+				firstPrimary = primary
+			} else {
+				if primary != firstPrimary {
+					panic(fmt.Sprintf("s.index='%v' primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v; topo='%v'", s.index, primary, firstPrimary, ks, id, shard, partitionID, topo.String()))
+				}
+			}
+
+			// Verify the invariant that the primaries agree. Just a sanity check.
+			primaryForShard := topo.GetPrimaryForShardReplication(s.index, shard)
+			if primaryForShard != firstPrimary {
+				panic(fmt.Sprintf("primaryForShard (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primaryForShard, firstPrimary, ks, id, shard, partitionID))
+			}
+
+			input := append(k, v...)
+			//vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), idBucket id=%x  key='%v'", partitionID, s.Path, id, ks)
+			_, _ = hasher.Write(input)
+			sum.IDCount++
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	sum.PrimaryNodeIndex = firstPrimary
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+	sum.Checksum = string(buf[:])
+	return sum, nil
+}
+
+func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error {
+	return s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			panic("bucketKeys not found")
+		}
+		cur := bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			walk(string(k), btou64(v))
+		}
+		return nil
+	})
+}
+func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error {
+	return s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketIDs)
+		if bkt == nil {
+			panic("bucketIDs not found")
+		}
+		cur := bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			walk(string(v), btou64(k))
+		}
+		return nil
+	})
+}
+
+// call s.notifyWrite() when done
+func (s *TranslateStore) SetFwdRevMaps(tx *bolt.Tx, fwd map[string]uint64, rev map[uint64]string) (err error) {
+
+	localTx := false
+	if tx == nil {
+		localTx = true
+		tx, err = s.db.Begin(true)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = tx.Rollback()
+		}()
+	}
+
+	// reinitialize buckets
+	err = tx.DeleteBucket(bucketKeys)
+	if err != nil {
+		return err
+	}
+	err = tx.DeleteBucket(bucketIDs)
+	if err != nil {
+		return err
+	}
+	if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
+		return err
+	} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
+		return err
+	}
+
+	key2id := tx.Bucket(bucketKeys)
+	for k, v := range fwd {
+		err := key2id.Put([]byte(k), u64tob(v))
+		if err != nil {
+			return err
+		}
+	}
+	id2key := tx.Bucket(bucketIDs)
+	for k, v := range rev {
+		err := id2key.Put(u64tob(k), []byte(v))
+		if err != nil {
+			return err
+		}
+	}
+	if localTx {
+		return tx.Commit()
+	}
+	return nil
+}
+
+func (s *TranslateStore) GetFwdRevMaps(tx *bolt.Tx) (fwd map[string]uint64, rev map[uint64]string, err error) {
+	fwd = make(map[string]uint64)
+	rev = make(map[uint64]string)
+
+	key2id := tx.Bucket(bucketKeys)
+
+	err = key2id.ForEach(func(k, v []byte) error {
+		fwd[string(k)] = btou64(v)
+		return nil
+	})
+	if err != nil {
+		return
+	}
+
+	id2key := tx.Bucket(bucketIDs)
+	err = id2key.ForEach(func(k, v []byte) error {
+		rev[btou64(k)] = string(v)
+		return nil
+	})
+	return
+}
+
+//var vv = pilosa.VV
+
+// helpers for repair
+
+// muint64 holds multiple unit64
+type muint64 struct {
+	slc []uint64
+}
+
+func (m *muint64) String() (s string) {
+	for _, e := range m.slc {
+		s += fmt.Sprintf("%x, ", e)
+	}
+	return
+}
+
+// mstring holds multiple strings
+type mstring struct {
+	slc []string
+}
+
+func (m *mstring) String() (s string) {
+	for _, e := range m.slc {
+		s += e + ","
+	}
+	return
+}
+
+func addToProblemKeys(problemKeys map[string]*muint64, k string, v uint64, noValue bool) {
+	mu, already := problemKeys[k]
+	if !already {
+		mu = &muint64{}
+		problemKeys[k] = mu
+	}
+	if !noValue {
+		mu.slc = append(mu.slc, v)
+	}
+}
+func addToProblemIDs(problemIDs map[uint64]*mstring, k uint64, v string, noValue bool) {
+	mu, already := problemIDs[k]
+	if !already {
+		mu = &mstring{}
+		problemIDs[k] = mu
+	}
+	if !noValue {
+		mu.slc = append(mu.slc, v)
+	}
+}
+
+// only actually apply the fixes if applyKeyRepairs is true.
+// if anything changed, return changed == true.
+func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) {
+	// strategy: get the full set of keys; the domain keys from
+	// the forward key->id mapping, and the range keys from the reverse id->key mapping.
+	// Then march through them and make sure they are mapped correctly.
+	// At the moment we do try to reuse dangling IDs instead of making
+	// new ones. This might not always be possible, but we hope for
+	// now that it suffices b/c it minimizes the amount of fragment
+	// re-write we may have to do.
+
+	/*
+		// ============ profiling ===============
+		fd, err := ioutil.TempFile(".", "cpu.prof")
+		if err != nil {
+			panic(err)
+		}
+		_ = pprof.StartCPUProfile(fd)
+		defer func() {
+			pprof.StopCPUProfile()
+			fd.Close()
+		}()
+		// ============ end profiling ===============
+	*/
+
+	tx, err := s.db.Begin(true)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_ = tx.Rollback()
+	}()
+	fwd, rev, err := s.GetFwdRevMaps(tx)
+	if err != nil {
+		return false, err
+	}
+
+	// place to store the correct stuff.
+	//
+	// fwd2, rev2: new, repaired versions.
+	// INVAR: they only contain (correct) invertible mappings.
+	fwd2 := make(map[string]uint64)
+	rev2 := make(map[uint64]string)
+
+	// and a place to store the problems.
+	problemKeys := make(map[string]*muint64)
+	problemIDs := make(map[uint64]*mstring)
+
+fwdscan:
+	for k, v := range fwd {
+		_, already := fwd2[k]
+		if already {
+			// k has already been repaired. don't worry about further.
+			continue fwdscan
+		} else {
+			// if its already invertible, then just keep it, no need to repair it
+
+			// INVAR: k is not in fwd2 (at least not yet).
+			rkey, ok := rev[v]
+			if !ok {
+				// k -> v -> X
+				addToProblemIDs(problemIDs, v, k, false)
+				addToProblemKeys(problemKeys, k, v, false)
+				continue fwdscan
+			}
+			if rkey == k {
+				// yay. a good, invertible, mapping. no repair needed.
+				if k == "" {
+					panic("bad empty key")
+				}
+				fwd2[k] = v
+				rev2[v] = k
+				continue fwdscan
+			}
+
+			// some kind of problem.
+			// what kind?
+			// Define problemKey as: 2nd key mapping to id already in fwd2.
+			// Define problemID  as: 2nd ID  mapping to key already in fwd2.
+
+			// k -> v -> rkey, and rkey != k.
+			v2, ok := fwd[rkey]
+			if ok {
+				// k -> v -> rkey -> v2, where v2 ?= v
+				if v2 == v {
+					//  k -> v -> rkey -> v
+					// just have a problemKey in k.
+
+					if rkey == "" {
+						panic("bad empty rkey")
+					}
+					fwd2[rkey] = v
+					rev2[v] = rkey
+					addToProblemKeys(problemKeys, k, v, true)
+					continue fwdscan
+				}
+				// this is i = 1 test case. :)
+
+				//  k -> v -> rkey -> v2, where v != v2, and k != rkey.
+				addToProblemKeys(problemKeys, k, v, false)
+				addToProblemIDs(problemIDs, v, rkey, false)
+				continue fwdscan
+			} else {
+				// k -> v -> rkey -> X(nil), and rkey != k.
+				addToProblemKeys(problemKeys, k, v, false)
+				addToProblemKeys(problemKeys, rkey, 0, true)
+				addToProblemIDs(problemIDs, v, rkey, false)
+			}
+		}
+	}
+revscan:
+	for id, key := range rev {
+		k1, already := rev2[id]
+		_ = k1
+		if already {
+			// fine, already there.
+			continue
+		}
+
+		// if its already invertible, keep it.
+		rid, ok := fwd[key]
+		if !ok {
+			// id -> key -> X
+			addToProblemKeys(problemKeys, key, 0, true)
+			addToProblemIDs(problemIDs, id, key, false)
+			continue revscan
+		}
+		if rid == id {
+			// id -> key -> id. good. but should have been added to fwd2/rev2 above.
+			panic("should have been added to fwd2/rev2 above!")
+
+		} else {
+			// id -> key -> rid, where id != rid
+			// so rid -> ?
+			keyr, ok := rev[rid]
+			if !ok {
+				// id -> key -> rid -> X, where id != rid
+				addToProblemKeys(problemKeys, key, rid, false)
+				addToProblemKeys(problemKeys, key, id, false)
+				addToProblemIDs(problemIDs, rid, key, false)
+				continue revscan
+			}
+			if keyr == key {
+				// id -> key -> rid -> key. So rid is correct and id is dangling.
+				//
+				// Heuristic: ASSUME here, that the 2 consistent links key->rid->key are correct,
+				// and that the single id -> key is in the wrong. This DOESN'T HAVE
+				// TO BE THE CASE.
+
+				if key == "" {
+					panic("bad empty key")
+				}
+				rev2[rid] = key
+				fwd2[key] = rid
+				addToProblemIDs(problemIDs, id, "", true)
+			} else {
+				// this is test case i = 0. Must handle it.
+
+				// id -> key -> rid -> keyr, id != rid, keyr != key.
+				id2, ok := fwd[keyr]
+				if ok && id2 == rid {
+					// id -> key -> rid -> keyr -> rid, id != rid, keyr != key.
+					// so rid -> keyr -> rid is good.
+
+					if keyr == "" {
+						panic("bad empty keyr")
+					}
+					fwd2[keyr] = rid
+					rev2[rid] = keyr
+					// and id -> key -> rid is bad, b/c id != rid.
+					addToProblemKeys(problemKeys, key, rid, false)
+					addToProblemIDs(problemIDs, id, key, false)
+					continue revscan
+				}
+				// one of these 3 cases holds. all have the same treatment.
+				// 1) id2 == id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, keyr != key.
+				// 2) id2 != id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, id2 != id, keyr != key.
+				// 3) !ok: id -> key -> rid -> keyr -> X, id != rid, keyr != key.
+				addToProblemIDs(problemIDs, id, key, false)
+				addToProblemKeys(problemKeys, key, rid, false)
+				addToProblemIDs(problemIDs, rid, keyr, false)
+			}
+		}
+
+	}
+	//vv("problemKeys = '%v'", problemKeys)
+	//vv("problemIDs  = '%v'", problemIDs)
+
+	newIDs := make(map[uint64]bool)
+
+	// assign new IDs to any problemKeys; but first
+	// try to reuse already allocated IDs that are just dangling.
+loopProblemKeys:
+	for key, ids := range problemKeys {
+		// first try a minor repair, maybe it was just mssing from rev
+		// and we can avoid allocate another id.
+
+		// sanity check
+		v2, already := fwd2[key]
+		if already {
+			panic(fmt.Sprintf("should not get here since fwd2 is only correct invertibles: key='%v', v2='%x'", key, v2))
+		}
+		// INVAR: we have no correct mapping for key in fwd2.
+
+		// treat the danglers as "suggestions" for the correction.
+		for k, id := range ids.slc {
+			_ = k
+			_, already = rev2[id]
+			if !already {
+				// is this correct?
+				// id is not in rev2, and key is not in fwd2.
+				// therefore, we can add them both and maintain consistency.
+
+				//vv("add %v to fwd2", key)
+				if key == "" {
+					panic("bad empty key")
+				}
+				fwd2[key] = id
+				rev2[id] = key
+				continue loopProblemKeys
+			}
+		}
+		// INVAR: key -> ? don't know. We didn't find a usable suggestion for the id.
+
+		// yes, we get here. We have key. We are looking for a suitable id for it.
+
+		// can we get a usable id from the problemIDs?
+		found := false
+	suggestions:
+		for idp, mkeyp := range problemIDs {
+			for _, candk := range mkeyp.slc {
+				//vv("checking problemIDs, ipd=%x, candk='%v'; candk==key is %v", idp, candk, candk == key)
+				if candk == key {
+					// we have a suggestion from problemIDs that idp might work, doing key -> idp.
+					// Validate that this is possible.
+					k2, already := rev2[idp]
+					_ = k2
+					if already {
+						//vv("idp is already in rev2: idp=%v, k2=%v", idp, k2)
+						continue suggestions
+					}
+					// idp works. put it in the correct set.
+					if key == "" {
+						panic("bad empty key")
+					}
+					rev2[idp] = key
+					fwd2[key] = idp
+					found = true
+					break suggestions
+				}
+			}
+		}
+		if !found {
+			id2 := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+			//vv("could not minor repair, allocating new id2 = %v instead", id2)
+			newIDs[id2] = true
+
+			if key == "" {
+				panic("bad empty key")
+			}
+			fwd2[key] = id2
+			rev2[id2] = key
+		}
+	} // end problemKeys
+
+	//for id, keys := range problemIDs {
+	//}
+
+	if verbose {
+		reportIfGainedOrLostIDs(s, fwd, fwd2, rev, rev2, newIDs)
+		reportIfGainedOrLostKeys(s, fwd, fwd2, rev, rev2)
+	}
+
+	adds, changes, changeIDs, err := makeStringKeyChanges(verbose, applyKeyRepairs, tx, s, topo, fwd, fwd2, rev, rev2, newIDs)
+	if err != nil {
+		return false, err
+	}
+	_, _, _ = adds, changes, changeIDs
+
+	//vv("changedIDs = '%#v'", changeIDs)
+	if len(adds) > 0 || len(changes) > 0 || len(changeIDs) > 0 || len(newIDs) > 0 {
+		changed = true
+	}
+
+	//vv("newIDs = '%#v'", newIDs)
+	//vv("fwd2 = '%#v'", fwd2)
+	//vv("rev2 = '%#v'", rev2)
+
+	err = tx.Commit()
+	if err == nil {
+		s.notifyWrite()
+	}
+	return changed, err
+}
+
+func reportIfGainedOrLostIDs(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string, newIDs map[uint64]bool) {
+	// get all IDs ever mentioned
+	before := make(map[uint64]bool)
+	after := make(map[uint64]bool)
+	for _, id := range fwd {
+		before[id] = true
+	}
+	for _, id := range fwd2 {
+		if !newIDs[id] {
+			after[id] = true
+		}
+	}
+	for id := range rev {
+		before[id] = true
+	}
+	for id := range rev2 {
+		if !newIDs[id] {
+			after[id] = true
+		}
+	}
+	nb := len(before)
+	na := len(after)
+	if nb != na {
+		fmt.Printf("# needs-repair: Num ID before %v != Num ID after %v, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", nb, na, s.Path, len(fwd), len(rev), len(fwd2), len(rev2))
+	}
+	if len(newIDs) > 0 {
+		fmt.Printf("# needs-repair: adding newIDs '%#v', for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", newIDs, s.Path, len(fwd), len(rev), len(fwd2), len(rev2))
+	}
+}
+func reportIfGainedOrLostKeys(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string) {
+	// get all IDs ever mentioned
+	nb := len(fwd)
+	na := len(fwd2)
+	if nb != na {
+		diffAB := mapDiffStrings(fwd, fwd2)
+		diffBA := mapDiffStrings(fwd2, fwd)
+
+		fmt.Printf("# needs-repair: Num Keys before != Num Keys after, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v.  fwd - fwd2 = '%#v';  fwd2-fwd = '%#v'\n", s.Path, len(fwd), len(rev), len(fwd2), len(rev2), diffAB, diffBA)
+	}
+}
+
+// return A - B
+func mapDiffStrings(mapA, mapB map[string]uint64) (r []string) {
+	for a := range mapA {
+		_, ok := mapB[a]
+		if !ok {
+			r = append(r, a)
+		}
+	}
+	sort.Strings(r)
+	return
+}
+
+type BeforeAfterKeyChange struct {
+	BeforeID uint64
+	AfterID  uint64
+}
+
+type BeforeAfterIDChange struct {
+	IsDelete     bool
+	IsAdd        bool
+	BeforeString string
+	AfterString  string
+}
+
+// do the minimal state update.
+// fwd2 is the "after" map, all string keys repaired.
+func makeStringKeyChanges(
+	verbose bool,
+	applyKeyRepairs bool,
+	tx *bolt.Tx,
+	s *TranslateStore,
+	topo *pilosa.Topology,
+	fwd, fwd2 map[string]uint64,
+	rev, rev2 map[uint64]string,
+	newIDs map[uint64]bool,
+) (
+	adds map[string]uint64,
+	changeKeys map[string]*BeforeAfterKeyChange,
+	changeIDs map[uint64]*BeforeAfterIDChange,
+	err error,
+) {
+	//vv("makeStringKeyChanges called")
+
+	//vv("fwd2 = '%#v'", fwd2)
+	//vv("rev2 = '%#v'", rev2)
+	//vv("fwd = '%#v'", fwd)
+	//vv("rev = '%#v'", rev)
+
+	var action string
+	if applyKeyRepairs {
+		action = "applying "
+	}
+
+	// addition of string key
+	adds = make(map[string]uint64)
+
+	// change of the mapping of key -> id.
+	changeKeys = make(map[string]*BeforeAfterKeyChange)
+
+	// changes to bucketIDs
+	changeIDs = make(map[uint64]*BeforeAfterIDChange)
+
+	localTx := false
+	if applyKeyRepairs && tx == nil {
+		localTx = true
+		tx, err = s.db.Begin(true)
+		if err != nil {
+			return
+		}
+		defer func() {
+			//vv("tx.Rollback happening")
+			_ = tx.Rollback()
+		}()
+	}
+
+	key2id := tx.Bucket(bucketKeys)
+	id2key := tx.Bucket(bucketIDs)
+
+	// make a copy of rev2 that we can delete from, to see if
+	// any additions left in rev2 need to be added after all of
+	// rev is analyzed.
+	rev2cp := make(map[uint64]string)
+	for id, k := range rev2 {
+		rev2cp[id] = k
+	}
+
+	// first we clean up any stale IDs from id2key. Then the fwd2 pass
+	// that follows will write to both key2id and id2key.
+	for id, key := range rev {
+		//vv("makeStringKeyChanges on rev2: id=%x -> key='%v'", id, key)
+		key2, ok := rev2[id]
+		if !ok {
+			changeIDs[id] = &BeforeAfterIDChange{IsDelete: true}
+			if verbose {
+				fmt.Printf("# %vkey-translation-delete-id: (id %x -> %v). Remaining for that key: ('%v' -> %x)\n", action, id, key, key, fwd2[key])
+			}
+			if applyKeyRepairs {
+				u := u64tob(id)
+				err = id2key.Delete(u)
+				if err != nil {
+					return
+				}
+			}
+			continue
+		}
+		delete(rev2cp, id)
+
+		if key2 != key {
+			u := u64tob(id)
+			k := []byte(key2)
+			changeIDs[id] = &BeforeAfterIDChange{
+				BeforeString: key,
+				AfterString:  key2,
+			}
+			if verbose {
+				fmt.Printf("# %vkey-translation-update-id: (id %x -> %v). fwd2 for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2])
+			}
+			if applyKeyRepairs {
+				err = id2key.Put(u, k)
+				if err != nil {
+					return
+				}
+			}
+		}
+	}
+	// anything leftover in rev2cp is stuff that is new, only
+	// in rev2 and not in rev. It needs to be added.
+	for id, key2 := range rev2cp {
+		u := u64tob(id)
+		k := []byte(key2)
+		changeIDs[id] = &BeforeAfterIDChange{
+			IsAdd: true,
+			//BeforeString: left empty
+			AfterString: key2,
+		}
+		if verbose {
+			fmt.Printf("# %vkey-translation-add-id: (id %x -> %v). Fwd for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2])
+		}
+		if applyKeyRepairs {
+			err = id2key.Put(u, k)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	// We assume here that fwd2 is a super-set of fwd. No string keys
+	// should be deleted in the repair. Confirm that.
+	for key, id := range fwd {
+		_, ok := fwd2[key]
+		if !ok {
+			panic(fmt.Sprintf("fwd2 is missing a string key from fwd. key='%v' -> id='%x'", key, id))
+		}
+	}
+
+	for key2, id2 := range fwd2 {
+		//vv("makeStringKeyChanges on fwd2, key2='%v', id2=%x", key2, id2)
+		isPrimary := false
+		if topo != nil {
+			primary := topo.GetPrimaryForColKeyTranslation(s.index, key2)
+			isPrimary = s.partitionID == primary
+		}
+		_ = isPrimary
+		id, ok := fwd[key2]
+		if !ok {
+			adds[key2] = id2
+
+			u2 := u64tob(id2)
+			k2 := []byte(key2)
+			if verbose {
+				fmt.Printf("# %vkey-translation-new-key: ('%v' -> %x) added: isPrimary: %v\n", action, key2, id2, isPrimary)
+			}
+			if applyKeyRepairs {
+				err = key2id.Put(k2, u2)
+				if err != nil {
+					return
+				}
+				err = id2key.Put(u2, k2)
+				if err != nil {
+					return
+				}
+			}
+			continue
+		}
+		if id != id2 {
+			changeKeys[key2] = &BeforeAfterKeyChange{
+				BeforeID: id,
+				AfterID:  id2,
+			}
+			if verbose {
+				fmt.Printf("# %vkey-translation-change-id: ('%v' -> %x) changes to ('%v' ->  %x); isPrimary: %v\n", action, key2, id, key2, id2, isPrimary)
+			}
+			if applyKeyRepairs {
+				u2 := u64tob(id2)
+				k2 := []byte(key2)
+
+				err = key2id.Put(k2, u2)
+				if err != nil {
+					return
+				}
+				err = id2key.Put(u2, k2)
+				if err != nil {
+					return
+				}
+			}
+		}
+	}
+
+	if localTx {
+		err = tx.Commit()
+	}
+	return
+}
+
+func (s *TranslateStore) DumpBolt(label string) {
+
+	fmt.Printf("dumping bolt %v : path='%v'\n", label, s.Path)
+
+	_ = s.KeyWalker(func(key string, col uint64) {
+		fmt.Printf("keyWalker: key '%v' -> col '%x'\n", key, col)
+	})
+	_ = s.IDWalker(func(key string, col uint64) {
+		fmt.Printf("idWalker: id '%x' -> key '%v'\n", col, key)
+	})
+
+	fmt.Printf("DONE with dumping bolt %v;   path='%v'\n", label, s.Path)
+
+}

boltdb/translate_test.go

@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"os"
 	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
@@ -27,6 +28,8 @@ import (
 	"github.com/pilosa/pilosa/v2/boltdb"
 )
 
+//var vv = pilosa.VV
+
 func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
@@ -216,6 +219,30 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	}
 }
 
+func TestTranslateStore_MaxID(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	// Generate a bunch of keys.
+	var lastk uint64
+	for i := 0; i < 1026; i++ {
+		k, err := s.TranslateKey(strconv.Itoa(i), true)
+		if err != nil {
+			t.Fatalf("translating %d: %v", i, err)
+		}
+		lastk = k
+	}
+
+	// Verify the max ID.
+	max, err := s.MaxID()
+	if err != nil {
+		t.Fatalf("checking max ID: %v", err)
+	}
+	if max != lastk {
+		t.Fatalf("last key is %d but max is %d", lastk, max)
+	}
+}
+
 func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		s := MustOpenNewTranslateStore()
@@ -508,8 +535,7 @@ func TestCryptoHashPerKey(t *testing.T) {
 		}
 
 		// done with setup
-
-		sum, err := s.ComputeTranslatorSummary()
+		sum, err := s.ComputeTranslatorSummaryCols(0, pilosa.NewTopology(&pilosa.Jmphasher{}, pilosa.DefaultPartitionN, 1, nil))
 		if err != nil {
 			panic(err)
 		}
@@ -533,3 +559,123 @@ func TestCryptoHashPerKey(t *testing.T) {
 	}
 
 }
+
+func TestTranslateStore_RepairNonInvertibleStringKeyTranslation(t *testing.T) {
+
+	const N = 6
+	// before repair
+	var fwd [N]map[string]uint64
+	var rev [N]map[uint64]string
+
+	// after repair
+	var fwd2 [N]map[string]uint64
+	var rev2 [N]map[uint64]string
+
+	// case 0: forward is messed up (unlikely but check for it anyway, be sure we can repair)
+	// "key0" -> id 0  // correct.
+	// "key1" -> id 0  // wrong. after Repair, should see key1 -> 1 (0xec0002)
+	//
+	// id 0 -> "key0" // correct
+	// id 1 -> "key1" // correct
+	//
+	fwd[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00001}
+	rev[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+	fwd2[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 1: reverse is messed up (we have seen this in the past)
+	// "key0" -> id 0  // correct
+	// "key1" -> id 1  // correct
+	//
+	// id 0 -> "key0" // correct.
+	// id 1 -> "key0" // wrong. after Repair, should see id 1 -> "key1"
+	//
+	fwd[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key0"}
+	fwd2[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 2: only present in reverse.
+	fwd[2] = map[string]uint64{}
+	rev[2] = map[uint64]string{0xec00001: "key0"}
+	fwd2[2] = map[string]uint64{"key0": 0xec00001}
+	rev2[2] = map[uint64]string{0xec00001: "key0"}
+
+	// case 3: same thing. with camoflage.
+	fwd[3] = map[string]uint64{"key1": 0xec00002}
+	rev[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+	fwd2[3] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 4: only present in forward.
+
+	fwd[4] = map[string]uint64{"key0": 0xec00001}
+	rev[4] = map[uint64]string{}
+	fwd2[4] = map[string]uint64{"key0": 0xec00001}
+	rev2[4] = map[uint64]string{0xec00001: "key0"}
+
+	// case 5: same thing. with camoflage.
+	fwd[5] = map[string]uint64{"key0": 0xec00001}
+	rev[5] = map[uint64]string{0xec00002: "key1"}
+	fwd2[5] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[5] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 6: we had an id, but b/c of the fix, that id is no longer used.
+	//         now that id might still be used in the fragment for a column,
+	//         and so we will need to remove that id/column from the fragment.
+	// encapsulated: "did it affect the state of the fields?"
+
+	for i := 0; i < 5; i++ {
+		//println("i = ", i)
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		if err := s.SetFwdRevMaps(nil, fwd[i], rev[i]); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := verifyState("setup", i, s, fwd[i], rev[i]); err != nil {
+			t.Fatal(err)
+		}
+
+		var topo *pilosa.Topology
+		verbose := false
+		applyKeyRepairs := true
+		changed, err := s.RepairKeys(topo, verbose, applyKeyRepairs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !changed {
+			t.Fatalf("expected changes!")
+		}
+
+		if err := verifyState("afterRepair", i, s, fwd2[i], rev2[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func verifyState(label string, i int, s *boltdb.TranslateStore, fwd map[string]uint64, rev map[uint64]string) error {
+
+	// verify the setup
+	const writable = true
+	for key, expectID := range fwd {
+		id, err := s.TranslateKey(key, !writable)
+		if err != nil {
+			return err
+		}
+		if id != expectID {
+			return fmt.Errorf("fwd %v problem. i=%v, for key '%v', expected %x, observed %x", label, i, key, expectID, id)
+		}
+	}
+	for id, expectKey := range rev {
+		key, err := s.TranslateID(id)
+		if err != nil {
+			return err
+		}
+		if key != expectKey {
+			return fmt.Errorf("rev %v problem. i=%v, for id '%x', expected %v, observed %v", label, i, id, expectKey, key)
+		}
+	}
+	return nil
+}

cluster.go

@@ -62,7 +62,7 @@ const (
 	resizeJobActionAdd    = "ADD"
 	resizeJobActionRemove = "REMOVE"
 
-	defaultConfirmDownRetries = 120
+	defaultConfirmDownRetries = 10
 	defaultConfirmDownSleep   = 1 * time.Second
 )
 
@@ -208,6 +208,9 @@ type cluster struct { // nolint: maligned
 	// The number of replicas a partition has.
 	ReplicaN int
 
+	// Human-readable name of the cluster.
+	Name string
+
 	// Threshold for logging long-running queries
 	// TODO(2.0) move this out of cluster. (why is it here??)
 	longQueryTime time.Duration
@@ -257,7 +260,7 @@ type cluster struct { // nolint: maligned
 // newCluster returns a new instance of Cluster with defaults.
 func newCluster() *cluster {
 	return &cluster{
-		Hasher:     &jmphasher{},
+		Hasher:     &Jmphasher{},
 		partitionN: DefaultPartitionN,
 		ReplicaN:   1,
 
@@ -700,6 +703,12 @@ func (c *cluster) Nodes() []*Node {
 	return ret
 }
 
+func (c *cluster) AllNodeStates() map[string]string {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.Topology.nodeStates
+}
+
 // removeNodeBasicSorted removes a node from the cluster, maintaining the sort
 // order. Returns true if the node was removed. unprotected.
 func (c *cluster) removeNodeBasicSorted(nodeID string) bool {
@@ -969,12 +978,13 @@ func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResize
 	return m, nil
 }
 
-// shardPartition returns the partition that a shard belongs to.
-func (c *cluster) shardPartition(index string, shard uint64) int {
-	return shardPartition(index, shard, c.partitionN)
+// shardPartition returns the shard-partition that a shard belongs to.
+// NOTE: this is DIFFERENT from the key-partition
+func (c *cluster) shardToShardPartition(index string, shard uint64) int {
+	return shardToShardPartition(index, shard, c.partitionN)
 }
 
-func shardPartition(index string, shard uint64, partitionN int) int {
+func shardToShardPartition(index string, shard uint64, partitionN int) int {
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], shard)
 
@@ -985,12 +995,13 @@ func shardPartition(index string, shard uint64, partitionN int) int {
 	return int(h.Sum64() % uint64(partitionN))
 }
 
-// keyPartition returns the partition that a key belongs to.
-func (c *cluster) keyPartition(index, key string) int {
-	return keyPartition(index, key, c.partitionN)
+// keyPartition returns the key-partition that a key belongs to.
+// NOTE: the key-partition is DIFFERENT from the shard-partition.
+func (topo *Topology) KeyPartition(index, key string) int {
+	return keyToKeyPartition(index, key, topo.PartitionN)
 }
 
-func keyPartition(index, key string, partitionN int) int {
+func keyToKeyPartition(index, key string, partitionN int) int {
 	// Hash the bytes and mod by partition count.
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(index))
@@ -1000,7 +1011,7 @@ func keyPartition(index, key string, partitionN int) int {
 
 // idPartition returns the partition that an id belongs to.
 func (c *cluster) idPartition(index string, id uint64) int {
-	return shardPartition(index, id/ShardWidth, c.partitionN)
+	return shardToShardPartition(index, id/ShardWidth, c.partitionN)
 }
 
 // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use.
@@ -1012,7 +1023,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
 
 // shardNodes returns a list of nodes that own a fragment. unprotected
 func (c *cluster) shardNodes(index string, shard uint64) []*Node {
-	return c.partitionNodes(c.shardPartition(index, shard))
+	return c.partitionNodes(c.shardToShardPartition(index, shard))
 }
 
 // KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use.
@@ -1024,7 +1035,7 @@ func (c *cluster) KeyNodes(index, key string) []*Node {
 
 // keyNodes returns a list of nodes that own a key. unprotected
 func (c *cluster) keyNodes(index, key string) []*Node {
-	return c.partitionNodes(c.keyPartition(index, key))
+	return c.partitionNodes(c.Topology.KeyPartition(index, key))
 }
 
 // ownsShard returns true if a host owns a fragment.
@@ -1068,8 +1079,14 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
 	}
 
 	// Determine primary owner node.
-	nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN)
-
+	if c.Topology == nil {
+		c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
+	}
+	nodeIndex := c.Topology.PrimaryNodeIndex(partitionID)
+	if nodeIndex < 0 {
+		// no nodes anyway
+		return nil
+	}
 	// Collect nodes around the ring.
 	nodes := make([]*Node, 0, replicaN)
 	for i := 0; i < replicaN; i++ {
@@ -1100,11 +1117,66 @@ func (c *cluster) unprotectedPrimaryPartitionNode(partition int) *Node {
 	return nil
 }
 
+func (topo *Topology) IsPrimary(nodeID string, partitionID int) bool {
+	primary := topo.PrimaryNodeIndex(partitionID)
+	return nodeID == topo.nodeIDs[primary]
+}
+
+func (topo *Topology) PrimaryNodeIndex(partitionID int) (nodeIndex int) {
+	n := len(topo.nodeIDs)
+	if n == 0 {
+		if topo.cluster != nil {
+			n = len(topo.cluster.nodes)
+		}
+	}
+	nodeIndex = topo.Hasher.Hash(uint64(partitionID), n)
+	return
+}
+
+func (topo *Topology) GetNonPrimaryReplicas(partitionID int) (nonPrimaryReplicas []string) {
+
+	primary := topo.PrimaryNodeIndex(partitionID)
+	nodeN := len(topo.nodeIDs)
+
+	// Collect nodes around the ring.
+	for i := 1; i < nodeN; i++ {
+		nodeID := topo.nodeIDs[(primary+i)%nodeN]
+		if i < topo.ReplicaN {
+			nonPrimaryReplicas = append(nonPrimaryReplicas, nodeID)
+		}
+	}
+	return
+}
+
+// the map replicaNodeIDs[nodeID] will have a true value for the primary nodeID, and false for others.
+func (topo *Topology) GetReplicasForPrimary(primary int) (replicaNodeIDs, nonReplicas map[string]bool) {
+	if primary < 0 {
+		// no nodes anyway
+		return
+	}
+	replicaNodeIDs = make(map[string]bool)
+	nonReplicas = make(map[string]bool)
+
+	nodeN := len(topo.nodeIDs)
+
+	// Collect nodes around the ring.
+	for i := 0; i < nodeN; i++ {
+		nodeID := topo.nodeIDs[(primary+i)%nodeN]
+		if i < topo.ReplicaN {
+			// mark true if primary
+			replicaNodeIDs[nodeID] = (i == 0)
+		} else {
+			nonReplicas[nodeID] = false
+		}
+	}
+	return
+}
+
 // containsShards is like OwnsShards, but it includes replicas.
 func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 {
 	var shards []uint64
 	_ = availableShards.ForEach(func(i uint64) error {
-		p := c.shardPartition(index, i)
+		p := c.shardToShardPartition(index, i)
 		// Determine the nodes for partition.
 		nodes := c.partitionNodes(p)
 		for _, n := range nodes {
@@ -1124,10 +1196,10 @@ type Hasher interface {
 }
 
 // jmphasher represents an implementation of jmphash. Implements Hasher.
-type jmphasher struct{}
+type Jmphasher struct{}
 
 // Hash returns the integer hash for the given key.
-func (h *jmphasher) Hash(key uint64, n int) int {
+func (h *Jmphasher) Hash(key uint64, n int) int {
 	b, j := int64(-1), int64(0)
 	for j < int64(n) {
 		b = j
@@ -1193,7 +1265,7 @@ func (c *cluster) waitForStarted() error {
 
 		c.logger.Printf("%v wait for joining to complete", c.Node.ID)
 		<-c.joining
-		c.logger.Printf("joining has completed")
+		c.logger.Printf("joining has completed. I am NodeID '%v'", c.Node.ID)
 	}
 	return nil
 }
@@ -1844,6 +1916,8 @@ func (n nodeIDs) ContainsID(id string) bool {
 }
 
 // Topology represents the list of hosts in the cluster.
+// Topology now encapsulates all knowledge needed to
+// determine the primary node in the replication scheme.
 type Topology struct {
 	mu      sync.RWMutex
 	nodeIDs []string
@@ -1853,14 +1927,68 @@ type Topology struct {
 	// nodeStates holds the state of each node according to
 	// the coordinator. Used during startup and data load.
 	nodeStates map[string]string
+
+	// moved Hasher, PartitionN and ReplicaN
+	// from cluster for standalone use and comprehension:
+
+	// Hashing algorithm used to assign partitions to nodes.
+	Hasher Hasher
+	// The number of partitions in the cluster.
+	PartitionN int
+	// The number of replicas a partition has.
+	ReplicaN int
+
+	// can be nil
+	cluster *cluster
 }
 
-func newTopology() *Topology {
+// NewTopology creates a Topology.
+//
+// The arguments and members hasher, partitionN, and
+// replicaN were refactored out of struct cluster
+// to allow pilosa-fsck to load a Topology from
+// backup and then compute primaries standalone -- without starting a cluster.
+// As pilosa-fsck operates on all backups at once from
+// a single cpu, starting a full cluster isn't possible.
+//
+// The hasher is the Hashing algorithm used to assign partitions to nodes.
+// The cluster c should be provided if possible by pilosa code;
+// the pilosa-fsck utility won't be able to provide it.
+//
+// For the cluster size N, the topology gives preference to
+// len(t.nodeIDs) before falling back on len(c.nodes).
+//
+func NewTopology(hasher Hasher, partitionN int, replicaN int, c *cluster) *Topology {
 	return &Topology{
+		Hasher:     hasher,
+		PartitionN: partitionN,
+		ReplicaN:   replicaN,
 		nodeStates: make(map[string]string),
+		cluster:    c,
 	}
 }
 
+func (t *Topology) String() string {
+	return fmt.Sprintf(`
+&pilosa.Topology{
+		nodeIDs:    %v,
+		clusterID:  %v,
+		nodeStates: %v,
+		PartitionN: %v,
+		ReplicaN:   %v,
+}
+`,
+		t.nodeIDs,
+		t.clusterID,
+		t.nodeStates,
+		t.PartitionN,
+		t.ReplicaN,
+	)
+}
+func (t *Topology) GetNodeIDs() []string {
+	return t.nodeIDs
+}
+
 // ContainsID returns true if id matches one of the topology's IDs.
 func (t *Topology) ContainsID(id string) bool {
 	t.mu.RLock()
@@ -1924,7 +2052,7 @@ func (t *Topology) encode() *internal.Topology {
 func (c *cluster) loadTopology() error {
 	buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology"))
 	if os.IsNotExist(err) {
-		c.Topology = newTopology()
+		c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
 		return nil
 	} else if err != nil {
 		return errors.Wrap(err, "reading file")
@@ -1934,7 +2062,7 @@ func (c *cluster) loadTopology() error {
 	if err := proto.Unmarshal(buf, &pb); err != nil {
 		return errors.Wrap(err, "unmarshalling")
 	}
-	top, err := decodeTopology(&pb)
+	top, err := DecodeTopology(&pb, c.Hasher, c.partitionN, c.ReplicaN, c)
 	if err != nil {
 		return errors.Wrap(err, "decoding")
 	}
@@ -1945,7 +2073,6 @@ func (c *cluster) loadTopology() error {
 
 // saveTopology writes the current topology to disk. unprotected.
 func (c *cluster) saveTopology() error {
-
 	if err := os.MkdirAll(c.Path, 0777); err != nil {
 		return errors.Wrap(err, "creating directory")
 	}
@@ -2452,6 +2579,27 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys
 	return ids, nil
 }
 
+// The boltdb key translation stores are partitioned, designated by partitionIDs. These
+// are shared between replicas, and one node is the primary for
+// replication. So with 4 nodes and 3-way replication, each node has 3/4 of
+// the translation stores on it.
+func (topo *Topology) GetPrimaryForColKeyTranslation(index, key string) (primary int) {
+	partitionID := topo.KeyPartition(index, key)
+	return topo.PrimaryNodeIndex(partitionID)
+}
+
+// should match cluster.go:1033 cluster.ownsShard(nodeID, index, shard)
+// 	return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID)
+func (t *Topology) GetPrimaryForShardReplication(index string, shard uint64) int {
+	n := len(t.nodeIDs)
+	if n == 0 {
+		return -1
+	}
+	partition := uint64(shardToShardPartition(index, shard, t.PartitionN))
+	nodeIndex := t.Hasher.Hash(partition, n)
+	return nodeIndex
+}
+
 func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) {
 	keyMap := make(map[string]uint64)
 
@@ -2463,7 +2611,7 @@ func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, ke
 	// Split keys by partition.
 	keysByPartition := make(map[int][]string, c.partitionN)
 	for key := range keySet {
-		partitionID := c.keyPartition(indexName, key)
+		partitionID := c.Topology.KeyPartition(indexName, key)
 		keysByPartition[partitionID] = append(keysByPartition[partitionID], key)
 	}
 
@@ -2642,12 +2790,13 @@ func encodeTopology(topology *Topology) *internal.Topology {
 	}
 }
 
-func decodeTopology(topology *internal.Topology) (*Topology, error) {
+// the cluster c is optional but give it if you have it.
+func DecodeTopology(topology *internal.Topology, hasher Hasher, partitionN, replicaN int, c *cluster) (*Topology, error) {
 	if topology == nil {
 		return nil, nil
 	}
 
-	t := newTopology()
+	t := NewTopology(hasher, partitionN, replicaN, c)
 	t.clusterID = topology.ClusterID
 	t.nodeIDs = topology.NodeIDs
 	sort.Slice(t.nodeIDs,

cluster_internal_test.go

@@ -381,7 +381,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.shardPartition(index, shard)
+		partitionID := c.shardToShardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -411,7 +411,7 @@ func TestHasher(t *testing.T) {
 		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
 	} {
 		for i, v := range tt.bucket {
-			hasher := &jmphasher{}
+			hasher := &Jmphasher{}
 			if got := hasher.Hash(tt.key, i+1); got != v {
 				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
 			}
@@ -1053,3 +1053,27 @@ func TestCluster_confirmNodeDownDown(t *testing.T) {
 		t.Errorf("expected node to be down")
 	}
 }
+
+func TestCluster_GetNonPrimaryReplicas(t *testing.T) {
+
+	c := newCluster()
+	c.ReplicaN = 3
+	topo := NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
+	c.Topology = topo
+	nNodes := 4
+	for i := 0; i < nNodes; i++ {
+		nodeID := fmt.Sprintf("node%d", i)
+		c.nodes = append(c.nodes, &Node{
+			ID:  nodeID,
+			URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
+		})
+		c.Topology.addID(nodeID)
+	}
+
+	partitionID := 256
+	nonPrimes := topo.GetNonPrimaryReplicas(partitionID)
+	m := len(nonPrimes)
+	if m != c.ReplicaN-1 {
+		t.Fatalf("expected 2 non primes, got %v", m)
+	}
+}

cmd/pilosa-chk/chk.go

@@ -76,8 +76,10 @@ func main() {
 
 	final := pilosa.NewAllTranslatorSummary()
 	const verbose = true
+	const checkKeys = false
+	const applyKeyRepairs = false
 	for _, idx := range holder.Indexes() {
-		asum, err := idx.ComputeTranslatorSummary(verbose)
+		asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID", 10)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -103,7 +105,7 @@ func main() {
 			fmt.Printf("==============================\n")
 			fmt.Printf("index: %v\n", idx.Name())
 			fmt.Printf("==============================\n")
-			idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog)
+			idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, nil, verbose)
 		}
 	}
 }

cmd/pilosa-fsck/Makefile

@@ -0,0 +1,36 @@
+.PHONY: install build release
+
+CLONE_URL=github.com/pilosa/pilosa
+VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
+VARIANT = Molecula
+VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
+BUILD_TIME := $(shell date -u +%FT%T%z)
+SHARD_WIDTH = 20
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)"
+GOOS = $(shell go env GOOS)
+
+# Install pilosa-fsck
+install:
+	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) 
+
+# Compile pilosa-fsck
+build:
+	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS)
+
+REL = release-pilosa-fsck.$(COMMIT).$(GOOS)
+
+release:
+	mkdir $(REL)
+	cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - )
+	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck
+	tar cf - $(REL) | gzip > $(REL).tar.gz
+	rm -rf $(REL)
+	mv $(REL).tar.gz ../..
+
+clean:
+	find . -name pilosa-fsck | xargs rm -f
+	rm -f release-pilosa-fsck*.tar.gz

cmd/pilosa-fsck/fsck.go

@@ -0,0 +1,985 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/internal"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pkg/errors"
+	"github.com/zeebo/blake3"
+)
+
+// pilosa-fsck :
+// an external customer tool (originally for Q2) to do 2 jobs:
+// Given a set of cluster backups (and their .id and .topology files)
+// mounted on the same file system, we can:
+// 1) scan for fragment differences between the primary and its replicas (default); or
+// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given).
+//
+// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
+// forcing a customer to upgrade or downgrade their installed version.
+
+// FsckConfig configures the dumpcols() and/or read() runs.
+type FsckConfig struct {
+	Fix    bool // -fix
+	FixCol bool // -fixcol
+
+	Colkeydump    bool   // -col
+	JustThisIndex string // -index
+
+	// -col column key dump only options:
+	// Dir         string
+	// PartitionID int
+	// ShowHeader  bool
+	// ShowKey     bool
+	// ShowID      bool
+
+	// not flags, just the Args() left after all other flags. Should be the list
+	// of pilosa (holder) directories for the cluster.
+	Dirs []string
+
+	Verbose bool // -v
+	Quiet   bool // -q
+
+	// manual workaround for not having PilosaConfigPath, if really need be.
+	ReplicaN         int    // -replicas
+	PilosaConfigPath string // -config
+
+	ParallelReaders int //  -readers
+
+	topo *pilosa.Topology
+}
+
+// call DefineFlags before myflags.Parse()
+func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol")
+	fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.")
+	//fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis")
+	fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet")
+
+	fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.")
+
+	fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.")
+
+	fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)")
+
+	fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.")
+
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo())
+		fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
+
+  -fix
+    	(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
+
+  -replicas R 
+        (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
+        the number of replicas maintained in the cluster. Must be the same as the 
+        [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
+
+  -index index_name
+        (optional) restrict to just this index. Otherwise we default to all indexes.
+
+  -readers PR
+        how many parallel readers to use to scan at once. PR==0 means do everything 
+        possible in parallel. PR==1 means serialize everything through a single reader. 
+        Adjust PR to control memory consumption if needed. As a practical limit, setting 
+        PR > 10000 will have no effect. (default is 10).
+
+  -q
+        be very quiet during analysis and repair
+
+`)
+		fmt.Fprintf(os.Stderr, `
+Welcome to pilosa-fsck. This is a scan and repair 
+tool that is modeled after the classic unix file 
+system utility fsck. 
+
+WARNING: DO NOT RUN ON A LIVE SYSTEM.
+
+The most important point to remember is that analysis
+and repair must be done *offline*.
+
+Just as fsck must be run on an unmounted disk, 
+pilosa-fsck must be run on a backup. It must 
+not be run on the directories where a live Pilosa system
+is serving queries. Instead, take a backup first.
+A backup is a set of N Pilosa data directories that have been
+copied from your live system. They must all 
+be visible and mounted on one filesystem together.
+
+pilosa-fsck can be run in scan-mode (without -fix),
+or in repair-mode with -fix. The console output
+supplies a log documenting the analysis 
+and showing what data changes would have been made.
+
+REQUIRED COMMAND LINE ARGUMENTS
+
+The paths to all the top-level Pilosa 
+data directories in a cluster must be given on the command
+line. The -replicas R flag is also always required. It
+must be correct for your cluser. Here R is the same as
+the [cluster] stanza "replicas = R" line from your
+pilosa.conf.
+
+Example:
+
+Suppose you are ready to run pilosa-fsck: 
+you have taken a backup of your four node Pilosa 
+cluster and stored it all on one filesystem with 
+all nodes visible and uncompressed. This
+is a pre-requisite to running pilosa-fsck. 
+Let's suppose we have replication R = 3 set.
+In this example, have stored our backed-up directories in 
+
+/backup/molecula
+
+and the four node backups are in 
+subdirectories node1/ node2/ node3/ node4/ under this:
+
+/backup/molecula/node1/
+/backup/molecula/node1/.pilosa/.id
+/backup/molecula/node1/.pilosa/.topology
+/backup/molecula/node1/.pilosa/myindex
+
+/backup/molecula/node2/
+/backup/molecula/node2/.pilosa/.id
+/backup/molecula/node2/.pilosa/.topology
+/backup/molecula/node2/.pilosa/myindex
+
+/backup/molecula/node3/
+/backup/molecula/node3/.pilosa/.id
+/backup/molecula/node3/.pilosa/.topology
+/backup/molecula/node3/.pilosa/myindex
+
+/backup/molecula/node4/
+/backup/molecula/node4/.pilosa/.id
+/backup/molecula/node4/.pilosa/.topology
+/backup/molecula/node4/.pilosa/myindex
+
+NOTE: your .pilosa directories need not be named .pilosa. They can
+be something else, such as when the -d flag to pilosa server was used.
+The .id file, the .topology file, and the index directories must be 
+found directly underneath.
+
+Then a typical invocation to scan a cluster backup for issues:
+
+$ cd /backup/molecula/
+$ pilosa-fsck -replicas 3  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+A typical invocation to repair the replication in the same backup:
+
+$ pilosa-fsck -replicas 3 -fix  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+In both cases, the .id and .topology files must 
+be present in the backups.
+
+Without -fix, no modifications will be made to the backups. Only
+by running with -fix will repairs be made. The user can safely
+always run with -fix to repair only if needed.
+
+A zero error code will be returned to the shell if no repairs were needed.
+
+A zero error code will be also be returned to the shell if 
+repairs were needed and they were accomplished under -fix.
+
+A non-zero error code indicates that repairs were needed but
+were not made.
+`)
+	}
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *FsckConfig) ValidateConfig() error {
+	if c.Fix {
+		c.FixCol = true
+	}
+	if c.ReplicaN == 0 && c.PilosaConfigPath == "" {
+		return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)")
+	}
+
+	if c.ReplicaN == 0 && c.PilosaConfigPath != "" {
+
+		if !FileExists(c.PilosaConfigPath) {
+			return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath)
+		}
+		by, err := ioutil.ReadFile(c.PilosaConfigPath)
+		if err != nil {
+			return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err)
+		}
+		srvcfg, err := server.ParseConfig(string(by))
+		if err != nil {
+			//vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err)
+
+			// fall back to manual parsing of config
+			lines := strings.Split(string(by), "\n")
+			clusterStart := -1
+			for i, line := range lines {
+				if strings.Contains(line, `[cluster]`) {
+					clusterStart = i
+				}
+				if i > clusterStart {
+					if strings.Contains(line, "replicas") {
+						split := strings.Split(line, "=")
+						ns := strings.TrimSpace(split[1])
+						n, err := strconv.Atoi(ns)
+						if err != nil {
+							return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err)
+						}
+						c.ReplicaN = n
+					}
+				}
+			}
+		} else {
+			c.ReplicaN = srvcfg.Cluster.ReplicaN
+		}
+		if c.ReplicaN == 0 {
+			return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath)
+		}
+		//vv("c.ReplicaN = %v", c.ReplicaN)
+	}
+	return nil
+}
+
+var ProgramName = "pilosa-fsck"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError)
+	cfg := &FsckConfig{}
+	cfg.DefineFlags(myflags)
+	cfg.Verbose = true
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+	dirs := myflags.Args()
+	nDir := len(dirs)
+	if nDir <= 0 && !cfg.Colkeydump {
+		fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName)
+		os.Exit(1)
+	}
+
+	cmdline := strings.Join(os.Args, " ")
+
+	// make sure all the dir are distinct
+	dup := make(map[string]bool)
+	for _, dir := range dirs {
+		if dup[dir] {
+			fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline)
+			os.Exit(1)
+		} else {
+			dup[dir] = true
+		}
+	}
+
+	fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo())
+	cwd, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err)
+		os.Exit(1)
+	}
+	fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd)
+	fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline)
+	t0 := time.Now()
+	fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0))
+	defer func() {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+	}()
+	cfg.Dirs = dirs
+
+	fixNeeded, err := cfg.Run()
+	if err != nil {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+	if fixNeeded && !cfg.Fix {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+		fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n")
+		os.Exit(1)
+	}
+}
+
+func (cfg *FsckConfig) Run() (fixNeeded bool, err error) {
+
+	//	if cfg.Colkeydump {
+	//	cfg.dumpcols()
+	//}
+
+	perNodeIndexMaps, clusterNodes, ats, err := cfg.read()
+	if err != nil {
+		return false, err
+	}
+
+	if cfg.FixCol {
+		err := cfg.RepairTranslationStores(ats)
+		if err != nil {
+			return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err)
+		}
+	}
+
+	//vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes)
+
+	fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats)
+	if err != nil {
+		return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err)
+	}
+	fixNeeded = ats.RepairNeeded || fixme
+	for _, report := range reports {
+		fmt.Printf("%v\n", report)
+	}
+	if len(reports) == 0 {
+		fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " "))
+	}
+	return
+}
+
+var _ = (&FsckConfig{}).dumpAts
+
+func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) {
+	fmt.Printf("# dumpAts:  	RepairNeeded=%v\n", ats.RepairNeeded)
+	for _, sum := range ats.Sums {
+		fmt.Printf("# sum = '%#v'\n", sum)
+	}
+
+}
+
+type group struct {
+	elem        []*pilosa.TranslatorSummary
+	partitionID int
+}
+
+func (g *group) String() (s string) {
+	for i, e := range g.elem {
+		s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String())
+	}
+	return
+}
+
+func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) {
+	indexMap := make(map[string]bool)
+	for _, sum := range ats.Sums {
+		if !indexMap[sum.Index] {
+			indexMap[sum.Index] = true
+			indexes = append(indexes, sum.Index)
+		}
+	}
+	sort.Strings(indexes)
+	return
+}
+
+func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) {
+
+	verbose := cfg.Verbose
+
+	// group by index first. then repair.
+	indexes := indexesFromAts(ats)
+
+	for _, index := range indexes {
+
+		if !cfg.DoingIndex(index) {
+			continue
+		}
+
+		m := make(map[int]*group)
+		for _, sum := range ats.Sums {
+
+			if !sum.IsColKey || sum.Index != index {
+				continue
+			}
+			grp := m[sum.PartitionID]
+			if grp == nil {
+				grp = &group{
+					partitionID: sum.PartitionID,
+				}
+				m[sum.PartitionID] = grp
+			}
+			grp.elem = append(grp.elem, sum)
+		}
+
+		for partitionID, group := range m {
+			_ = partitionID
+			prim := -1
+			keyCount := 0
+			for k, e := range group.elem {
+				if e.IsPrimary {
+					prim = k
+				}
+				keyCount += e.KeyCount
+			}
+			if prim == -1 {
+				panic(fmt.Sprintf("no primary found for group '%v'", group.String()))
+			}
+
+			primary := group.elem[prim]
+			primaryChecksum := primary.Checksum
+			for _, e := range group.elem {
+				if e.IsPrimary {
+					continue
+				}
+				// is e a replica? not necessarily! have to check.
+				if !e.IsReplica {
+					//if verbose {
+					// since this will happen even on a fix point, where it is already empty,
+					// we don't report it again.
+					//fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath)
+					//}
+					err := os.RemoveAll(e.StorePath)
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath))
+					}
+					store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN)
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath))
+					}
+					err = store.Close()
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath))
+					}
+					continue
+				}
+				// INVAR: e is a replica for this paritionID.
+				// Copy from primary if checksums are different.
+				if e.Checksum != primaryChecksum {
+					from := group.elem[prim].StorePath
+					dest := e.StorePath
+					if verbose {
+						fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest)
+					}
+					err := cp(from, dest)
+					if err != nil {
+						return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+/*
+func (cfg *FsckConfig) dumpcols() {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	dir := cfg.Dir
+	index := cfg.Index
+	partitionID := cfg.PartitionID
+	showKey := cfg.ShowKey
+	showID := cfg.ShowID
+
+	if !quiet {
+		fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir)
+	}
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+	err := holder.Open()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if cfg.ShowHeader {
+		fmt.Println("# columnKey columId")
+	}
+	id_key := make(map[uint64]string)
+	key_id := make(map[string]uint64)
+	for _, idx := range holder.Indexes() {
+		fmt.Printf("# Looking '%v'\n", idx.Name())
+		if idx.Name() == index {
+			store := idx.TranslateStore(partitionID)
+			fmt.Printf("# Key By ID partitionID = %v\n", partitionID)
+			err := store.KeyWalker(func(key string, col uint64) {
+				key_id[key] = col
+				if showKey {
+					fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID)
+				}
+			})
+			panicOn(err)
+		}
+	}
+	for _, idx := range holder.Indexes() {
+		if idx.Name() == index {
+			store := idx.TranslateStore(partitionID)
+			//fmt.Printf("# ID ByKey\n")
+			err := store.IDWalker(func(key string, col uint64) {
+				id_key[col] = key
+				if showID {
+					fmt.Printf("# '%v' %v\n", key, col)
+				}
+			})
+			panicOn(err)
+		}
+	}
+	fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key))
+	fmt.Println("id_key")
+	for k, v := range id_key {
+		l, ok := key_id[v]
+		if ok {
+			if k != l {
+				fmt.Printf("# X: %v %v %v\n", k, l, v)
+			}
+		} else {
+			fmt.Printf("# key not in id %v\n", v)
+		}
+	}
+	fmt.Println("key_id")
+	for k, v := range key_id {
+		l, ok := id_key[v]
+		if ok {
+			if k != l {
+				fmt.Printf("# T: %v %v %v\n", k, l, v)
+			}
+		} else {
+			fmt.Printf("# id not in key %v\n", v)
+		}
+	}
+}
+*/
+
+func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) {
+
+	final = pilosa.NewAllTranslatorSummary()
+
+	dirs := cfg.Dirs
+	for _, dir := range dirs {
+		idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		final.Append(atsNode)
+		clusterNodes = append(clusterNodes, nodeID)
+		perNodeIndexMaps = append(perNodeIndexMaps, idx2frag)
+	}
+	return
+}
+
+func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+
+	if !quiet {
+		fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir)
+	}
+
+	jmphasher := &pilosa.Jmphasher{}
+	partitionN := pilosa.DefaultPartitionN
+	replicaN := cfg.ReplicaN
+	topo, err := loadTopology(dir, jmphasher, partitionN, replicaN)
+	if err != nil {
+		return nil, "", nil, err
+	}
+	cfg.topo = topo
+	//vv("topo = '%#v'", topo)
+	nodeIDs := topo.GetNodeIDs()
+	//vv("nodeIDs = '%#v'", nodeIDs)
+	nNodes := len(nodeIDs)
+	nDir := len(cfg.Dirs)
+	if nDir != nNodes {
+		return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs)
+	}
+
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+
+	nodeID, err = holder.LoadNodeID()
+	panicOn(err)
+	//vv("nodeID = '%v'", nodeID)
+	err = holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if !quiet {
+		fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
+	}
+	var indexes []*pilosa.Index
+
+	const checkKeys = true
+	atsNode = pilosa.NewAllTranslatorSummary()
+	for _, idx := range holder.Indexes() {
+
+		if !cfg.DoingIndex(idx.Name()) {
+			continue
+		}
+
+		//vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol)
+
+		asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders)
+		if err != nil {
+			log.Fatal(err)
+		}
+		atsNode.Append(asum)
+		indexes = append(indexes, idx)
+	}
+	atsNode.Sort()
+
+	hasher := blake3.New()
+	if !quiet {
+		fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir)
+	}
+	for _, sum := range atsNode.Sums {
+		if !quiet {
+			fmt.Printf("# index: %v  partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		}
+		_, _ = hasher.Write([]byte(sum.Checksum))
+	}
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	if !quiet {
+		fmt.Printf("# all-checksum = blake3-%x\n", buf)
+	}
+
+	// fragment analysis
+
+	showBits := false
+	showOpsLog := false
+	idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node.
+	for _, idx := range indexes {
+		if verbose {
+			fmt.Printf("# ==============================\n")
+			fmt.Printf("# index: %v\n", idx.Name())
+			fmt.Printf("# ==============================\n")
+		}
+		frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose)
+		frgsum.Dir = dir
+		frgsum.NodeID = nodeID
+		idx2frag[idx.Name()] = frgsum
+	}
+
+	_ = holder.Close()
+
+	//vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple.
+
+	return
+}
+
+func (cfg *FsckConfig) DoingIndex(index string) bool {
+	if cfg.JustThisIndex == "" {
+		// scan all indexes
+		return true
+	}
+	if index == cfg.JustThisIndex {
+		// scan just this one
+		return true
+	}
+	return false
+}
+
+// from cluster.go:1924
+func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) {
+
+	buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology"))
+	if err != nil {
+		return nil, err
+	}
+
+	var pb internal.Topology
+	err = proto.Unmarshal(buf, &pb)
+	if err != nil {
+		return nil, err
+	}
+
+	return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil)
+}
+
+func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	allIndex := make(map[string]bool)
+	for _, mp := range perNodeIndexMaps {
+		for index := range mp {
+			allIndex[index] = true
+		}
+	}
+	if !quiet {
+		vv("allIndex = '%#v'", allIndex)
+	}
+	for index := range allIndex {
+		if !quiet {
+			vv("on index '%v'", index)
+		}
+		nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary)
+		for _, mp := range perNodeIndexMaps {
+			sum := mp[index]
+			if sum == nil {
+				continue
+			}
+			nodes2fragsum[sum.NodeID] = sum
+		}
+		fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats)
+		if err != nil {
+			return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err)
+		}
+		fixNeeded = fixNeeded || fixme
+		reports = append(reports, report)
+	}
+	return fixNeeded, reports, nil
+}
+
+func (cfg *FsckConfig) analyzeThisIndex(
+	index string,
+	nodes2fragsum map[string]*pilosa.IndexFragmentSummary,
+	ats *pilosa.AllTranslatorSummary,
+) (fixNeeded bool, report string, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	var removedBytes int64
+	var copiedBytes int64
+	var changedFiles int64
+	var totalFiles int64
+	var overwrittenBytes int64
+	var totalBytes int64
+
+	if !quiet {
+		vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'",
+			index, len(nodes2fragsum), nodes2fragsum)
+	}
+
+	for node, sum := range nodes2fragsum {
+		if !quiet {
+			fmt.Printf("# on node '%v'\n", node)
+		}
+		// do they disagree on who is the primary?
+		// for each fragment, do they disagree on the checksum?
+
+		// Q: which nodes are supposed to have data, and which
+		// nodes are not supposed to have data?
+
+		//	loopFragSum:
+		for relpath, fragsum := range sum.RelPath2fsum {
+			fragsum.NodeID = node
+			totalFiles++
+			//vv("checking %v on node %v", relpath, node)
+
+			replicas, nonReplicas := cfg.topo.GetReplicasForPrimary(fragsum.Primary)
+			_, _ = replicas, nonReplicas
+			//vv("replicas = '%#v'", replicas)
+			//vv("nonReplicas = '%#v'", nonReplicas)
+
+			err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum)
+			if err != nil {
+				return fixNeeded, "", err
+			}
+
+			// find the primary's checksum
+			primaryChecksum := ""
+			var primaryFragSum *pilosa.FragSum
+			for node, isPrimary := range replicas {
+				if isPrimary {
+					primarySum := nodes2fragsum[node]
+					primaryFragSum = primarySum.RelPath2fsum[relpath]
+					if primaryFragSum == nil {
+
+						// This seems clear indication that we have the topology wrong.
+						// When the topology is right, there are NO errors of this kind.
+						//
+						msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas)
+						vv(msg)
+						fmt.Fprintf(os.Stderr, "%v\n", msg)
+						panic(msg) // stop. the fixes are going to be wrong.
+					} else {
+						primaryChecksum = primaryFragSum.Checksum
+						primaryFragSum.NodeID = node
+						primaryFragSum.ScanDone = true
+					}
+					break
+				}
+			}
+			if primaryChecksum == "" {
+				return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum)
+			}
+
+			// is this a non-replica?
+			_, isNon := nonReplicas[fragsum.NodeID]
+			if isNon {
+				removedBytes += FileSize(fragsum.AbsPath)
+				changedFiles++
+
+				//vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID)
+				if !quiet {
+					fmt.Printf("rm %v             #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum)
+				}
+				if cfg.Fix {
+					err := os.Remove(fragsum.AbsPath)
+					if err != nil {
+						return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err)
+					}
+				}
+			} else {
+				presz := FileSize(fragsum.AbsPath)
+				totalBytes += presz
+
+				checksum := fragsum.Checksum
+				if checksum != primaryChecksum {
+					copiedBytes += FileSize(primaryFragSum.AbsPath)
+					changedFiles++
+					overwrittenBytes += presz
+
+					if !quiet {
+						fmt.Printf("cp %v %v        #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum)
+					}
+					if cfg.Fix {
+						err := cp(primaryFragSum.AbsPath, fragsum.AbsPath)
+						if err != nil {
+							return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'",
+								primaryFragSum.AbsPath, fragsum.AbsPath, err)
+						}
+					}
+				}
+			}
+			fragsum.ScanDone = true
+		}
+	}
+	nDir := len(nodes2fragsum)
+
+	keyCount, idCount := cfg.getKeyIDCounts(index, ats)
+
+	fixNeeded = changedFiles > 0 || ats.RepairNeeded
+	var actionTaken string
+	var wouldBe string
+	if cfg.Fix || cfg.FixCol {
+		if fixNeeded {
+			actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*"
+			wouldBe = "sync repairs made:"
+		} else {
+			wouldBe = ""
+			actionTaken = "NO REPAIR NEEDED."
+		}
+	} else {
+		if fixNeeded {
+			wouldBe = "sync actions that would be taken under -fix:"
+			actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted."
+		} else {
+			wouldBe = ""
+			actionTaken = "NO REPAIR NEEDED."
+		}
+	}
+	var fragUpdate string
+	if changedFiles > 0 {
+		fragUpdate = fmt.Sprintf(`
+#  %v
+#    copied bytes: %v
+#    file bytes overwritten: %v
+#    new bytes added: %v
+#    new bytes is %0.01f%% of %v total bytes
+#    removed %v bytes from non-replicas
+#    changed file count %v (%0.01f%%; total files=%v)
+#
+`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles))
+	}
+
+	report = fmt.Sprintf(`
+# ========================================================
+#   pilosa-fsck final report 
+#
+#     run with    -fix: %v 
+#
+# index examined: '%v'
+#
+# nodes examined: %v
+#       -replicas %v replication factor used
+#
+# feature data examined: %v bytes
+# feature files examined: %v files
+#
+# key-translation-stores examined: %v
+#                       key-count: %v   over all replicas
+#                        id-count: %v   over all replicas
+#
+# %v
+# %v
+# ========================================================
+`,
+		cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate)
+	return
+}
+
+func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error {
+	for node := range replicas {
+		if nodes2fragsum[node] == nil {
+			return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum)
+		}
+	}
+	return nil
+}
+
+func cp(fromPath, toPath string) (err error) {
+	tmpTo := toPath + ".fsck.tmp"
+	toFd, err := os.Create(tmpTo)
+	if err != nil {
+		return err
+	}
+	defer toFd.Close()
+	fromFd, err := os.Open(fromPath)
+	if err != nil {
+		return err
+	}
+	defer fromFd.Close()
+
+	_, err = io.Copy(toFd, fromFd)
+	if err != nil {
+		return err
+	}
+	err = toFd.Close()
+	if err != nil {
+		return err
+	}
+	return os.Rename(tmpTo, toPath)
+}
+
+func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) {
+	for _, sum := range ats.Sums {
+		if sum.Index == index {
+			keyCount += sum.KeyCount
+			idCount += sum.IDCount
+		}
+	}
+	return
+}

cmd/pilosa-fsck/fsck_test.go

@@ -0,0 +1,445 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/hash"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_Repair(t *testing.T) {
+
+	// a) setup 1 primary + 3 replicas of disagree-ing cluster dirs.
+
+	nNodes := 4
+	nReplicas := 3
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[1]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[2]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[3]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	// note: do not defer c.Close() here. We manually close below.
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+
+	ctx := context.Background()
+
+	index := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(index))
+	field := make([]*pilosa.Field, len(index))
+	var err error
+
+	for i := range index {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	rowID := uint64(1)
+	timestamp := int64(0)
+
+	for i := range index {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          index[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+		//qcx.Reset()
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID)
+
+		// Query node0.
+		if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
+		}
+
+		// Query node1.
+		if err := test.RetryUntil(5*time.Second, func() error {
+			if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
+				return err
+			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+				return fmt.Errorf("unexpected column keys: %#v", keys)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// end of setup.
+
+	// partitionID in use: 6, 31, 57, 133, 185, 235
+	targetPartition := 31  // which partitionID we mess with.
+	targetNode := nodes[0] // this is the first replica.
+	targetIndex := index[0]
+	// 0 first replica
+	// 1 second replica
+	// 2  -- not a replica
+	// 3 primary
+
+	cfg := &FsckConfig{
+		Fix:    false,
+		FixCol: false,
+		Quiet:  true,
+		//Verbose:  true,
+		ReplicaN:        nReplicas,
+		Dirs:            dirs,
+		ParallelReaders: 5,
+	}
+	panicOn(cfg.ValidateConfig())
+
+	// for this test, mess up a replica that is not the primary.
+
+	h := targetNode.API.Holder()
+	idx[0] = h.Index(index[0])
+	store := idx[0].TranslateStore(targetPartition)
+	fwd, rev := getFwdRev(store, targetPartition)
+	//vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev)
+
+	// # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001
+	presz := len(rev)
+	delete(rev, fwd["col5"])
+	postsz := len(rev)
+
+	if postsz == presz {
+		panic("did not delete any key!")
+	}
+
+	bolt := store.(*boltdb.TranslateStore)
+	//vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path))
+	//bolt.DumpBolt("pre-corruption")
+
+	if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil {
+		t.Fatal(err)
+	}
+	//vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path))
+	//bolt.DumpBolt("post-corruption")
+
+	//fwd3, rev3 := getFwdRev(store, targetPartition)
+	//vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3)
+
+	targetIndex1 := "morty"
+	targetPartition1 := 226 // for "col11"
+	// # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}'
+	//# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}'
+	idx[1] = h.Index(index[1])
+	store1 := idx[1].TranslateStore(targetPartition1)
+	fwd1, rev1 := getFwdRev(store1, targetPartition1)
+	//vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1)
+
+	presz1 := len(rev1)
+	delete(rev1, fwd1["col11"])
+	postsz1 := len(rev1)
+
+	if postsz1 == presz1 {
+		panic("did not delete any key!")
+	}
+	bolt1 := store1.(*boltdb.TranslateStore)
+	if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil {
+		t.Fatal(err)
+	}
+
+	// done corrupting.
+	for _, nd := range nodes {
+		nd.Command.Close()
+	}
+	//panicOn(bolt.Open())
+	//bolt.DumpBolt("post-corruption, after Close. bolt:")
+	//bolt.Close()
+
+	//chksums := getChecksums(dirs, cfg, targetPartition)
+	//vv("post corruption, pre repair chksums = '%#v'", chksums)
+
+	// first we check that the corruption can be detected
+	// by our test with the checksums.
+
+	chk, err := check(dirs, cfg, targetIndex, targetPartition)
+	_ = chk
+	//vv("pre-fix, chk='%v'; err='%v'", chk, err)
+
+	if err == nil {
+		panic("expected to see checksums not match! but no corruption detected.")
+	}
+
+	chk1, err := check(dirs, cfg, targetIndex1, targetPartition1)
+	_ = chk1
+	//vv("pre-fix, chk1='%v'; err='%v'", chk1, err)
+
+	if err == nil {
+		panic("expected to see checksums not match! but no corruption detected.")
+	}
+
+	// b) running in reporting mode only should report that a fix is needed.
+	fixNeeded, err := cfg.Run()
+	panicOn(err)
+	if !fixNeeded {
+		panic("fix should be needed now, before repair")
+	}
+
+	// c) run the fix.
+	cfg.Fix = true
+	cfg.FixCol = true
+
+	fixNeeded, err = cfg.Run()
+	panicOn(err)
+	if !fixNeeded {
+		panic("fix should be marked needed if repair was made")
+	}
+
+	// d) check that the replicas all look like the primary.
+
+	//chksums = getChecksums(dirs, cfg, targetPartition)
+	//vv("after repair chksums = '%#v'", chksums)
+
+	chk, err = check(dirs, cfg, targetIndex, targetPartition)
+	_ = chk
+	//vv("chk = '%v' after repair; err='%v'", chk, err)
+	panicOn(err)
+
+	chk1, err = check(dirs, cfg, targetIndex1, targetPartition1)
+	_ = chk1
+	//vv("chk = '%v' after repair; err='%v'", chk, err)
+	panicOn(err)
+
+	// e) run again, should see no fix needed.
+	fixNeeded, err = cfg.Run()
+	panicOn(err)
+	if fixNeeded {
+		panic("should see no fix needed after the prior repair")
+	}
+}
+
+func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) {
+	fwd = make(map[string]uint64)
+	rev = make(map[uint64]string)
+	_ = store.KeyWalker(func(key string, col uint64) {
+		//vv("partition %v, key '%v' -> %x", partitionID, key, col)
+		fwd[key] = col
+	})
+	_ = store.IDWalker(func(key string, col uint64) {
+		//vv("partition %v, id %x -> '%v'", partitionID, col, key)
+		rev[col] = key
+	})
+	return
+}
+
+func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) {
+	//vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition)
+	//defer vv("returning from check()")
+
+	firstChecksum := ""
+	firstDir := ""
+	firstStorePath := ""
+	quiet := cfg.Quiet
+	defer func() {
+		cfg.Quiet = quiet
+	}()
+	cfg.Quiet = true
+	for i := range dirs {
+		dir := dirs[i]
+		_, _, ats, err := cfg.readOneDir(dir)
+		panicOn(err)
+		indexes := indexesFromAts(ats)
+		//vv("indexes = '%#v'", indexes)
+
+		for _, index := range indexes {
+
+			if index != targetIndex {
+				continue
+			}
+			for _, s := range ats.Sums {
+				//vv(" s= '%#v'", s)
+				if s.Index != index {
+					//vv("skipping s.Index '%v' != index '%v'", s.Index, index)
+					continue
+				}
+				if s.PartitionID != targetPartition {
+					continue
+				}
+				//vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+
+				//"index '%v';  s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'",
+				//s.PartitionID, targetPartition, s.Index, index,
+				//s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum)
+
+				if s.IsPrimary || s.IsReplica {
+					chksum := s.Checksum
+					if firstChecksum == "" {
+
+						firstChecksum = chksum
+						firstDir = dir
+						firstStorePath = s.StorePath
+
+					} else {
+						//vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum)
+
+						if chksum != firstChecksum {
+							return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath)
+						}
+					}
+				}
+			}
+		}
+	}
+	return firstChecksum, nil
+}
+
+var _ = getChecksums
+
+func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) {
+
+	for i := range dirs {
+		dir := dirs[i]
+		_, _, ats, err := cfg.readOneDir(dir)
+		panicOn(err)
+
+		for _, s := range ats.Sums {
+			if s.PartitionID != targetPartition {
+				continue
+			}
+			chksum = append(chksum, s.Checksum)
+		}
+	}
+	return
+}
+
+/* on shardwidth 20
+# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001
+# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2'
+# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001
+# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5'
+# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001
+# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10'
+# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001
+# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7'
+# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001
+# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3'
+# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001
+# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9'
+*/
+
+var _ = fileChecksum
+
+func fileChecksum(path string) string {
+	by, err := ioutil.ReadFile(path)
+	panicOn(err)
+	return hash.Blake3sum16(by)
+}

cmd/pilosa-fsck/release-pilosa-fsck/.gitignore

@@ -0,0 +1 @@
+pilosa-fsck

cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md

@@ -0,0 +1,252 @@
+Design for pilosa-fsck
+======================
+
+Problem Background
+------------------
+
+Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster.
+
+Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data,
+and Row-Key data are replicated. Only the first two, Roaring data and Column-Key
+data are relevant here. Broadly, the Roaring bitmap data
+forms the central features -- the bits -- of a large, sparse bitmap matrix.
+The Column-Keys are the labels for the columns at the top margin of this matrix.
+
+For speed, the Roaring bitmap data is stored separately from the
+Key data. The Roaring data is stored in sharded files
+within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/...
+The Key translation data is stored in sharded BoltDB databases within
+the PILOSA-DATA-DIR/index_name/_key directory.
+
+The current approach to Roaring file replication involves an
+eventually consistent mechanism that uses an Anti-Entropy agent to
+fix partial or incomplete replication from the primary shard to all
+replica shards.
+
+Unfortunately, the Anti-Entropy agent approach has proved inadequate on two
+fronts. First, it does not provide for immediately consistent reads in the
+event that the primary is lost. Second, the Anti-Entropy agent itself experienced
+out-of-memory issues that have yet to be resolved.
+
+Therefore, work is now underway to replace this replication
+approach with a more consistent design. 
+
+However, in the meantime, for our customers in production with Molecula
+Pilosa, we wish to provide a means to re-establish correct replication.
+Thus even in the event of a node failure followed by a read from a replica, the
+returned read will be correct.
+
+The pilosa-fsck tool can therefore be seen as a temporary, stop-gap
+measure to address immediate issues while the cluster replication
+mechanism is replaced.
+
+The second factor motivating the creation of pilosa-fsck was the discovery
+of a bug in the Key-translation process. Unfortunately this was a hard
+to reproduce bug. It happened only on the customer's premises,
+and only after running the system for a long time, with a
+large amount of data, and with various eccentric node failures
+and recoveries.
+
+However, we were able to reproduce a plausible explanation.
+Non-primary replicas were creating keys when they should have been
+forwarding the request to the primary. Correcting this bug is impetus
+for the v2.1.4 release of Molecula Pilosa.
+
+A fine point here: since we were not able to precisely reproduce the customer's
+issue in the development environment, we cannot guarantee with 100%
+certainty that we have actually addressed the bug that the customer
+was seeing.
+
+Therefore we also desired an additional insurance
+policy. We wished to be able to empower customers to proactively discover any
+future Key-translation issues that happen in their on-premise systems.
+
+To do this, we proposed providing select customers with the pilosa-fsck
+tool which can analyze their offline backups for issues.
+
+Optionally, these issues can also be repaired in-place in the
+offline backup on which pilosa-fsck is run.
+
+The -fix flag repairs both kinds of replication issues.
+
+Solution Approach: mechanism of action
+--------------------------------------
+
+The pilosa-fsck is run offline on a full set of backups taken from
+all nodes in a Pilosa cluster. It runs on a single computer that
+must be separate from the production or staging Pilosa environments.
+
+When run, pilosa-fsck analyzes the differences between the
+primary and its replicas. Both the Roaring
+files and the Key translation databases are analyzed.
+The computer running pilosa-fsck must have the same or more
+memory as the Pilosa nodes in the cluster, as it will
+"pretend" to be each Pilosa node in turn. However, as each
+node's backup is closed before the next node's backup is
+opened, we do not require substantially more memory than a single
+production node. Short Blake3 cryptographic checksums are
+computed for each Roaring fragment and each Key translation
+database. These are held in memory (and printed to the log)
+for comparing nodes. This comparison forms the heart of
+the consistency checks, and is the basis for any subsequent
+repair.
+
+We recommend capturing both stdout and stderr to a log.
+Use `&> log` or  `2>&1 > log` at the end of the
+pilosa-fsck invocation to save a log of the run to disk.
+
+In a typical cluster, the Replication factor R may be less
+than the number of nodes N in the cluster. For example, while
+N may be 4, the R may be only 3. In this example, within
+each replicated shard, one node will be the primary for
+that shard, two nodes will be non-primary replicas, and one
+node will be a non-replica. Note that the designation
+of primary changes for different Roaring shards within an index,
+even on a single node.
+
+The essence of the the -fix repair operation that pilosa-fsck
+can do is this: it will copy from the primary to the
+the non-primary replicas. Further, it will remove data from
+any non-replica node if it was mistakenly present.
+
+The pilosa-fsck output log will contain
+a sequence of command line 'cp' and 'rm' commands.
+These commands are merely a record (with
+accompanying justifcation in the comment following the
+command) of what actions would be performed to repair
+the Roaring file data.
+
+Only with -fix will the repair actions actually happen
+during the pilosa-fsck run.
+
+
+Details: running pilosa-fsck
+----------------------------
+
+Errors in invocation are reported on stderr and the program will exit with a non-zero
+error code if invocation errors are present. A non-zero error code
+is returned if a repair is needed and -fix was not given.
+
+A -fix run will return a zero error code to the shell if the fix was
+successfully made; or if no fix was required.
+
+The log of the run is printed to stdout.
+
+The -h flag to pilosa-fsck prints a summary of its operation
+and a guide to laying out the backup directories.
+
+The help is reproduced below.
+
+~~~
+$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct  5 2020 1:28PM, 9dacbccf)
+
+Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
+
+  -fix
+    	(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
+
+  -replicas R 
+        (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
+        the number of replicas maintained in the cluster. Must be the same as the 
+        [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
+
+  -q
+        be very quiet during analysis and repair
+
+
+Welcome to pilosa-fsck. This is a scan and repair 
+tool that is modeled after the classic unix file 
+system utility fsck. 
+
+WARNING: DO NOT RUN ON A LIVE SYSTEM.
+
+The most important point to remember is that analysis
+and repair must be done *offline*.
+
+Just as fsck must be run on an unmounted disk, 
+pilosa-fsck must be run on a backup. It must 
+not be run on the directories where a live Pilosa system
+is serving queries. Instead, take a backup first.
+A backup is a set of N Pilosa data directories that have been
+copied from your live system. They must all 
+be visible and mounted on one filesystem together.
+
+pilosa-fsck can be run in scan-mode (without -fix),
+or in repair-mode with -fix. The console output
+supplies a log documenting the analysis 
+and showing what data changes would have been made.
+
+REQUIRED COMMAND LINE ARGUMENTS
+
+The paths to all the top-level Pilosa 
+data directories in a cluster must be given on the command
+line. The -replicas R flag is also always required. It
+must be correct for your cluser. Here R is the same as
+the [cluster] stanza "replicas = R" line from your
+pilosa.conf.
+
+Example:
+
+Suppose you are ready to run pilosa-fsck: 
+you have taken a backup of your four node Pilosa 
+cluster and stored it all on one filesystem with 
+all nodes visible and uncompressed. This
+is a pre-requisite to running pilosa-fsck. 
+Let's suppose we have replication R = 3 set.
+In this example, have stored our backed-up directories in 
+
+/backup/molecula
+
+and the four node backups are in 
+subdirectories node1/ node2/ node3/ node4/ under this:
+
+/backup/molecula/node1/
+/backup/molecula/node1/.pilosa/.id
+/backup/molecula/node1/.pilosa/.topology
+/backup/molecula/node1/.pilosa/myindex
+
+/backup/molecula/node2/
+/backup/molecula/node2/.pilosa/.id
+/backup/molecula/node2/.pilosa/.topology
+/backup/molecula/node2/.pilosa/myindex
+
+/backup/molecula/node3/
+/backup/molecula/node3/.pilosa/.id
+/backup/molecula/node3/.pilosa/.topology
+/backup/molecula/node3/.pilosa/myindex
+
+/backup/molecula/node4/
+/backup/molecula/node4/.pilosa/.id
+/backup/molecula/node4/.pilosa/.topology
+/backup/molecula/node4/.pilosa/myindex
+
+NOTE: your .pilosa directories need not be named .pilosa. They can
+be something else, such as when the -d flag to pilosa server was used.
+The .id file, the .topology file, and the index directories must be 
+found directly underneath.
+
+Then a typical invocation to scan a cluster backup for issues:
+
+$ cd /backup/molecula/
+$ pilosa-fsck -replicas 3  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+A typical invocation to repair the replication in the same backup:
+
+$ pilosa-fsck -replicas 3 -fix  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+In both cases, the .id and .topology files must 
+be present in the backups.
+
+Without -fix, no modifications will be made to the backups. Only
+by running with -fix will repairs be made. The user can safely
+always run with -fix to repair only if needed.
+
+A zero error code will be returned to the shell if no repairs were needed.
+
+A zero error code will be also be returned to the shell if 
+repairs were needed and they were accomplished under -fix.
+
+A non-zero error code indicates that repairs were needed but
+were not made.
+
+~~~

cmd/pilosa-fsck/release-pilosa-fsck/example.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set +x
+export PATH=.:${PATH}
+
+# unpack the sample Molecula Pilosa cluster.
+tar xf backups.tar.gz
+
+
+# check if repair is needed.
+pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
+
+
+# yes, so do the repairs. This can be done first (only) as well.
+#
+pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
+
+
+# check again if you like
+#
+pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa

cmd/pilosa-fsck/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("# %s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) int64 {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return 0
+	}
+	return fi.Size()
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/random-query/main.go

@@ -0,0 +1,377 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	nethttp "net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+)
+
+// RandomQueryConfig
+type RandomQueryConfig struct {
+
+	// user facing flags
+	HostPort   string // -hostport
+	TreeDepth  int    // -d
+	QueryCount int    // -n
+	Verbose    bool   // -v
+
+	IndexMap map[string]*Features
+
+	API  *pilosa.API
+	Info []*pilosa.IndexInfo
+
+	BitmapFunc []string
+
+	Rnd *rand.Rand
+}
+
+type API interface {
+
+	// InternalClient
+	Schema(ctx context.Context) ([]*pilosa.IndexInfo, error)
+	Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error)
+
+	// API for contrast; just a little different:
+	//Schema(ctx context.Context) []*IndexInfo
+	//Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error)
+}
+
+// have to wrap because the ugly little differences between InternalClient and API
+type wrapper struct {
+	api *pilosa.API
+}
+
+func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	return w.api.Schema(ctx), nil
+}
+
+func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	r, err := w.api.Query(ctx, queryRequest)
+	return &r, err
+}
+
+func wrapApiToInternalClient(api *pilosa.API) *wrapper {
+	return &wrapper{api: api}
+}
+
+// call DefineFlags before myflags.Parse()
+func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
+	fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
+	fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
+	fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *RandomQueryConfig) ValidateConfig() error {
+	if c.TreeDepth < 1 {
+		return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth)
+	}
+	if c.QueryCount < 0 {
+		return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
+	}
+	return nil
+}
+
+var ProgramName = "random-query"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
+	cfg := NewRandomQueryConfig()
+	cfg.DefineFlags(myflags)
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+
+	err = cfg.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *RandomQueryConfig) Run() (err error) {
+	remoteClient := nethttp.DefaultClient
+	cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	totalQ := 0
+	loops := 0
+	t0 := time.Now()
+
+NewSetup:
+	err = cfg.Setup(cli)
+	if err != nil {
+		return err
+	}
+
+	if len(cfg.IndexMap) == 0 {
+		return fmt.Errorf("no rows to query")
+	}
+
+	var indexes []string
+	for index := range cfg.IndexMap {
+		indexes = append(indexes, index)
+	}
+
+	for j := 0; ; j++ {
+		if cfg.QueryCount > 0 {
+			if j >= cfg.QueryCount {
+				break
+			}
+		} else {
+			// else keep doing queries forever...
+			if loops > 0 && loops%500 == 0 {
+				// ...but account for any new data arrived by getting
+				// the schema and rows again every so often.
+				loops++
+				goto NewSetup
+			}
+		}
+		if totalQ%100 == 0 {
+			dur := time.Since(t0)
+			if dur > 0 {
+				qps := 1e9 * float64(totalQ) / float64(dur)
+				AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: %0.02f", totalQ, dur, qps)
+			}
+		}
+
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		if cfg.Verbose {
+			fmt.Printf("pql = '%v'\n", pql)
+		}
+
+		// Query node0.
+		res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
+			return err
+		}
+		if cfg.Verbose {
+			fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+		}
+		totalQ++
+		loops++
+	}
+
+	return nil
+}
+
+type Features struct {
+	Slc []IndexFieldRow
+}
+
+func NewRandomQueryConfig() *RandomQueryConfig {
+	return &RandomQueryConfig{
+		IndexMap: make(map[string]*Features),
+	}
+}
+
+type IndexFieldRow struct {
+	Index    string
+	Field    string
+	RowID    uint64
+	RowKey   string
+	IsRowKey bool
+}
+
+// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
+// and spits back a PQL query
+//
+func (cfg *RandomQueryConfig) Setup(api API) (err error) {
+
+	ctx := context.Background()
+	cfg.Info, err = api.Schema(ctx)
+	if err != nil {
+		return err
+	}
+	for i, ii := range cfg.Info {
+		_ = i
+		for k, fld := range ii.Fields {
+			_ = k
+			if fld.Options.Type == "set" {
+				pql := fmt.Sprintf("Rows(%v)", fld.Name)
+
+				res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
+				panicOn(err)
+				if cfg.Verbose {
+					fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+				}
+				// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
+				// e.g.
+				// success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"}
+				// success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}'
+
+				switch x := res.Results[0].(type) {
+				case *pilosa.RowIdentifiers:
+					// internalClient gets this
+					cfg.AddResponse(ii.Name, fld.Name, x)
+				case pilosa.RowIdentifiers:
+					// test gets this
+					cfg.AddResponse(ii.Name, fld.Name, &x)
+				}
+			}
+		}
+	}
+
+	cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
+	seed := int64(42)
+	cfg.Rnd = rand.New(rand.NewSource(seed))
+
+	return nil
+}
+
+func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers) {
+	for _, rowID := range x.Rows {
+		cfg.AddFeature(index, field, rowID, "", false)
+	}
+	for _, rowKey := range x.Keys {
+		cfg.AddFeature(index, field, 0, rowKey, true)
+	}
+}
+
+func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
+
+	tree := cfg.GenTree(index, cfg.TreeDepth)
+
+	if cfg.Verbose {
+		fmt.Printf("%v\n", tree.StringIndent(0))
+	}
+	pql = tree.ToPQL()
+
+	// avoid using too much bandwidth, just count the final bitmap.
+	pql = fmt.Sprintf("Count(%v)", pql)
+	return
+}
+
+type Tree struct {
+	Chd []*Tree
+
+	S string
+}
+
+func (tr *Tree) StringIndent(ind int) (s string) {
+	spc := strings.Repeat("    ", ind)
+	spc1 := strings.Repeat("    ", ind+1)
+	var chds []string
+	leaf := true
+	if len(tr.Chd) == 0 {
+		// leaf
+	} else {
+		leaf = false
+		for _, chd := range tr.Chd {
+			chds = append(chds, chd.StringIndent(ind+1))
+		}
+	}
+	if leaf {
+		s += fmt.Sprintf("%v %v\n", spc1, tr.S)
+	} else {
+		for i, c := range chds {
+			if i == 0 {
+				s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c)
+			} else {
+				s += fmt.Sprintf("%v", c)
+			}
+		}
+	}
+	return
+}
+
+func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
+	if depth == 0 {
+		slc := cfg.IndexMap[index].Slc
+		//vv("depth is 0, slc = '%#v'", slc)
+		r := cfg.Rnd.Intn(len(slc))
+		fea := slc[r]
+		if fea.IsRowKey {
+			return &Tree{S: fmt.Sprintf("Row(%v='%v')", fea.Field, fea.RowKey)}
+		}
+		return &Tree{S: fmt.Sprintf("Row(%v=%v)", fea.Field, fea.RowID)}
+	}
+
+	r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
+	f := cfg.BitmapFunc[r]
+	tr = &Tree{S: f}
+	numChild := 2
+	switch f {
+	case "Union", "Intersect", "Xor":
+		numChild = cfg.Rnd.Intn(8) + 2
+	case "Not":
+		numChild = 1
+	case "Difference":
+		numChild = 2
+	}
+	for i := 0; i < numChild; i++ {
+		tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
+	}
+	return
+}
+
+func (tr *Tree) ToPQL() (s string) {
+
+	if len(tr.Chd) == 0 {
+		// leaf
+		return tr.S
+	}
+
+	var chds []string
+	for _, c := range tr.Chd {
+		chds = append(chds, c.ToPQL())
+	}
+	all := strings.Join(chds, ", ")
+	return fmt.Sprintf("%v(%v)", tr.S, all)
+}
+
+func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool) {
+
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	f.Slc = append(f.Slc, IndexFieldRow{
+		Index:    index,
+		Field:    field,
+		RowID:    rowID,
+		RowKey:   rowKey,
+		IsRowKey: isRowKey,
+	})
+}

cmd/random-query/main_test.go

@@ -0,0 +1,165 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"math/rand"
+	"strconv"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_RandomQuery(t *testing.T) {
+
+	cfg := NewRandomQueryConfig()
+
+	nNodes := 1
+	nReplicas := 1
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	defer c.Close()
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+	_ = dirs
+
+	ctx := context.Background()
+
+	indexes := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(indexes))
+	field := make([]*pilosa.Field, len(indexes))
+
+	var err error
+
+	for i := range indexes {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	timestamp := int64(0)
+
+	for i := range indexes {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, uint64(j))
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexes[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+		//vv("rowIDs = '%#v'", rowIDs)
+		//vv("colKeys = '%#v'", colKeys)
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+		//qcx.Reset()
+	}
+	// end of setup.
+
+	panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
+
+	for j := 0; j < 4; j++ {
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		//vv("pql = '%v'", pql)
+
+		// Query node0.
+		res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = res
+		//vv("success on pql = '%v'; res='%v'", pql, res.Results[0])
+	}
+}

cmd/random-query/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("# %s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) int64 {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return 0
+	}
+	return fi.Size()
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

ctl/server.go

@@ -47,6 +47,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
 	flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
+	flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.")
 
 	// Translation
 	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")

dbshard.go

@@ -320,6 +320,9 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) {
 			}
 		}
 	}
+	// allow the index to be created again anew.
+	delete(per.dbh.Index, index)
+
 	return
 }
 
@@ -436,7 +439,7 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
 		if len(per.types) == 1 && per.types[0] == roaringTxn {
 			// roaring txn are nil/fake anyway. Don't freak out.
 		} else {
-			panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs))
+			panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types)))
 		}
 	}
 	if !ok {

executor.go

@@ -1074,14 +1074,16 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct")
 	defer span.Finish()
 
-	field := c.Args["field"]
-	if field == "" {
-		return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name)
+	field, hasField, err := c.StringArg("field")
+	if err != nil {
+		return SignedRow{}, errors.Wrap(err, "loading field option in Distinct query")
+	} else if !hasField {
+		return SignedRow{}, fmt.Errorf("missing field option in Distinct query")
 	}
 
 	// Execute calls in bulk on each remote node and merge.
 	mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) {
-		return e.executeDistinctShard(ctx, qcx, index, c, shard)
+		return e.executeDistinctShard(ctx, qcx, index, field, c, shard)
 	}
 
 	// Merge returned results at coordinating node.
@@ -1098,7 +1100,7 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string,
 		return SignedRow{}, err
 	}
 	other, _ := result.(SignedRow)
-	other.field = field.(string)
+	other.field = field
 
 	return other, nil
 }
@@ -1385,11 +1387,15 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s
 
 // executeDistinctShard executes a Distinct call on a single shard, yielding
 // a SignedRow of the values found.
-func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) {
+func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result SignedRow, err error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard")
 	defer span.Finish()
 
 	idx := e.Holder.Index(index)
+	field := e.Holder.Field(index, fieldName)
+	if field == nil {
+		return SignedRow{}, ErrFieldNotFound
+	}
 
 	var filter *Row
 	var filterBitmap *roaring.Bitmap
@@ -1406,13 +1412,6 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str
 		}
 	}
 
-	fieldName, _ := c.Args["field"].(string)
-
-	field := e.Holder.Field(index, fieldName)
-	if field == nil {
-		return result, nil
-	}
-
 	bsig := field.bsiGroup(fieldName)
 	if bsig == nil {
 		return result, nil
@@ -1425,7 +1424,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str
 	tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
 	defer finisher(&err)
 
-	existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1)
+	existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1)
 	if err != nil {
 		return result, err
 	}
@@ -1436,7 +1435,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str
 		return result, nil
 	}
 
-	signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2)
+	signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2)
 	if err != nil {
 		return result, nil
 	}
@@ -1444,7 +1443,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str
 	dataBitmaps := make([]*roaring.Bitmap, depth)
 
 	for i := uint64(0); i < depth; i++ {
-		dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3))
+		dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*(i+2), ShardWidth*(i+3))
 		if err != nil {
 			return result, err
 		}
@@ -2593,10 +2592,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string,
 	// in order to represent `Rows` for the field.
 	var views = []string{viewStandard}
 
-	// Handle `int` and `time` fields.
 	switch f.Type() {
-	case FieldTypeInt:
-		return nil, errors.New("int fields not supported by Rows() query")
+	case FieldTypeSet, FieldTypeMutex:
 	case FieldTypeTime:
 		var err error
 
@@ -2658,6 +2655,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string,
 			// Determine the views based on the specified time range.
 			views = viewsByTimeRange(viewStandard, fromTime, toTime, q)
 		}
+	default:
+		return nil, errors.Errorf("%s fields not supported by Rows() query", f.Type())
 	}
 
 	start := uint64(0)

executor_test.go

@@ -4851,6 +4851,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
 	defer c.Close()
 	c.CreateField(t, "i", pilosa.IndexOptions{}, "general")
 	c.CreateField(t, "i", pilosa.IndexOptions{}, "integer", pilosa.OptFieldTypeInt(-1000, 1000))
+	c.CreateField(t, "i", pilosa.IndexOptions{}, "decimal", pilosa.OptFieldTypeDecimal(2))
+	c.CreateField(t, "i", pilosa.IndexOptions{}, "bool", pilosa.OptFieldTypeBool())
 
 	tests := []struct {
 		query string
@@ -4884,6 +4886,18 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
 			query: "GroupBy(Rows(integer), prev=-1)",
 			error: "unknown arg 'prev'",
 		},
+		{
+			query: "Rows(integer)",
+			error: "int fields not supported by Rows() query",
+		},
+		{
+			query: "Rows(decimal)",
+			error: "decimal fields not supported by Rows() query",
+		},
+		{
+			query: "Rows(bool)",
+			error: "bool fields not supported by Rows() query",
+		},
 	}
 
 	for i, test := range tests {
@@ -5293,7 +5307,7 @@ func TestExecutor_ForeignIndex(t *testing.T) {
 	)
 
 	// Populate parent data.
-	c.Query(t, "parent", `
+	c.Query(t, "parent", fmt.Sprintf(`
 			Set("one", general=1)
 			Set("two", general=1)
 			Set("three", general=1)
@@ -5302,25 +5316,25 @@ func TestExecutor_ForeignIndex(t *testing.T) {
 			Set("twenty-two", general=2)
 			Set("twenty-three", general=2)
 
-			Set("one", general=3)
-			Set("twenty-one", general=3)
-		`)
+			Set("one", general=%d)
+			Set("twenty-one", general=%d)
+		`, ShardWidth, ShardWidth))
 
 	// Populate child data.
-	c.Query(t, "child", `
+	c.Query(t, "child", fmt.Sprintf(`
 			Set(1, parent_id="one")
 			Set(2, parent_id="two")
-			Set(3, parent_id="one")
+			Set(%d, parent_id="one")
 			Set(4, parent_id="twenty-one")
-		`)
+		`, ShardWidth))
 
 	// Populate color data.
-	c.Query(t, "child", `
+	c.Query(t, "child", fmt.Sprintf(`
 			Set(1, color="red")
 			Set(2, color="blue")
-			Set(3, color="blue")
+			Set(%d, color="blue")
 			Set(4, color="red")
-		`)
+		`, ShardWidth))
 
 	distinct := c.Query(t, "child", `Distinct(index="child", field="parent_id")`).Results[0].(pilosa.SignedRow)
 	if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) {
@@ -5328,7 +5342,7 @@ func TestExecutor_ForeignIndex(t *testing.T) {
 	}
 
 	eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row)
-	if !reflect.DeepEqual(eq.Columns(), []uint64{1, 3}) {
+	if !reflect.DeepEqual(eq.Columns(), []uint64{1, ShardWidth}) {
 		t.Fatalf("unexpected columns: %v", eq.Columns())
 	}
 
@@ -5337,7 +5351,7 @@ func TestExecutor_ForeignIndex(t *testing.T) {
 		t.Fatalf("unexpected columns: %v", neq.Columns())
 	}
 
-	join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row)
+	join := c.Query(t, "parent", fmt.Sprintf(`Intersect(Row(general=%d), Distinct(Row(color="blue"), index="child", field="parent_id"))`, ShardWidth)).Results[0].(*pilosa.Row)
 	if !reflect.DeepEqual(join.Keys, []string{"one"}) {
 		t.Fatalf("unexpected keys: %v", join.Keys)
 	}

fragment.go

@@ -798,26 +798,28 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
 
 	// From the given row, get the rowSegment for this shard.
 	seg := row.segment(f.shard)
-	if seg == nil {
-		return changed, nil
-	}
-
-	// Put each container from rowSegment to fragment storage.
-	citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent)
-	for citer.Next() {
-		k, c := citer.Value()
-		if err := tx.PutContainer(f.index, f.field, f.view, f.shard, headContainerKey+(k%(1<<shardVsContainerExponent)), c); err != nil {
-			return changed, err
+	if seg != nil {
+		// Put each container from rowSegment to fragment storage.
+		citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent)
+		for citer.Next() {
+			k, c := citer.Value()
+			if err := tx.PutContainer(f.index, f.field, f.view, f.shard, headContainerKey+(k%(1<<shardVsContainerExponent)), c); err != nil {
+				return changed, err
+			}
 		}
-	}
 
-	// Update the row in cache.
-	if f.CacheType != CacheTypeNone {
-		n, err := tx.CountRange(f.index, f.field, f.view, f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth)
-		if err != nil {
-			return changed, err
+		// Update the row in cache.
+		if f.CacheType != CacheTypeNone {
+			n, err := tx.CountRange(f.index, f.field, f.view, f.shard, rowID*ShardWidth, (rowID+1)*ShardWidth)
+			if err != nil {
+				return changed, err
+			}
+			f.cache.BulkAdd(rowID, n)
+		}
+	} else {
+		if f.CacheType != CacheTypeNone {
+			f.cache.BulkAdd(rowID, 0)
 		}
-		f.cache.BulkAdd(rowID, n)
 	}
 
 	// invalidate rowCache for this row.

fragment_internal_test.go

@@ -33,13 +33,12 @@ import (
 	"testing"
 	"testing/quick"
 
-	"golang.org/x/sync/errgroup"
-
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 // Test flags
@@ -268,6 +267,19 @@ func TestFragment_SetRow(t *testing.T) {
 	} else if n := f.mustRow(tx, rowID).Count(); n != 3 {
 		t.Fatalf("unexpected count (reopen): %d", n)
 	}
+
+	// verify that setting something from a row which lacks a segment for
+	// this fragment's shard still clears this fragment correctly.
+	notOurs := NewRow(8*ShardWidth + 1024)
+	if changed, err := f.unprotectedSetRow(tx, notOurs, rowID); err != nil {
+		t.Fatal(err)
+	} else if !changed {
+		t.Fatalf("setRow didn't report a change")
+	}
+	if cols := f.mustRow(tx, rowID).Columns(); len(cols) != 0 {
+		t.Fatalf("expected setting a row with no entries to clear the cache")
+	}
+	panicOn(tx.Commit())
 }
 
 // Ensure a fragment can set & read a value.
@@ -2853,7 +2865,7 @@ func BenchmarkImportRoaring(b *testing.B) {
 						// care whether this succeeds,
 						// but if it's happening we want
 						// it to be done.
-						_ = defaultSnapshotQueue.Await(f)
+						_ = f.holder.SnapshotQueue.Await(f)
 						f.Clean(b)
 						b.Fatalf("import error: %v", err)
 					}
@@ -2897,7 +2909,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 								err := frags[j].importRoaringT(txs[j], data[j], false)
 								// error unimportant if it happened, but we want
 								// any snapshots to have finished.
-								_ = defaultSnapshotQueue.Await(frags[j])
+								_ = frags[j].holder.SnapshotQueue.Await(frags[j])
 								return err
 							})
 						}
@@ -2942,7 +2954,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
 								}
-								err = defaultSnapshotQueue.Immediate(frags[j])
+								err = frags[j].holder.SnapshotQueue.Immediate(frags[j])
 								if err != nil {
 									b.Fatalf("snapshot after import: %v", err)
 								}
@@ -2955,7 +2967,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 									defer txs[j].Rollback()
 
 									err := frags[j].importRoaringT(txs[j], updata, false)
-									err2 := defaultSnapshotQueue.Await(frags[j])
+									err2 := frags[j].holder.SnapshotQueue.Await(frags[j])
 									if err == nil {
 										err = err2
 									}
@@ -3029,7 +3041,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
-						err = defaultSnapshotQueue.Immediate(f)
+						err = f.holder.SnapshotQueue.Immediate(f)
 						if err != nil {
 							b.Errorf("snapshot after import error: %v", err)
 						}
@@ -3039,7 +3051,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 							f.Clean(b)
 							b.Errorf("import error: %v", err)
 						}
-						err = defaultSnapshotQueue.Await(f)
+						err = f.holder.SnapshotQueue.Await(f)
 						if err != nil {
 							b.Errorf("snapshot after import error: %v", err)
 						}
@@ -3402,7 +3414,7 @@ func (f *fragment) Clean(t testing.TB) {
 		// badger doesn't need snapshot, so this stuff is skipped.
 		// The snapshot queue stuff doesn't work under badger.
 		if f.idx.NeedsSnapshot() {
-			err := defaultSnapshotQueue.Await(f)
+			err := f.holder.SnapshotQueue.Await(f)
 			if err != nil {
 				t.Fatalf("snapshot failed before sanity check: %v", err)
 			}
@@ -4228,7 +4240,7 @@ func TestUnionInPlaceMapped(t *testing.T) {
 	// it's used only in computation of things that usually don't go to
 	// disk, which is why we handle this specially in testing and not
 	// generically.
-	err = defaultSnapshotQueue.Immediate(f)
+	err = f.holder.SnapshotQueue.Immediate(f)
 	if err != nil {
 		t.Fatalf("snapshot after union-in-place: %v", err)
 	}

go.mod

@@ -12,6 +12,8 @@ require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
+	github.com/dustin/go-humanize v1.0.0
+	github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
 	github.com/glycerine/lmdb-go v1.9.32
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.1

holder.go

@@ -1218,7 +1218,7 @@ func (h *Holder) setFileLimit() {
 	}
 }
 
-func (h *Holder) loadNodeID() (string, error) {
+func (h *Holder) LoadNodeID() (string, error) {
 	idPath := path.Join(h.path, ".id")
 	h.Logger.Printf("load NodeID: %s", idPath)
 	if err := os.MkdirAll(h.path, 0777); err != nil {
@@ -1227,7 +1227,9 @@ func (h *Holder) loadNodeID() (string, error) {
 
 	nodeIDBytes, err := ioutil.ReadFile(idPath)
 	if err == nil {
-		return strings.TrimSpace(string(nodeIDBytes)), nil
+		nodeid := strings.TrimSpace(string(nodeIDBytes))
+		h.Logger.Printf("I am NodeID: %s", nodeid)
+		return nodeid, nil
 	}
 	if !os.IsNotExist(err) {
 		return "", errors.Wrap(err, "reading file")
@@ -1237,6 +1239,7 @@ func (h *Holder) loadNodeID() (string, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "writing file")
 	}
+	h.Logger.Printf("I am NodeID: %s", nodeID)
 	return nodeID, nil
 }
 
@@ -1275,6 +1278,8 @@ type holderSyncer struct {
 	// Translation sync handling.
 	readers []TranslateEntryReader
 
+	syncers errgroup.Group
+
 	// Stats
 	Stats stats.StatsClient
 
@@ -1569,6 +1574,7 @@ func (s *holderSyncer) stopTranslationSync() error {
 			return rd.Close()
 		})
 	}
+	g.Go(s.syncers.Wait)
 	return g.Wait()
 }
 
@@ -1658,7 +1664,11 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error {
 		}
 		s.readers = append(s.readers, rd)
 
-		go func() { defer rd.Close(); s.readIndexTranslateReader(rd) }()
+		s.syncers.Go(func() error {
+			defer rd.Close()
+			s.readIndexTranslateReader(rd)
+			return nil
+		})
 	}
 
 	return nil
@@ -1697,8 +1707,11 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error {
 	}
 	s.readers = append(s.readers, rd)
 
-	go func() { defer rd.Close(); s.readFieldTranslateReader(rd) }()
-
+	s.syncers.Go(func() error {
+		defer rd.Close()
+		s.readFieldTranslateReader(rd)
+		return nil
+	})
 	return nil
 }
 
@@ -1718,7 +1731,7 @@ func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
 		}
 
 		// Apply replication to store.
-		store := idx.TranslateStore(s.Cluster.keyPartition(entry.Index, entry.Key))
+		store := idx.TranslateStore(s.Cluster.Topology.KeyPartition(entry.Index, entry.Key))
 		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
 			s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key)
 			return

http/client_test.go

@@ -588,6 +588,42 @@ func TestClient_ImportRoaring(t *testing.T) {
 	}
 }
 
+// Ensure client can bulk import data with multiple views and not deadlock.
+func TestClient_ImportRoaring_MultiView(t *testing.T) {
+	cluster := test.MustNewCluster(t, 2)
+	for _, c := range cluster.Nodes {
+		c.Config.Cluster.ReplicaN = 2
+	}
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer cluster.Close()
+
+	_, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+	_, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+
+	// Send import request.
+	host := cluster.GetNode(0).URL()
+	c := MustNewClient(host, http.GetHTTPClient(nil))
+	req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}}
+	req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
+	req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, req); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // Ensure client can bulk import data.
 func TestClient_ImportKeys(t *testing.T) {
 	t.Run("SingleNode", func(t *testing.T) {

http/handler.go

@@ -381,8 +381,6 @@ func newRouter(handler *Handler) http.Handler {
 	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
 	router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema")
 	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
-	router.HandleFunc("/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
-	router.HandleFunc("/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
 	router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
 	router.HandleFunc("/transaction/", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
 	router.HandleFunc("/transaction/{id}", handler.handleGetTransaction).Methods("GET").Name("GetTransaction")
@@ -392,6 +390,10 @@ func newRouter(handler *Handler) http.Handler {
 	router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries")
 	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
 
+	router.HandleFunc("/ui/usage", handler.handleGetUsage).Methods("GET").Name("GetUsage")
+	router.HandleFunc("/ui/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
+	router.HandleFunc("/ui/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
+
 	// /internal endpoints are for internal use only; they may change at any time.
 	// DO NOT rely on these for external applications!
 	router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
@@ -657,6 +659,40 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// handleGetUsage handles GET /ui/usage requests.
+func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+	usageIndexes, usageTotal, err := h.api.Usage()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+
+	disk := diskUsage{
+		Total:   usageTotal,
+		Indexes: usageIndexes,
+	}
+
+	usage := getUsageResponse{
+		Disk: disk,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(usage); err != nil {
+		h.logger.Printf("write status response error: %s", err)
+	}
+}
+
+type getUsageResponse struct {
+	Disk diskUsage `json:"bytesOnDisk"`
+}
+type diskUsage struct {
+	Total   int64            `json:"total"`
+	Indexes map[string]int64 `json:"indexes"`
+}
+
 // handleGetStatus handles GET /status requests.
 func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
@@ -664,9 +700,10 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	status := getStatusResponse{
-		State:   h.api.State(),
-		Nodes:   h.api.Hosts(r.Context()),
-		LocalID: h.api.Node().ID,
+		State:       h.api.State(),
+		Nodes:       h.api.Hosts(r.Context()),
+		LocalID:     h.api.Node().ID,
+		ClusterName: h.api.ClusterName(),
 	}
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(status); err != nil {
@@ -722,9 +759,10 @@ type getSchemaResponse struct {
 }
 
 type getStatusResponse struct {
-	State   string         `json:"state"`
-	Nodes   []*pilosa.Node `json:"nodes"`
-	LocalID string         `json:"localID"`
+	State       string         `json:"state"`
+	Nodes       []*pilosa.Node `json:"nodes"`
+	LocalID     string         `json:"localID"`
+	ClusterName string         `json:"clusterName"`
 }
 
 func hash(s string) string {

index.go

@@ -26,6 +26,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/glycerine/idem"
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2/hash"
 	"github.com/pilosa/pilosa/v2/internal"
@@ -33,6 +34,7 @@ import (
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
+	"github.com/zeebo/blake3"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -734,6 +736,19 @@ func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []u
 
 type AllTranslatorSummary struct {
 	Sums []*TranslatorSummary
+
+	RepairNeeded bool
+}
+
+func (ats *AllTranslatorSummary) Checksum() string {
+	ats.Sort()
+	hasher := blake3.New()
+	for _, sum := range ats.Sums {
+		_, _ = hasher.Write([]byte(sum.Checksum))
+	}
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+	return fmt.Sprintf("blake3-%x", buf)
 }
 
 func NewAllTranslatorSummary() *AllTranslatorSummary {
@@ -741,6 +756,7 @@ func NewAllTranslatorSummary() *AllTranslatorSummary {
 }
 func (ats *AllTranslatorSummary) Append(b *AllTranslatorSummary) {
 	ats.Sums = append(ats.Sums, b.Sums...)
+	ats.RepairNeeded = ats.RepairNeeded || b.RepairNeeded
 }
 
 func (ats *AllTranslatorSummary) Sort() {
@@ -761,57 +777,244 @@ func (ats *AllTranslatorSummary) Sort() {
 		if a.PartitionID > b.PartitionID {
 			return false
 		}
-		return a.Field < b.Field
+		if a.Field < b.Field {
+			return true
+		}
+		if a.Field > b.Field {
+			return false
+		}
+		return a.NodeID < b.NodeID
 	})
 }
 
 // sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil
-func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
+func (idx *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string, parallelReaders int) (ats *AllTranslatorSummary, err error) {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
 
 	ats = &AllTranslatorSummary{}
+	var atsMu sync.Mutex
 
-	fmt.Printf("\nindex: %v\n=================\n", i.name)
-	for _, fld := range i.fields {
-		sum, err := fld.translateStore.ComputeTranslatorSummary()
-		if err != nil {
-			return ats, err
-		}
-		sum.Field = fld.name
-		sum.Index = i.Name()
-		sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
-
-		if verbose {
-			fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
-		}
-		ats.Sums = append(ats.Sums, sum)
+	if verbose {
+		fmt.Printf("\n# index: %v\n# =================\n", idx.name)
 	}
 
-	fmt.Printf("====================\n")
+	jobQ := make(chan func() error, 10000)
+	var errmu sync.Mutex
 
-	for partitionID, store := range i.translateStores {
-		sum, err := store.ComputeTranslatorSummary()
-		if err != nil {
-			return ats, err
-		}
-		if sum == nil {
-			// probably one of the Noop stores
-			continue
-		}
-		sum.PartitionID = partitionID
-		sum.Index = i.Name()
-
-		sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
-		if verbose {
-			fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID)
-		}
-		ats.Sums = append(ats.Sums, sum)
+	if parallelReaders < 1 {
+		// turn it up to 11
+		parallelReaders = 10000
 	}
+
+	halters := make([]*idem.Halter, parallelReaders)
+	for j := 0; j < parallelReaders; j++ {
+		h := idem.NewHalter()
+		halters[j] = h
+	}
+	for _, h := range halters {
+		go func(h *idem.Halter) {
+			defer h.MarkDone()
+			for {
+				select {
+				case <-h.ReqStop.Chan:
+					return
+				case f, ok := <-jobQ:
+					if !ok || f == nil {
+						// channel closed, finish up
+						return
+					}
+
+					err1 := f()
+					if err1 != nil {
+						errmu.Lock()
+						if err == nil {
+							err = err1
+						}
+						errmu.Unlock()
+						// an error occurred, tell everyone to stop
+						for _, h2 := range halters {
+							h2.RequestStop()
+						}
+						return
+					}
+				}
+			}
+		}(h)
+	}
+
+floop:
+	for _, fld := range idx.fields {
+		fld := fld
+
+		fun := func() error {
+			//vv("ComputeTranslatorSummary() on fld '%v'", fld.name)
+			sum, err := fld.translateStore.ComputeTranslatorSummaryRows()
+			if err != nil {
+				return err
+			}
+			sum.Field = fld.name
+			sum.Index = idx.Name()
+			sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, idx.Name())))
+			sum.IsColKey = false
+			if verbose {
+				fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
+			}
+			atsMu.Lock()
+			ats.Sums = append(ats.Sums, sum)
+			atsMu.Unlock()
+			return nil
+		}
+
+		select {
+		case <-halters[0].ReqStop.Chan:
+			break floop
+		case jobQ <- fun:
+		}
+	} // end floop
+
+	if verbose {
+		fmt.Printf("# ====================\n")
+	}
+
+tloop:
+	for partitionID, store := range idx.translateStores {
+		partitionID := partitionID
+		store := store
+
+		fun2 := func() error {
+			//vv("ComputeTranslatorSummary() running on store.Path = '%v'", store.GetStorePath())
+			if checkKeys {
+				prim := topo.PrimaryNodeIndex(partitionID)
+				primID := topo.nodeIDs[prim]
+
+				// note: we fix irrespective of nodeID == primID now, so that we
+				// get a fine grain report of what maps were off.
+
+				if verbose {
+					// This is pilosa-fsck output, not regular log.
+					fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID)
+				}
+				changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs)
+				if err != nil {
+					return errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()")
+				}
+				if changed {
+					atsMu.Lock()
+					ats.RepairNeeded = true
+					atsMu.Unlock()
+				}
+			}
+
+			// key repair has to be above, because we compute the checksum below.
+
+			sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo)
+			if err != nil {
+				return err
+			}
+			if sum == nil {
+				// probably one of the Noop stores from the tests.
+				return nil
+			}
+			sum.IsColKey = true
+			sum.PartitionID = partitionID
+			sum.Index = idx.Name()
+			sum.StorePath = store.GetStorePath()
+			sum.NodeID = nodeID
+			sum.IsPrimary = topo.IsPrimary(nodeID, partitionID)
+
+			replicas := topo.GetNonPrimaryReplicas(partitionID)
+			for _, replica := range replicas {
+				if nodeID == replica {
+					sum.IsReplica = true
+					break
+				}
+			}
+
+			sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, idx.Name())))
+			if verbose {
+				// This is not regular index logging. This is output of the pilosa-fsck tool.
+				// So it must be printing straight to stdout.
+				fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex)
+			}
+			atsMu.Lock()
+			ats.Sums = append(ats.Sums, sum)
+			atsMu.Unlock()
+
+			return nil
+		}
+		select {
+		case <-halters[0].ReqStop.Chan:
+			break tloop
+		case jobQ <- fun2:
+		}
+	} // end tloop
+
+	close(jobQ) // tell the workers no more jobs.
+
+	// wait for everyone to finish
+	for _, h := range halters {
+		<-h.Done.Chan
+	}
+	return ats, err
+}
+
+// returned by WriteFragmentChecksums
+type IndexFragmentSummary struct {
+	Dir       string
+	NodeID    string
+	Index     string
+	IndexPath string
+	Frg       []*FragSum
+
+	RelPath2fsum map[string]*FragSum
+}
+
+func (ifs *IndexFragmentSummary) String() (s string) {
+	s = fmt.Sprintf(`&pilosa.IndexFragmentSummary{
+  Dir:    '%v'
+  NodeID: '%v'
+  Index:     '%v'
+  IndexPath: '%v'
+`, ifs.Dir, ifs.NodeID, ifs.Index, ifs.IndexPath)
+	for _, frg := range ifs.Frg {
+		s += frg.String() + "\n"
+	}
+	s += "}\n"
 	return
 }
 
-func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) {
+// used in IndexFragmentSummary
+type FragSum struct {
+	AbsPath string
+	RelPath string
+
+	// critically, NodeID is how pilosa-fsck figures out if this
+	// fragment should be deleted if it is on a node it should not be.
+	NodeID string
+
+	Index    string
+	Field    string
+	View     string
+	Shard    uint64
+	Hotbits  int
+	Checksum string
+	Primary  int
+
+	ScanDone bool // pilosa-fsck will set this once done to avoid repairing multiple times.
+}
+
+func (fsum *FragSum) String() (s string) {
+	return fmt.Sprintf("%#v", fsum)
+}
+
+// if verbose, then print to w.
+func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool, topo *Topology, verbose bool) (sum *IndexFragmentSummary) {
+	sum = &IndexFragmentSummary{
+		Index:        idx.name,
+		IndexPath:    idx.path,
+		RelPath2fsum: make(map[string]*FragSum),
+	}
 	paths, err := listFilesUnderDir(idx.path, false, "", true)
 	panicOn(err)
 	index := idx.name
@@ -822,14 +1025,37 @@ func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) {
 			continue // ignore .meta paths
 		}
 		abspath := idx.path + sep + relpath
+		primary := topo.GetPrimaryForShardReplication(index, shard)
 
 		checksum, hotbits := RoaringFragmentChecksum(abspath, index, field, view, shard)
-		fmt.Fprintf(w, "frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v\n", checksum, field, view, shard, hotbits)
+		if verbose {
+			fmt.Fprintf(w, "# frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v primary:%03v\n", checksum, field, view, shard, hotbits, primary)
+		}
+		fsum := &FragSum{
+			AbsPath:  abspath,
+			RelPath:  relpath,
+			Index:    index,
+			Field:    field,
+			View:     view,
+			Shard:    shard,
+			Hotbits:  hotbits,
+			Checksum: checksum,
+			Primary:  primary,
+		}
+		sum.Frg = append(sum.Frg, fsum)
+		_, already := sum.RelPath2fsum[relpath]
+		if already {
+			panic(fmt.Sprintf("relpath '%v' was already present!?!", relpath))
+		}
+		sum.RelPath2fsum[relpath] = fsum
 		n++
 	}
 	if n == 0 {
-		fmt.Fprintf(w, "empty index '%v'", idx.path)
+		if verbose {
+			fmt.Fprintf(w, "empty index '%v'", idx.path)
+		}
 	}
+	return
 }
 
 func (idx *Index) Txf() *TxFactory {

license.exceptions

@@ -19,3 +19,7 @@
 ./cmd/pilosa-keydump/vprint.go
 ./cmd/pilosa-keydump/keydump.go
 ./synthload/vprint.go
+./proto/vdsm/vdsm.proto
+./proto/vdsm/vdsm.pb.go
+./cmd/pilosa-fsck/vprint.go
+./cmd/random-query/vprint.go

mock/translator.go

@@ -37,7 +37,10 @@ type TranslateStore struct {
 	EntryReaderFunc   func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error)
 }
 
-func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) {
+func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) {
+	return
+}
+func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) {
 	return
 }
 
@@ -93,6 +96,14 @@ func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) {
 	return 0, nil
 }
 
+func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) {
+	return
+}
+
+func (s *TranslateStore) GetStorePath() string {
+	return ""
+}
+
 var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil)
 
 type TranslateEntryReader struct {
@@ -107,3 +118,10 @@ func (r *TranslateEntryReader) Close() error {
 func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	return r.ReadEntryFunc(entry)
 }
+
+func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error {
+	panic("TODO")
+}
+func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error {
+	panic("TODO")
+}

proto/interface.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package pilosa
+package proto
 
 import (
 	"fmt"

proto/pilosa.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // source: pilosa.proto
 
-package pilosa
+package proto
 
 import (
 	context "context"
@@ -24,349 +24,6 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
-type VDS struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *VDS) Reset()         { *m = VDS{} }
-func (m *VDS) String() string { return proto.CompactTextString(m) }
-func (*VDS) ProtoMessage()    {}
-func (*VDS) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{0}
-}
-
-func (m *VDS) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_VDS.Unmarshal(m, b)
-}
-func (m *VDS) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_VDS.Marshal(b, m, deterministic)
-}
-func (m *VDS) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_VDS.Merge(m, src)
-}
-func (m *VDS) XXX_Size() int {
-	return xxx_messageInfo_VDS.Size(m)
-}
-func (m *VDS) XXX_DiscardUnknown() {
-	xxx_messageInfo_VDS.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_VDS proto.InternalMessageInfo
-
-func (m *VDS) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-type GetVDSsRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *GetVDSsRequest) Reset()         { *m = GetVDSsRequest{} }
-func (m *GetVDSsRequest) String() string { return proto.CompactTextString(m) }
-func (*GetVDSsRequest) ProtoMessage()    {}
-func (*GetVDSsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{1}
-}
-
-func (m *GetVDSsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetVDSsRequest.Unmarshal(m, b)
-}
-func (m *GetVDSsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetVDSsRequest.Marshal(b, m, deterministic)
-}
-func (m *GetVDSsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetVDSsRequest.Merge(m, src)
-}
-func (m *GetVDSsRequest) XXX_Size() int {
-	return xxx_messageInfo_GetVDSsRequest.Size(m)
-}
-func (m *GetVDSsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetVDSsRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetVDSsRequest proto.InternalMessageInfo
-
-type GetVDSsResponse struct {
-	Vdss                 []*VDS   `protobuf:"bytes,1,rep,name=vdss,proto3" json:"vdss,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *GetVDSsResponse) Reset()         { *m = GetVDSsResponse{} }
-func (m *GetVDSsResponse) String() string { return proto.CompactTextString(m) }
-func (*GetVDSsResponse) ProtoMessage()    {}
-func (*GetVDSsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{2}
-}
-
-func (m *GetVDSsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetVDSsResponse.Unmarshal(m, b)
-}
-func (m *GetVDSsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetVDSsResponse.Marshal(b, m, deterministic)
-}
-func (m *GetVDSsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetVDSsResponse.Merge(m, src)
-}
-func (m *GetVDSsResponse) XXX_Size() int {
-	return xxx_messageInfo_GetVDSsResponse.Size(m)
-}
-func (m *GetVDSsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetVDSsResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetVDSsResponse proto.InternalMessageInfo
-
-func (m *GetVDSsResponse) GetVdss() []*VDS {
-	if m != nil {
-		return m.Vdss
-	}
-	return nil
-}
-
-type GetVDSRequest struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *GetVDSRequest) Reset()         { *m = GetVDSRequest{} }
-func (m *GetVDSRequest) String() string { return proto.CompactTextString(m) }
-func (*GetVDSRequest) ProtoMessage()    {}
-func (*GetVDSRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{3}
-}
-
-func (m *GetVDSRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetVDSRequest.Unmarshal(m, b)
-}
-func (m *GetVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetVDSRequest.Marshal(b, m, deterministic)
-}
-func (m *GetVDSRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetVDSRequest.Merge(m, src)
-}
-func (m *GetVDSRequest) XXX_Size() int {
-	return xxx_messageInfo_GetVDSRequest.Size(m)
-}
-func (m *GetVDSRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetVDSRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetVDSRequest proto.InternalMessageInfo
-
-func (m *GetVDSRequest) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-type GetVDSResponse struct {
-	Vds                  *VDS     `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *GetVDSResponse) Reset()         { *m = GetVDSResponse{} }
-func (m *GetVDSResponse) String() string { return proto.CompactTextString(m) }
-func (*GetVDSResponse) ProtoMessage()    {}
-func (*GetVDSResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{4}
-}
-
-func (m *GetVDSResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_GetVDSResponse.Unmarshal(m, b)
-}
-func (m *GetVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_GetVDSResponse.Marshal(b, m, deterministic)
-}
-func (m *GetVDSResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_GetVDSResponse.Merge(m, src)
-}
-func (m *GetVDSResponse) XXX_Size() int {
-	return xxx_messageInfo_GetVDSResponse.Size(m)
-}
-func (m *GetVDSResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_GetVDSResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetVDSResponse proto.InternalMessageInfo
-
-func (m *GetVDSResponse) GetVds() *VDS {
-	if m != nil {
-		return m.Vds
-	}
-	return nil
-}
-
-type PostVDSRequest struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Keys                 bool     `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"`
-	TrackExistence       bool     `protobuf:"varint,3,opt,name=trackExistence,proto3" json:"trackExistence,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *PostVDSRequest) Reset()         { *m = PostVDSRequest{} }
-func (m *PostVDSRequest) String() string { return proto.CompactTextString(m) }
-func (*PostVDSRequest) ProtoMessage()    {}
-func (*PostVDSRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{5}
-}
-
-func (m *PostVDSRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PostVDSRequest.Unmarshal(m, b)
-}
-func (m *PostVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PostVDSRequest.Marshal(b, m, deterministic)
-}
-func (m *PostVDSRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PostVDSRequest.Merge(m, src)
-}
-func (m *PostVDSRequest) XXX_Size() int {
-	return xxx_messageInfo_PostVDSRequest.Size(m)
-}
-func (m *PostVDSRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_PostVDSRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_PostVDSRequest proto.InternalMessageInfo
-
-func (m *PostVDSRequest) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-func (m *PostVDSRequest) GetKeys() bool {
-	if m != nil {
-		return m.Keys
-	}
-	return false
-}
-
-func (m *PostVDSRequest) GetTrackExistence() bool {
-	if m != nil {
-		return m.TrackExistence
-	}
-	return false
-}
-
-type PostVDSResponse struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *PostVDSResponse) Reset()         { *m = PostVDSResponse{} }
-func (m *PostVDSResponse) String() string { return proto.CompactTextString(m) }
-func (*PostVDSResponse) ProtoMessage()    {}
-func (*PostVDSResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{6}
-}
-
-func (m *PostVDSResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_PostVDSResponse.Unmarshal(m, b)
-}
-func (m *PostVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_PostVDSResponse.Marshal(b, m, deterministic)
-}
-func (m *PostVDSResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_PostVDSResponse.Merge(m, src)
-}
-func (m *PostVDSResponse) XXX_Size() int {
-	return xxx_messageInfo_PostVDSResponse.Size(m)
-}
-func (m *PostVDSResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_PostVDSResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_PostVDSResponse proto.InternalMessageInfo
-
-type DeleteVDSRequest struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *DeleteVDSRequest) Reset()         { *m = DeleteVDSRequest{} }
-func (m *DeleteVDSRequest) String() string { return proto.CompactTextString(m) }
-func (*DeleteVDSRequest) ProtoMessage()    {}
-func (*DeleteVDSRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{7}
-}
-
-func (m *DeleteVDSRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_DeleteVDSRequest.Unmarshal(m, b)
-}
-func (m *DeleteVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_DeleteVDSRequest.Marshal(b, m, deterministic)
-}
-func (m *DeleteVDSRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DeleteVDSRequest.Merge(m, src)
-}
-func (m *DeleteVDSRequest) XXX_Size() int {
-	return xxx_messageInfo_DeleteVDSRequest.Size(m)
-}
-func (m *DeleteVDSRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_DeleteVDSRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DeleteVDSRequest proto.InternalMessageInfo
-
-func (m *DeleteVDSRequest) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-type DeleteVDSResponse struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *DeleteVDSResponse) Reset()         { *m = DeleteVDSResponse{} }
-func (m *DeleteVDSResponse) String() string { return proto.CompactTextString(m) }
-func (*DeleteVDSResponse) ProtoMessage()    {}
-func (*DeleteVDSResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{8}
-}
-
-func (m *DeleteVDSResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_DeleteVDSResponse.Unmarshal(m, b)
-}
-func (m *DeleteVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_DeleteVDSResponse.Marshal(b, m, deterministic)
-}
-func (m *DeleteVDSResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DeleteVDSResponse.Merge(m, src)
-}
-func (m *DeleteVDSResponse) XXX_Size() int {
-	return xxx_messageInfo_DeleteVDSResponse.Size(m)
-}
-func (m *DeleteVDSResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_DeleteVDSResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo
-
 type QueryPQLRequest struct {
 	Index                string   `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"`
 	Pql                  string   `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"`
@@ -379,7 +36,7 @@ func (m *QueryPQLRequest) Reset()         { *m = QueryPQLRequest{} }
 func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) }
 func (*QueryPQLRequest) ProtoMessage()    {}
 func (*QueryPQLRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{9}
+	return fileDescriptor_ef0691a44d1e275c, []int{0}
 }
 
 func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error {
@@ -425,7 +82,7 @@ func (m *QuerySQLRequest) Reset()         { *m = QuerySQLRequest{} }
 func (m *QuerySQLRequest) String() string { return proto.CompactTextString(m) }
 func (*QuerySQLRequest) ProtoMessage()    {}
 func (*QuerySQLRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{10}
+	return fileDescriptor_ef0691a44d1e275c, []int{1}
 }
 
 func (m *QuerySQLRequest) XXX_Unmarshal(b []byte) error {
@@ -465,7 +122,7 @@ func (m *StatusError) Reset()         { *m = StatusError{} }
 func (m *StatusError) String() string { return proto.CompactTextString(m) }
 func (*StatusError) ProtoMessage()    {}
 func (*StatusError) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{11}
+	return fileDescriptor_ef0691a44d1e275c, []int{2}
 }
 
 func (m *StatusError) XXX_Unmarshal(b []byte) error {
@@ -513,7 +170,7 @@ func (m *RowResponse) Reset()         { *m = RowResponse{} }
 func (m *RowResponse) String() string { return proto.CompactTextString(m) }
 func (*RowResponse) ProtoMessage()    {}
 func (*RowResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{12}
+	return fileDescriptor_ef0691a44d1e275c, []int{3}
 }
 
 func (m *RowResponse) XXX_Unmarshal(b []byte) error {
@@ -566,7 +223,7 @@ func (m *Row) Reset()         { *m = Row{} }
 func (m *Row) String() string { return proto.CompactTextString(m) }
 func (*Row) ProtoMessage()    {}
 func (*Row) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{13}
+	return fileDescriptor_ef0691a44d1e275c, []int{4}
 }
 
 func (m *Row) XXX_Unmarshal(b []byte) error {
@@ -607,7 +264,7 @@ func (m *TableResponse) Reset()         { *m = TableResponse{} }
 func (m *TableResponse) String() string { return proto.CompactTextString(m) }
 func (*TableResponse) ProtoMessage()    {}
 func (*TableResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{14}
+	return fileDescriptor_ef0691a44d1e275c, []int{5}
 }
 
 func (m *TableResponse) XXX_Unmarshal(b []byte) error {
@@ -661,7 +318,7 @@ func (m *ColumnInfo) Reset()         { *m = ColumnInfo{} }
 func (m *ColumnInfo) String() string { return proto.CompactTextString(m) }
 func (*ColumnInfo) ProtoMessage()    {}
 func (*ColumnInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{15}
+	return fileDescriptor_ef0691a44d1e275c, []int{6}
 }
 
 func (m *ColumnInfo) XXX_Unmarshal(b []byte) error {
@@ -717,7 +374,7 @@ func (m *ColumnResponse) Reset()         { *m = ColumnResponse{} }
 func (m *ColumnResponse) String() string { return proto.CompactTextString(m) }
 func (*ColumnResponse) ProtoMessage()    {}
 func (*ColumnResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{16}
+	return fileDescriptor_ef0691a44d1e275c, []int{7}
 }
 
 func (m *ColumnResponse) XXX_Unmarshal(b []byte) error {
@@ -893,7 +550,7 @@ func (m *Decimal) Reset()         { *m = Decimal{} }
 func (m *Decimal) String() string { return proto.CompactTextString(m) }
 func (*Decimal) ProtoMessage()    {}
 func (*Decimal) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{17}
+	return fileDescriptor_ef0691a44d1e275c, []int{8}
 }
 
 func (m *Decimal) XXX_Unmarshal(b []byte) error {
@@ -944,7 +601,7 @@ func (m *InspectRequest) Reset()         { *m = InspectRequest{} }
 func (m *InspectRequest) String() string { return proto.CompactTextString(m) }
 func (*InspectRequest) ProtoMessage()    {}
 func (*InspectRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{18}
+	return fileDescriptor_ef0691a44d1e275c, []int{9}
 }
 
 func (m *InspectRequest) XXX_Unmarshal(b []byte) error {
@@ -1018,7 +675,7 @@ func (m *Uint64Array) Reset()         { *m = Uint64Array{} }
 func (m *Uint64Array) String() string { return proto.CompactTextString(m) }
 func (*Uint64Array) ProtoMessage()    {}
 func (*Uint64Array) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{19}
+	return fileDescriptor_ef0691a44d1e275c, []int{10}
 }
 
 func (m *Uint64Array) XXX_Unmarshal(b []byte) error {
@@ -1057,7 +714,7 @@ func (m *StringArray) Reset()         { *m = StringArray{} }
 func (m *StringArray) String() string { return proto.CompactTextString(m) }
 func (*StringArray) ProtoMessage()    {}
 func (*StringArray) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{20}
+	return fileDescriptor_ef0691a44d1e275c, []int{11}
 }
 
 func (m *StringArray) XXX_Unmarshal(b []byte) error {
@@ -1099,7 +756,7 @@ func (m *IdsOrKeys) Reset()         { *m = IdsOrKeys{} }
 func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) }
 func (*IdsOrKeys) ProtoMessage()    {}
 func (*IdsOrKeys) Descriptor() ([]byte, []int) {
-	return fileDescriptor_ef0691a44d1e275c, []int{21}
+	return fileDescriptor_ef0691a44d1e275c, []int{12}
 }
 
 func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error {
@@ -1166,15 +823,6 @@ func (*IdsOrKeys) XXX_OneofWrappers() []interface{} {
 }
 
 func init() {
-	proto.RegisterType((*VDS)(nil), "pilosa.VDS")
-	proto.RegisterType((*GetVDSsRequest)(nil), "pilosa.GetVDSsRequest")
-	proto.RegisterType((*GetVDSsResponse)(nil), "pilosa.GetVDSsResponse")
-	proto.RegisterType((*GetVDSRequest)(nil), "pilosa.GetVDSRequest")
-	proto.RegisterType((*GetVDSResponse)(nil), "pilosa.GetVDSResponse")
-	proto.RegisterType((*PostVDSRequest)(nil), "pilosa.PostVDSRequest")
-	proto.RegisterType((*PostVDSResponse)(nil), "pilosa.PostVDSResponse")
-	proto.RegisterType((*DeleteVDSRequest)(nil), "pilosa.DeleteVDSRequest")
-	proto.RegisterType((*DeleteVDSResponse)(nil), "pilosa.DeleteVDSResponse")
 	proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest")
 	proto.RegisterType((*QuerySQLRequest)(nil), "pilosa.QuerySQLRequest")
 	proto.RegisterType((*StatusError)(nil), "pilosa.StatusError")
@@ -1193,64 +841,54 @@ func init() {
 func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) }
 
 var fileDescriptor_ef0691a44d1e275c = []byte{
-	// 897 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdd, 0x72, 0xdb, 0x44,
-	0x14, 0xb6, 0x90, 0x6a, 0x59, 0xc7, 0x89, 0x93, 0x6c, 0x20, 0xb8, 0x1e, 0xa0, 0x66, 0x3b, 0x53,
-	0xcc, 0xc0, 0x94, 0x12, 0x28, 0x4c, 0x21, 0x5c, 0x34, 0x75, 0xc1, 0x19, 0x60, 0x70, 0xd7, 0xd4,
-	0xd7, 0x6c, 0xac, 0x75, 0xd0, 0x54, 0xd1, 0x3a, 0x5a, 0x39, 0xa9, 0x5f, 0x80, 0x37, 0xe0, 0x0d,
-	0x78, 0x0b, 0xae, 0x78, 0x33, 0x66, 0x7f, 0x25, 0xb9, 0x31, 0x2d, 0xb9, 0xd3, 0x39, 0xdf, 0x77,
-	0xfe, 0xf6, 0xec, 0x9e, 0x23, 0xd8, 0x5a, 0x24, 0x29, 0x17, 0xf4, 0xfe, 0x22, 0xe7, 0x05, 0x47,
-	0x4d, 0x2d, 0xe1, 0xdb, 0xe0, 0x4f, 0x87, 0x13, 0x84, 0x20, 0xc8, 0xe8, 0x39, 0xeb, 0x7a, 0x7d,
-	0x6f, 0x10, 0x11, 0xf5, 0x8d, 0x77, 0xa1, 0xf3, 0x03, 0x2b, 0xa6, 0xc3, 0x89, 0x20, 0xec, 0x62,
-	0xc9, 0x44, 0x81, 0x0f, 0x61, 0xc7, 0x69, 0xc4, 0x82, 0x67, 0x82, 0xa1, 0x3b, 0x10, 0x5c, 0xc6,
-	0x42, 0x74, 0xbd, 0xbe, 0x3f, 0x68, 0x1f, 0xb6, 0xef, 0x9b, 0x20, 0xd3, 0xe1, 0x84, 0x28, 0x00,
-	0xdf, 0x85, 0x6d, 0x6d, 0x63, 0x9c, 0x5c, 0x1b, 0xea, 0x33, 0x1b, 0xca, 0xf9, 0x7d, 0x1f, 0xfc,
-	0xcb, 0x58, 0x28, 0xd2, 0x9a, 0x5b, 0xa9, 0xc7, 0xbf, 0x41, 0x67, 0xcc, 0xc5, 0x6b, 0xdc, 0x4a,
-	0xdd, 0x0b, 0xb6, 0x12, 0xdd, 0xb7, 0xfa, 0xde, 0xa0, 0x45, 0xd4, 0x37, 0xba, 0x07, 0x9d, 0x22,
-	0xa7, 0xb3, 0x17, 0x4f, 0x5f, 0x26, 0xa2, 0x60, 0xd9, 0x8c, 0x75, 0x7d, 0x85, 0xae, 0x69, 0xf1,
-	0x1e, 0xec, 0xb8, 0x08, 0x3a, 0x27, 0x7c, 0x0f, 0x76, 0x87, 0x2c, 0x65, 0x05, 0x7b, 0x4d, 0x35,
-	0xfb, 0xb0, 0x57, 0xe1, 0x19, 0xe3, 0x47, 0xb0, 0xf3, 0x6c, 0xc9, 0xf2, 0xd5, 0xf8, 0xd9, 0x4f,
-	0xd6, 0xf6, 0x6d, 0xb8, 0x95, 0x64, 0x31, 0x7b, 0x69, 0x8c, 0xb5, 0x80, 0x76, 0xc1, 0x5f, 0x5c,
-	0xa4, 0x2a, 0xe7, 0x88, 0xc8, 0x4f, 0x7c, 0xd7, 0x98, 0x4e, 0x4a, 0xd3, 0x5d, 0xf0, 0xc5, 0x45,
-	0x6a, 0x0c, 0xe5, 0x27, 0xfe, 0x16, 0xda, 0x93, 0x82, 0x16, 0x4b, 0xf1, 0x34, 0xcf, 0x79, 0x2e,
-	0xf3, 0x7a, 0xc2, 0x63, 0x9d, 0xd7, 0x36, 0x51, 0xdf, 0xa8, 0x0b, 0xe1, 0xcf, 0x4c, 0x08, 0x7a,
-	0xc6, 0x8c, 0x77, 0x2b, 0xe2, 0xbf, 0x3c, 0x68, 0x13, 0x7e, 0xe5, 0x4e, 0xff, 0x53, 0x08, 0x7f,
-	0x67, 0x34, 0x66, 0xb9, 0x6d, 0x2c, 0xb2, 0x1d, 0x78, 0xc2, 0xd3, 0xe5, 0x79, 0x76, 0x92, 0xcd,
-	0x39, 0xb1, 0x14, 0xf4, 0x00, 0xc2, 0x99, 0x52, 0xcb, 0x93, 0x96, 0xec, 0x83, 0x3a, 0xdb, 0xba,
-	0x25, 0x96, 0x86, 0x1e, 0xd6, 0x92, 0x55, 0x1d, 0x68, 0x1f, 0xee, 0x5b, 0xab, 0x0a, 0x44, 0xaa,
-	0x3c, 0xfc, 0x35, 0xf8, 0x84, 0x5f, 0x55, 0xe3, 0x79, 0x6f, 0x14, 0x0f, 0xff, 0xe9, 0xc1, 0xf6,
-	0xaf, 0xf4, 0x34, 0x65, 0x37, 0xac, 0xf0, 0x0e, 0x04, 0x39, 0xbf, 0xb2, 0xe5, 0xb9, 0xeb, 0x28,
-	0x8f, 0x4c, 0x01, 0x37, 0x2d, 0xe8, 0x08, 0xa0, 0x0c, 0x77, 0xed, 0x15, 0xee, 0x41, 0x2b, 0xa6,
-	0x05, 0x2d, 0x56, 0x0b, 0xdb, 0x34, 0x27, 0xe3, 0x3f, 0x7c, 0xe8, 0xd4, 0x2b, 0x46, 0x1f, 0x40,
-	0x24, 0x8a, 0x3c, 0xc9, 0xce, 0xa6, 0xd4, 0xdc, 0x8e, 0x51, 0x83, 0x94, 0x2a, 0x89, 0x2f, 0x93,
-	0xac, 0xf8, 0xea, 0x4b, 0x89, 0x4b, 0x7f, 0x81, 0xc4, 0x9d, 0x0a, 0xbd, 0x07, 0x2d, 0x07, 0xcb,
-	0x22, 0xfc, 0x51, 0x83, 0x38, 0x0d, 0xea, 0x41, 0x78, 0xca, 0x79, 0x2a, 0xc1, 0x40, 0x3e, 0x9a,
-	0x51, 0x83, 0x58, 0x85, 0xc2, 0x52, 0x7e, 0x2a, 0xb1, 0x5b, 0x7d, 0x6f, 0xb0, 0xa5, 0x30, 0xad,
-	0x40, 0xdf, 0x41, 0x47, 0x87, 0x78, 0x9c, 0xe7, 0x74, 0x25, 0x29, 0xcd, 0xfa, 0x01, 0x3d, 0x2f,
-	0xd1, 0x51, 0x83, 0xac, 0x91, 0xa5, 0xb9, 0xae, 0xc0, 0x99, 0x87, 0xeb, 0xe7, 0xeb, 0x50, 0x69,
-	0x5e, 0x27, 0xa3, 0x3e, 0xc0, 0x3c, 0xe5, 0xd4, 0x54, 0xd5, 0xea, 0x7b, 0x03, 0x6f, 0xd4, 0x20,
-	0x15, 0x1d, 0xfa, 0x1c, 0x20, 0x66, 0xb3, 0xe4, 0x9c, 0xaa, 0xd2, 0x22, 0xe5, 0x7c, 0xc7, 0x3a,
-	0x1f, 0x6a, 0x44, 0x9a, 0x94, 0xa4, 0xe3, 0x36, 0x44, 0xfa, 0x72, 0x4d, 0x69, 0x8a, 0x1f, 0x42,
-	0x68, 0x58, 0xf2, 0x4d, 0x5f, 0xd2, 0x74, 0xa9, 0x9b, 0xe8, 0x13, 0x2d, 0x48, 0xad, 0x98, 0xd1,
-	0x54, 0xb7, 0xd0, 0x27, 0x5a, 0xc0, 0x7f, 0x7b, 0xd0, 0x39, 0xc9, 0xc4, 0x82, 0xcd, 0x8a, 0xff,
-	0x1e, 0x09, 0x9f, 0x54, 0x1f, 0x98, 0x4c, 0x6e, 0xcf, 0x26, 0x77, 0x12, 0x8b, 0x5f, 0xf2, 0x1f,
-	0xd9, 0x4a, 0x94, 0x6f, 0x0b, 0xc3, 0xd6, 0x3c, 0x49, 0x0b, 0x96, 0x7f, 0x9f, 0xb0, 0x34, 0x16,
-	0x5d, 0xbf, 0xef, 0x0f, 0x22, 0x52, 0xd3, 0xc9, 0x30, 0x69, 0x72, 0x9e, 0x14, 0xaa, 0x8d, 0x01,
-	0xd1, 0x02, 0x3a, 0x80, 0x26, 0x9f, 0xcf, 0x05, 0x2b, 0x54, 0x07, 0x03, 0x62, 0x24, 0xc9, 0xbe,
-	0x90, 0xf3, 0x47, 0x75, 0x2d, 0x22, 0x5a, 0xc0, 0x1f, 0x42, 0xbb, 0xd2, 0x36, 0x79, 0x79, 0x2f,
-	0x69, 0xaa, 0x5f, 0x53, 0x40, 0xd4, 0xb7, 0xa4, 0x54, 0x5a, 0x53, 0xa3, 0x44, 0x86, 0x72, 0x06,
-	0x91, 0xab, 0x01, 0x7d, 0x04, 0x7e, 0xe2, 0x86, 0xfe, 0x86, 0xcb, 0x21, 0x19, 0xe8, 0xe3, 0xca,
-	0x60, 0xdf, 0x78, 0x0f, 0x14, 0xe5, 0xb8, 0x09, 0x81, 0x7c, 0x2c, 0x87, 0xff, 0x04, 0xd0, 0x1c,
-	0x2b, 0x1a, 0x3a, 0x82, 0xd0, 0xac, 0x31, 0xe4, 0x26, 0x47, 0x7d, 0xd3, 0xf5, 0xde, 0x7d, 0x45,
-	0x6f, 0xc6, 0x78, 0x03, 0x3d, 0x82, 0xa6, 0x56, 0xa2, 0x77, 0xea, 0x24, 0x6b, 0x7b, 0xb0, 0xae,
-	0x76, 0xa6, 0x47, 0x10, 0x9a, 0x9d, 0x52, 0x06, 0xae, 0xaf, 0xb1, 0x32, 0xf0, 0xfa, 0xf2, 0x69,
-	0xa0, 0x63, 0x88, 0xdc, 0x5a, 0x41, 0xdd, 0xf2, 0x7a, 0xd6, 0x37, 0x52, 0xef, 0xf6, 0x35, 0x48,
-	0x25, 0x83, 0x96, 0x5d, 0x25, 0xc8, 0x85, 0x5a, 0x5b, 0x2e, 0xbd, 0xfd, 0xea, 0x7c, 0x73, 0xb6,
-	0x0f, 0x3c, 0xf4, 0x18, 0xb6, 0x2d, 0xf7, 0x79, 0x46, 0xf3, 0xd5, 0x66, 0x17, 0xee, 0x68, 0x6a,
-	0x53, 0xb7, 0x92, 0xc0, 0xf8, 0x95, 0x04, 0xc6, 0xff, 0x23, 0x81, 0xf1, 0xf5, 0x09, 0x8c, 0xdf,
-	0x20, 0x81, 0x6f, 0x20, 0x34, 0x6f, 0xae, 0xec, 0x41, 0xfd, 0x11, 0x6e, 0x0c, 0x7f, 0xda, 0x54,
-	0xff, 0x4e, 0x5f, 0xfc, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x1e, 0xff, 0x07, 0x47, 0x4b, 0x09, 0x00,
-	0x00,
+	// 745 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x72, 0xd3, 0x3a,
+	0x10, 0x8e, 0x6b, 0x37, 0x89, 0x37, 0xfd, 0x3b, 0xea, 0x39, 0x3d, 0x99, 0xce, 0x99, 0x83, 0xeb,
+	0x5e, 0x10, 0x06, 0xa6, 0x2d, 0x81, 0xc2, 0x00, 0xe5, 0xa2, 0x2d, 0x30, 0xe9, 0x00, 0x43, 0xaa,
+	0xd2, 0x5e, 0x70, 0xa7, 0xc4, 0x4a, 0xea, 0x41, 0xb1, 0x12, 0xcb, 0x69, 0xc9, 0x0b, 0xf0, 0x06,
+	0xbc, 0x01, 0x6f, 0xc1, 0x3d, 0xcf, 0xc5, 0x48, 0xb2, 0x1c, 0xbb, 0x10, 0xa6, 0xf4, 0xca, 0xda,
+	0xfd, 0xbe, 0xd5, 0xee, 0x6a, 0x7f, 0x0c, 0x0b, 0xc3, 0x90, 0x71, 0x41, 0xb6, 0x86, 0x31, 0x4f,
+	0x38, 0x2a, 0x6b, 0xc9, 0x7f, 0x02, 0xcb, 0xc7, 0x63, 0x1a, 0x4f, 0xda, 0xc7, 0x6f, 0x30, 0x1d,
+	0x8d, 0xa9, 0x48, 0xd0, 0xdf, 0x30, 0x1f, 0x46, 0x01, 0xfd, 0x54, 0xb7, 0x3c, 0xab, 0xe1, 0x62,
+	0x2d, 0xa0, 0x15, 0xb0, 0x87, 0x23, 0x56, 0x9f, 0x53, 0x3a, 0x79, 0xf4, 0x37, 0x53, 0xd3, 0x93,
+	0xa9, 0xe9, 0x0a, 0xd8, 0x62, 0xc4, 0x52, 0x43, 0x79, 0xf4, 0x9f, 0x41, 0xed, 0x24, 0x21, 0xc9,
+	0x58, 0xbc, 0x8c, 0x63, 0x1e, 0x23, 0x04, 0xce, 0x21, 0x0f, 0xa8, 0x62, 0x2c, 0x62, 0x75, 0x46,
+	0x75, 0xa8, 0xbc, 0xa5, 0x42, 0x90, 0x3e, 0x4d, 0x6f, 0x37, 0xa2, 0xff, 0xd5, 0x82, 0x1a, 0xe6,
+	0x97, 0x98, 0x8a, 0x21, 0x8f, 0x04, 0x45, 0xf7, 0xa0, 0x72, 0x4e, 0x49, 0x40, 0x63, 0x51, 0xb7,
+	0x3c, 0xbb, 0x51, 0x6b, 0xa2, 0xad, 0x34, 0xa9, 0x43, 0xce, 0xc6, 0x83, 0xe8, 0x28, 0xea, 0x71,
+	0x6c, 0x28, 0x68, 0x07, 0x2a, 0x5d, 0xa5, 0x16, 0xf5, 0x39, 0xc5, 0x5e, 0x2b, 0xb2, 0xcd, 0xb5,
+	0xd8, 0xd0, 0xd0, 0x6e, 0x21, 0xd8, 0xba, 0xed, 0x59, 0x8d, 0x5a, 0x73, 0xd5, 0x58, 0xe5, 0x20,
+	0x9c, 0xe7, 0xf9, 0x8f, 0xc1, 0xc6, 0xfc, 0x32, 0xef, 0xcf, 0xba, 0x96, 0x3f, 0xff, 0x8b, 0x05,
+	0x8b, 0xef, 0x49, 0x87, 0xd1, 0x1b, 0x66, 0x78, 0x0b, 0x9c, 0x98, 0x5f, 0x9a, 0xf4, 0x6a, 0x86,
+	0x2a, 0x9f, 0x4c, 0x01, 0x37, 0x4d, 0x68, 0x0f, 0x60, 0xea, 0x4e, 0xd6, 0x2c, 0x22, 0x03, 0x9a,
+	0x56, 0x55, 0x9d, 0xd1, 0x3a, 0x54, 0x03, 0x92, 0x90, 0x64, 0x32, 0x34, 0x45, 0xcb, 0x64, 0xff,
+	0xb3, 0x0d, 0x4b, 0xc5, 0x8c, 0xd1, 0xff, 0xe0, 0x8a, 0x24, 0x0e, 0xa3, 0xfe, 0x19, 0x49, 0xbb,
+	0xa3, 0x55, 0xc2, 0x53, 0x95, 0xc4, 0xc7, 0x61, 0x94, 0x3c, 0x7a, 0x28, 0x71, 0x79, 0x9f, 0x23,
+	0xf1, 0x4c, 0x85, 0xfe, 0x83, 0x6a, 0x06, 0xcb, 0x24, 0xec, 0x56, 0x09, 0x67, 0x1a, 0xb4, 0x0e,
+	0x95, 0x0e, 0xe7, 0x4c, 0x82, 0x8e, 0x67, 0x35, 0xaa, 0xad, 0x12, 0x36, 0x0a, 0x85, 0x31, 0xde,
+	0x91, 0xd8, 0xbc, 0x67, 0x35, 0x16, 0x14, 0xa6, 0x15, 0xe8, 0x39, 0x2c, 0x69, 0x17, 0xfb, 0x71,
+	0x4c, 0x26, 0x92, 0x52, 0x2e, 0x3e, 0xd0, 0xe9, 0x14, 0x6d, 0x95, 0xf0, 0x15, 0xb2, 0x34, 0xd7,
+	0x19, 0x64, 0xe6, 0x95, 0xab, 0xef, 0x9b, 0xa1, 0xd2, 0xbc, 0x48, 0x46, 0x1e, 0x40, 0x8f, 0x71,
+	0x92, 0x66, 0x55, 0xf5, 0xac, 0x86, 0xd5, 0x2a, 0xe1, 0x9c, 0x0e, 0xdd, 0x07, 0x08, 0x68, 0x37,
+	0x1c, 0x10, 0x95, 0x9a, 0xab, 0x2e, 0x5f, 0x36, 0x97, 0xbf, 0xd0, 0x88, 0x34, 0x99, 0x92, 0x0e,
+	0x6a, 0xe0, 0xea, 0xe6, 0x3a, 0x23, 0xcc, 0xdf, 0x85, 0x4a, 0xca, 0x92, 0x33, 0x7d, 0x41, 0xd8,
+	0x58, 0x17, 0xd1, 0xc6, 0x5a, 0x90, 0x5a, 0xd1, 0x25, 0x4c, 0x97, 0xd0, 0xc6, 0x5a, 0xf0, 0xbf,
+	0x59, 0xb0, 0x74, 0x14, 0x89, 0x21, 0xed, 0x26, 0xbf, 0x5f, 0x09, 0x77, 0xf3, 0x03, 0x26, 0x83,
+	0xfb, 0xcb, 0x04, 0x77, 0x14, 0x88, 0x77, 0xf1, 0x6b, 0x3a, 0x11, 0xd3, 0xd9, 0xf2, 0x61, 0xa1,
+	0x17, 0xb2, 0x84, 0xc6, 0xaf, 0x42, 0xca, 0x02, 0x51, 0xb7, 0x3d, 0xbb, 0xe1, 0xe2, 0x82, 0x4e,
+	0xba, 0x61, 0xe1, 0x20, 0x4c, 0x54, 0x19, 0x1d, 0xac, 0x05, 0xb4, 0x06, 0x65, 0xde, 0xeb, 0x09,
+	0x9a, 0xa8, 0x0a, 0x3a, 0x38, 0x95, 0x24, 0x7b, 0x24, 0xf7, 0x8f, 0xaa, 0x9a, 0x8b, 0xb5, 0xe0,
+	0x6f, 0x40, 0x2d, 0x57, 0x36, 0xd9, 0xbc, 0x17, 0x84, 0xe9, 0x69, 0x72, 0xb0, 0x3a, 0x4b, 0x4a,
+	0xae, 0x34, 0x05, 0x8a, 0x9b, 0x52, 0xfa, 0xe0, 0x66, 0x39, 0xa0, 0xdb, 0x60, 0x87, 0x81, 0x50,
+	0xb9, 0xcf, 0x6c, 0x0e, 0xc9, 0x40, 0x77, 0xc0, 0xf9, 0x48, 0x27, 0xe6, 0x35, 0x66, 0xf4, 0x81,
+	0xa2, 0x1c, 0x94, 0xc1, 0x91, 0xc3, 0xd2, 0xfc, 0x3e, 0x07, 0xe5, 0xb6, 0xa2, 0xa1, 0x3d, 0xa8,
+	0x9a, 0x7d, 0x8a, 0xfe, 0x35, 0xb6, 0x57, 0x36, 0xec, 0xfa, 0x6a, 0x7e, 0xc8, 0xd3, 0xf1, 0xf2,
+	0x4b, 0x3b, 0x16, 0xda, 0x87, 0x45, 0xc3, 0x3d, 0x8d, 0x48, 0x3c, 0x99, 0x7d, 0xc5, 0x3f, 0x06,
+	0x28, 0xac, 0x1e, 0xbf, 0x94, 0x05, 0xd0, 0xfe, 0x29, 0x80, 0xf6, 0x1f, 0x04, 0xd0, 0xfe, 0x75,
+	0x00, 0xed, 0x6b, 0x04, 0xf0, 0x14, 0x2a, 0x69, 0xe3, 0xa1, 0x6c, 0x77, 0x16, 0x3b, 0x71, 0xa6,
+	0xfb, 0x83, 0xcd, 0x0f, 0x1b, 0xfd, 0x30, 0x39, 0x1f, 0x77, 0xb6, 0xba, 0x7c, 0xb0, 0xad, 0x49,
+	0xe6, 0x73, 0xd1, 0xdc, 0x56, 0x7f, 0xbd, 0x4e, 0x59, 0x7d, 0x1e, 0xfc, 0x08, 0x00, 0x00, 0xff,
+	0xff, 0x60, 0xce, 0x2e, 0x49, 0x0c, 0x07, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -1265,10 +903,6 @@ const _ = grpc.SupportPackageIsVersion4
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
 type PilosaClient interface {
-	GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error)
-	GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error)
-	PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error)
-	DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error)
 	QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error)
 	QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error)
 	QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error)
@@ -1284,42 +918,6 @@ func NewPilosaClient(cc *grpc.ClientConn) PilosaClient {
 	return &pilosaClient{cc}
 }
 
-func (c *pilosaClient) GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) {
-	out := new(GetVDSsResponse)
-	err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDSs", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *pilosaClient) GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) {
-	out := new(GetVDSResponse)
-	err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDS", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *pilosaClient) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) {
-	out := new(PostVDSResponse)
-	err := c.cc.Invoke(ctx, "/pilosa.Pilosa/PostVDS", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *pilosaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) {
-	out := new(DeleteVDSResponse)
-	err := c.cc.Invoke(ctx, "/pilosa.Pilosa/DeleteVDS", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 func (c *pilosaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) {
 	stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/pilosa.Pilosa/QuerySQL", opts...)
 	if err != nil {
@@ -1436,10 +1034,6 @@ func (x *pilosaInspectClient) Recv() (*RowResponse, error) {
 
 // PilosaServer is the server API for Pilosa service.
 type PilosaServer interface {
-	GetVDSs(context.Context, *GetVDSsRequest) (*GetVDSsResponse, error)
-	GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error)
-	PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error)
-	DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error)
 	QuerySQL(*QuerySQLRequest, Pilosa_QuerySQLServer) error
 	QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error)
 	QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error
@@ -1451,18 +1045,6 @@ type PilosaServer interface {
 type UnimplementedPilosaServer struct {
 }
 
-func (*UnimplementedPilosaServer) GetVDSs(ctx context.Context, req *GetVDSsRequest) (*GetVDSsResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetVDSs not implemented")
-}
-func (*UnimplementedPilosaServer) GetVDS(ctx context.Context, req *GetVDSRequest) (*GetVDSResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method GetVDS not implemented")
-}
-func (*UnimplementedPilosaServer) PostVDS(ctx context.Context, req *PostVDSRequest) (*PostVDSResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented")
-}
-func (*UnimplementedPilosaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented")
-}
 func (*UnimplementedPilosaServer) QuerySQL(req *QuerySQLRequest, srv Pilosa_QuerySQLServer) error {
 	return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented")
 }
@@ -1483,78 +1065,6 @@ func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) {
 	s.RegisterService(&_Pilosa_serviceDesc, srv)
 }
 
-func _Pilosa_GetVDSs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetVDSsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(PilosaServer).GetVDSs(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/pilosa.Pilosa/GetVDSs",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(PilosaServer).GetVDSs(ctx, req.(*GetVDSsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Pilosa_GetVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetVDSRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(PilosaServer).GetVDS(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/pilosa.Pilosa/GetVDS",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(PilosaServer).GetVDS(ctx, req.(*GetVDSRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Pilosa_PostVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(PostVDSRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(PilosaServer).PostVDS(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/pilosa.Pilosa/PostVDS",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(PilosaServer).PostVDS(ctx, req.(*PostVDSRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Pilosa_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(DeleteVDSRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(PilosaServer).DeleteVDS(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/pilosa.Pilosa/DeleteVDS",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(PilosaServer).DeleteVDS(ctx, req.(*DeleteVDSRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
 func _Pilosa_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(QuerySQLRequest)
 	if err := stream.RecvMsg(m); err != nil {
@@ -1658,22 +1168,6 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "pilosa.Pilosa",
 	HandlerType: (*PilosaServer)(nil),
 	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "GetVDSs",
-			Handler:    _Pilosa_GetVDSs_Handler,
-		},
-		{
-			MethodName: "GetVDS",
-			Handler:    _Pilosa_GetVDS_Handler,
-		},
-		{
-			MethodName: "PostVDS",
-			Handler:    _Pilosa_PostVDS_Handler,
-		},
-		{
-			MethodName: "DeleteVDS",
-			Handler:    _Pilosa_DeleteVDS_Handler,
-		},
 		{
 			MethodName: "QuerySQLUnary",
 			Handler:    _Pilosa_QuerySQLUnary_Handler,

proto/pilosa.proto

@@ -1,42 +1,9 @@
 syntax = "proto3";
 package pilosa;
 
-import "public.proto";
+//import "public.proto";
 
-message VDS {
-  string name = 1;
-}
-
-message GetVDSsRequest {
-}
-
-message GetVDSsResponse {
-  repeated VDS vdss = 1;
-}
-
-message GetVDSRequest {
-  string name = 1;
-}
-
-message GetVDSResponse {
-  VDS vds = 1;
-}
-
-message PostVDSRequest {
-  string name = 1;
-  bool keys = 2;
-  bool trackExistence = 3;
-}
-
-message PostVDSResponse {
-}
-
-message DeleteVDSRequest {
-  string name = 1;
-}
-
-message DeleteVDSResponse {
-}
+option go_package = "github.com/pilosa/pilosa/v2/proto";
 
 message QueryPQLRequest {
   string index = 1;
@@ -117,14 +84,10 @@ message IdsOrKeys {
 }
 
 service Pilosa {
-  rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {};
-  rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {};
-  rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {};
-  rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {};
   rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {};
   rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {};
   rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};
   rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {};
   rpc Inspect(InspectRequest) returns (stream RowResponse) {};
-  rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {};
+  //rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {};
 }

proto/vdsm/vdsm.proto

@@ -0,0 +1,89 @@
+syntax = "proto3";
+package vdsm;
+
+option go_package = "github.com/pilosa/pilosa/v2/proto/vdsm";
+
+import "pilosa.proto";
+
+// deprecated
+message DataSource {
+  int64 id = 1;
+  string name = 2;
+  string description = 3;
+  string type = 4;
+  map<string, string> config = 5;
+  string status = 6;
+}
+
+message VDS {
+  string id = 1;
+  string name = 2;
+  string description = 3;
+  string pilosa_index = 4;
+  repeated DataSource datasources = 5;
+}
+
+message GetVDSsRequest {
+}
+
+message GetVDSsResponse {
+  repeated VDS vdss = 1;
+}
+
+message GetVDSRequest {
+  oneof idOrName {
+    string id = 1;
+    string name = 2;
+  }
+}
+
+message GetVDSResponse {
+  VDS vds = 1;
+}
+
+message PostVDSRequest {
+  string definition = 1;
+}
+
+message PostVDSResponse {
+  string id = 1;
+  string uri = 2;
+}
+
+message DeleteVDSRequest {
+  oneof idOrName {
+    string id = 1;
+    string name = 2;
+  }
+}
+
+message DeleteVDSResponse {
+}
+
+// copied from pilosa.proto to use "vds" instead of "index"
+message QueryPQLRequest {
+  string vds = 1;
+  string pql = 2;
+}
+
+// copied from pilosa.proto to use "vds" instead of "index" and "records" instead of "columns"
+message InspectRequest {
+  string vds = 1;
+  pilosa.IdsOrKeys records = 2;
+  repeated string filterFields = 3;
+  uint64 limit = 4;
+  uint64 offset = 5;
+  string query = 6;
+}
+
+service Molecula {
+  rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {};
+  rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {};
+  rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {};
+  rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {};
+  rpc QuerySQL(pilosa.QuerySQLRequest) returns (stream pilosa.RowResponse) {};
+  rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {};
+  rpc QueryPQL(QueryPQLRequest) returns (stream pilosa.RowResponse) {};
+  rpc QueryPQLUnary(QueryPQLRequest) returns (pilosa.TableResponse) {};
+  rpc Inspect(InspectRequest) returns (stream pilosa.RowResponse) {};
+}

rbf/cursor_test.go

@@ -76,8 +76,6 @@ func TestCursor_FirstNext_Quick(t *testing.T) {
 		t.Skip("-short enabled, skipping")
 	} else if is32Bit() {
 		t.Skip("32-bit build, skipping quick check tests")
-	} else if rbf.RaceEnabled {
-		t.Skip("race detection enabled, skipping")
 	}
 
 	const n = 10000
@@ -198,8 +196,6 @@ func TestCursor_LastPrev_Quick(t *testing.T) {
 		t.Skip("-short enabled, skipping")
 	} else if is32Bit() {
 		t.Skip("32-bit build, skipping quick check tests")
-	} else if rbf.RaceEnabled {
-		t.Skip("race detection enabled, skipping")
 	}
 
 	const n = 10000
@@ -312,8 +308,6 @@ func TestCursor_Union(t *testing.T) {
 			t.Skip("-short enabled, skipping")
 		} else if is32Bit() {
 			t.Skip("32-bit build, skipping quick check tests")
-		} else if rbf.RaceEnabled {
-			t.Skip("race detection enabled, skipping")
 		}
 
 		QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
@@ -393,8 +387,6 @@ func TestCursor_Intersect(t *testing.T) {
 			t.Skip("-short enabled, skipping")
 		} else if is32Bit() {
 			t.Skip("32-bit build, skipping quick check tests")
-		} else if rbf.RaceEnabled {
-			t.Skip("race detection enabled, skipping")
 		}
 
 		QuickCheck(t, func(t *testing.T, rand *rand.Rand) {

rbf/db.go

@@ -41,12 +41,14 @@ const (
 type DB struct {
 	data        []byte           // mmap data
 	file        *os.File         // file descriptor
-	segments    []*WALSegment    // write-ahead log
 	rootRecords []*RootRecord    // cached root records
 	pageMap     *immutable.Map   // pgno-to-WALID mapping
 	txs         map[*Tx]struct{} // active transactions
 	opened      bool             // true if open
 
+	wcache   []byte       // wal write cache
+	segments []WALSegment // write-ahead log
+
 	mu     sync.RWMutex // general mutex
 	rwmu   sync.Mutex   // mutex for restricting single writer
 	exclmu sync.RWMutex // mutex for locking out everyone but a single writer
@@ -71,6 +73,7 @@ func NewDB(path string) *DB {
 	db := &DB{
 		txs:     make(map[*Tx]struct{}),
 		pageMap: immutable.NewMap(&uint32Hasher{}),
+		wcache:  make([]byte, MaxWALSegmentFileSize+PageSize),
 		Path:    path,
 		MaxSize: DefaultMaxSize,
 	}
@@ -84,12 +87,10 @@ func (db *DB) DataPath() string {
 
 // WALPath returns the path to the WAL directory.
 func (db *DB) WALPath() string {
-
 	return filepath.Join(db.Path, "wal")
 }
 
 func CreateDirIfNotExist(path string) {
-
 	dir := filepath.Dir(path)
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
 		err = os.MkdirAll(dir, 0755)
@@ -99,10 +100,16 @@ func CreateDirIfNotExist(path string) {
 	}
 }
 
+// TxN returns the number of active transactions.
+func (db *DB) TxN() int {
+	db.mu.RLock()
+	defer db.mu.RUnlock()
+	return len(db.txs)
+}
+
 // Open opens a database with the file specified in Path.
 // Creates a new file if one does not already exist.
 func (db *DB) Open() (err error) {
-
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
@@ -143,7 +150,7 @@ func (db *DB) Open() (err error) {
 	// Open write-ahead log & checkpoint to the end since no transactions are open.
 	if err := db.openWALSegments(); err != nil {
 		return fmt.Errorf("wal open: %w", err)
-	} else if err := db.checkpoint(true); err != nil {
+	} else if err := db.checkpoint(true, &nopLocker{}); err != nil {
 		return fmt.Errorf("checkpoint: %w", err)
 	}
 
@@ -171,27 +178,66 @@ func (db *DB) openWALSegments() error {
 	}
 
 	// Truncate everything after the last successful meta page.
-	if walID, err := db.findLastWALMetaPage(); err != nil {
+	if walID, err := findLastWALMetaPage(db.segments); err != nil {
 		return err
-	} else if err := db.truncateWALAfter(walID); err != nil {
+	} else if db.segments, err = truncateWALAfter(db.segments, walID); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// checkpoint copies pages from WAL segments into the main DB file. This can
+// updateWALSegment updates or adds a segment.
+func (db *DB) updateWALSegment(s WALSegment) {
+	segments := make([]WALSegment, len(db.segments), len(db.segments)+1)
+	copy(segments, db.segments)
+
+	// Find the matching segment using the path.
+	segment := walSegmentByPath(segments, s.Path)
+
+	// Update existing segment if it already exists.
+	// Otherwise append segment to the end.
+	if segment != nil {
+		*segment = s
+	} else {
+		assert(len(segments) == 0 || segments[len(segments)-1].MinWALID < s.MinWALID)
+		segments = append(segments, s)
+	}
+
+	// Replace DB segment list.
+	db.segments = segments
+}
+
+// Checkpoint copies pages from WAL segments into the main DB file. This can
 // only copy pages that aren't in use by an active transaction. The page map
 // is rebuilt as well for all WAL pages still in use.
 //
 // If exclusive is true, all WAL writes are flushed to disk.
-func (db *DB) checkpoint(exclusive bool) error {
-	if !db.opened {
+func (db *DB) Checkpoint() error {
+	return db.checkpoint(false, &db.mu)
+}
+
+// checkpoint moves WAL segments to the main DB file.
+//
+// Note that mu should db.mu when called through DB.Checkpoint() but it
+// can be &nopLocker if called under lock. The external API will be used
+// to periodically checkpoint outside of a transaction and the locking
+// must be used only in the beginning (to obtain the segment list) and at
+// the end (when removing old segments from the list). If the entire function
+// were to obtain a lock then it would block all new read & write transactions.
+func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error {
+	// Obtain a snapshot of WAL segments at the start.
+	mu.Lock()
+	opened := db.opened
+	segments := db.segments
+	mu.Unlock()
+
+	if !opened {
 		return nil
 	}
 
 	// Determine last checkpointed WAL ID.
-	page, err := db.readPage(nil, 0)
+	page, err := db.readDBPage(0)
 	if err != nil {
 		return err
 	}
@@ -206,7 +252,7 @@ func (db *DB) checkpoint(exclusive bool) error {
 	pageMap := immutable.NewMap(&uint32Hasher{})
 	for {
 		// Determine last page of transaction.
-		metaWALID, err := db.findNextWALMetaPage(walID)
+		metaWALID, err := findNextWALMetaPage(segments, walID)
 		if err == io.EOF {
 			break
 		} else if err != nil {
@@ -215,9 +261,9 @@ func (db *DB) checkpoint(exclusive bool) error {
 
 		// Loop over pages in the transaction.
 		for ; walID <= metaWALID; walID++ {
-			canCheckpoint := exclusive || minActiveWALID == 0 || walID <= minActiveWALID
+			canCheckpoint := exclusive || minActiveWALID == 0 || walID < minActiveWALID
 
-			page, err := db.readWALPage(walID)
+			page, err := readWALPage(segments, walID)
 			if err != nil {
 				return err
 			}
@@ -242,13 +288,13 @@ func (db *DB) checkpoint(exclusive bool) error {
 			// Ensure we actually read the bitmap data in when we checkpoint.
 			// NOTE: The walID variable is incremented above in the pgno check.
 			if isBitmapHeader {
-				if page, err = db.readWALPage(walID); err != nil {
+				if page, err = readWALPage(segments, walID); err != nil {
 					return err
 				}
 			}
 
 			// Write page data into main db file.
-			if err := db.writePage(pgno, page); err != nil {
+			if err := db.writeDBPage(pgno, page); err != nil {
 				return err
 			}
 
@@ -259,100 +305,66 @@ func (db *DB) checkpoint(exclusive bool) error {
 		}
 	}
 
+	// Ensure WAL pages are fully copied & synced to DB file.
+	if err := fsync(db.file); err != nil {
+		return fmt.Errorf("db file sync: %w", err)
+	}
+
 	// Remove WAL segments that have been checkpointed.
 	if maxCheckpointedWALID != 0 {
-		for len(db.segments) > 0 {
-			segment := db.segments[0]
+		for _, segment := range segments {
 			if segment.MaxWALID() > maxCheckpointedWALID {
 				break
 			}
 
-			segpath := segment.Path()
-			if err := segment.Close(); err != nil {
-				return err
-			} else if err := os.Remove(segpath); err != nil {
+			if err := func() error {
+				mu.Lock()
+				defer mu.Unlock()
+				return db.removeWALSegment(segment.Path)
+			}(); err != nil {
 				return err
 			}
-			db.segments, db.segments[0] = db.segments[1:], nil
 		}
 	}
 
 	// Ensure all segments are flushed and there is no remapped pages.
 	if exclusive {
+		mu.Lock()
 		assert(len(db.segments) == 0)
 		assert(pageMap.Len() == 0)
+		mu.Unlock()
 	}
 
 	db.pageMap = pageMap
 	return nil
 }
 
-// truncateWALAfter removes all pages in the WAL after walID.
-func (db *DB) truncateWALAfter(walID int64) error {
-	for i := len(db.segments) - 1; i >= 0; i-- {
-		segment := db.segments[i]
-		if segment.MaxWALID() <= walID {
-			break
-		}
-
-		// Drop entire segment if all pages are after WAL ID.
-		if walID < segment.MinWALID() {
+// removeWALSegment closes and deletes the segment with the given path.
+//
+// The DB's segment list is entirely replaced so that transactions with
+// a reference to the old list can continue to use it without a lock.
+func (db *DB) removeWALSegment(path string) error {
+	newSegments := make([]WALSegment, 0, len(db.segments))
+	for _, segment := range db.segments {
+		// Close and remove if path matches.
+		if segment.Path == path {
 			if err := segment.Close(); err != nil {
 				return err
-			} else if err := os.Remove(segment.Path()); err != nil {
+			} else if err := os.Remove(segment.Path); err != nil {
 				return err
 			}
-			db.segments, db.segments[i] = db.segments[:len(db.segments)-1], nil
 			continue
 		}
 
-		// If we only remove some of the WAL pages then truncate and exit
-		// since segments before this will retain all their pages.
-		return segment.TruncateAfter(walID)
+		// Otherwise append to new slice of segments.
+		newSegments = append(newSegments, segment)
 	}
+
+	// Replace entire slice of segments.
+	db.segments = newSegments
 	return nil
 }
 
-func (db *DB) findNextWALMetaPage(walID int64) (metaWALID int64, err error) {
-	maxWALID := db.maxWALID()
-
-	for ; walID <= maxWALID; walID++ {
-		// Read page data from WAL and return if it is a meta page.
-		page, err := db.readWALPage(walID)
-		if err != nil {
-			return walID, err
-		} else if IsMetaPage(page) {
-			return walID, nil
-		}
-
-		// Skip over next page if this is a bitmap header.
-		if IsBitmapHeader(page) {
-			walID++
-		}
-	}
-
-	return -1, io.EOF
-}
-
-func (db *DB) findLastWALMetaPage() (walID int64, err error) {
-	if len(db.segments) == 0 {
-		return 0, nil
-	}
-
-	var maxMetaWALID int64
-	maxWALID := db.maxWALID()
-	for walID := db.minWALID(); walID <= maxWALID; walID++ {
-		if page, err := db.readWALPage(walID); err != nil {
-			return walID, err
-		} else if IsBitmapHeader(page) {
-			walID++ // skip next page for bitmap headers
-		} else if IsMetaPage(page) {
-			maxMetaWALID = walID // save max meta WAL ID
-		}
-	}
-	return maxMetaWALID, nil
-}
-
 // minActiveWALID returns the lowest WAL ID in use by any active transaction.
 // Returns 0 if no transactions are active.
 func (db *DB) minActiveWALID() int64 {
@@ -365,147 +377,8 @@ func (db *DB) minActiveWALID() int64 {
 	return walID
 }
 
-// ActiveWALSegment returns the most recent WAL segment.
-func (db *DB) ActiveWALSegment() *WALSegment {
-	db.mu.RLock()
-	defer db.mu.RUnlock()
-	return db.activeWALSegment()
-}
-
-func (db *DB) activeWALSegment() *WALSegment {
-	if len(db.segments) == 0 {
-		return nil
-	}
-	return db.segments[len(db.segments)-1]
-}
-
-// MinWALID returns the lowest WAL ID available in the WAL.
-func (db *DB) MinWALID() int64 {
-	db.mu.RLock()
-	defer db.mu.RUnlock()
-	return db.minWALID()
-}
-
-func (db *DB) minWALID() int64 {
-	if len(db.segments) == 0 {
-		return 0
-	}
-	return db.segments[0].MinWALID()
-}
-
-// MaxWALID returns the highest WAL ID available in the WAL.
-func (db *DB) MaxWALID() int64 {
-	db.mu.RLock()
-	defer db.mu.RUnlock()
-	return db.maxWALID()
-}
-
-func (db *DB) maxWALID() int64 {
-
-	if len(db.segments) == 0 {
-		return 0
-	}
-	s := db.segments[len(db.segments)-1]
-	return s.MaxWALID()
-}
-
-// WALPageN returns the number of pages across all segments.
-func (db *DB) WALPageN() int64 {
-	db.mu.RLock()
-	defer db.mu.RUnlock()
-
-	var n int64
-	for _, s := range db.segments {
-		n += int64(s.PageN())
-	}
-	return n
-}
-
-// SyncWAL flushes the active segment to disk.
-func (db *DB) SyncWAL() error {
-	if s := db.ActiveWALSegment(); s != nil {
-		return s.Sync()
-	}
-	return nil
-}
-
-// readWALPage reads a single page at the given WAL ID.
-func (db *DB) readWALPage(walID int64) ([]byte, error) {
-	// TODO(BBJ): Binary search for segment.
-	for _, s := range db.segments {
-		if walID >= s.MinWALID() && walID <= s.MaxWALID() {
-			return s.ReadWALPage(walID)
-		}
-	}
-	return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID)
-}
-
-func (db *DB) writeWALPage(page []byte, isMeta bool) (walID int64, err error) {
-	if err := db.ensureWritableWALSegment(); err != nil {
-		return 0, err
-	}
-	return db.activeWALSegment().WriteWALPage(page, isMeta)
-}
-
-func (db *DB) writeBitmapPage(pgno uint32, page []byte) (walID int64, err error) {
-
-	if err := db.ensureWritableWALSegment(); err != nil {
-		return 0, err
-	}
-
-	// Write header page for next bitmap page.
-	buf := make([]byte, PageSize)
-	writePageNo(buf[:], pgno)
-	writeFlags(buf[:], PageTypeBitmapHeader)
-	// TODO(BBJ): Write checksum.
-	if _, err := db.activeWALSegment().WriteWALPage(buf, false); err != nil {
-		return 0, fmt.Errorf("write bitmap header: %w", err)
-	}
-
-	// Write the bitmap page and return its WALID.
-	return db.activeWALSegment().WriteWALPage(page, false)
-}
-
-func (db *DB) ensureWritableWALSegment() error {
-	if s := db.activeWALSegment(); s != nil && s.Size() < MaxWALSegmentFileSize {
-		return nil
-	}
-	return db.addWALSegment()
-}
-
-// addWALSegment appends a new, writable segment and closing an existing segments for write.
-func (db *DB) addWALSegment() error {
-
-	// If we have a current active WAL segment then close it and start the
-	// next segment from the next WAL ID. If there is no existing WAL segments,
-	// read the last checkpointed WAL ID from the DB and start after that.
-	var base int64
-	if s := db.activeWALSegment(); s != nil {
-		base = s.MaxWALID() + 1
-		if err := s.CloseForWrite(); err != nil {
-			return err
-		}
-	} else {
-		page, err := db.readPage(db.pageMap, 0)
-		if err != nil {
-			return err
-		}
-		base = readMetaWALID(page) + 1
-	}
-
-	// Create new segment file.
-	s := NewWALSegment(filepath.Join(db.WALPath(), FormatWALSegmentPath(base)))
-	if err := s.Open(); err != nil {
-		return fmt.Errorf("add wal segment: %w", err)
-	}
-	db.segments = append(db.segments, s)
-
-	return nil
-}
-
 // Close closes the database.
 func (db *DB) Close() (err error) {
-
 	// TODO(bbj): Add wait group to hang until last Tx is complete.
 
 	// Wait for writer lock.
@@ -542,12 +415,12 @@ func (db *DB) Close() (err error) {
 
 // closeWALSegments closes the WAL and all its segments.
 func (db *DB) closeWALSegments() (err error) {
-
 	for _, s := range db.segments {
 		if e := s.Close(); e != nil && err == nil {
 			err = e
 		}
 	}
+	db.segments = nil
 	return err
 }
 
@@ -613,7 +486,6 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) {
 
 // Size returns the size of the database & WAL, in bytes.
 func (db *DB) Size() (int64, error) {
-
 	db.mu.RLock()
 	defer db.mu.RUnlock()
 
@@ -621,38 +493,27 @@ func (db *DB) Size() (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return db.walSize() + fi.Size(), nil
+	return walSize(db.segments) + fi.Size(), nil
 }
 
 // WALSize returns the size of all WAL segments, in bytes.
 func (db *DB) WALSize() int64 {
-
 	db.mu.RLock()
 	defer db.mu.RUnlock()
-	return db.walSize()
-}
-
-func (db *DB) walSize() int64 {
-
-	var sz int64
-	for _, s := range db.segments {
-		sz += s.Size()
-	}
-	return sz
+	return walSize(db.segments)
 }
 
 // WALSegments returns the WAL segments currently on the DB.
-// This should only be used for debugging & testing purposes.
-func (db *DB) WALSegments() []*WALSegment {
-
+func (db *DB) WALSegments() []WALSegment {
 	db.mu.RLock()
 	defer db.mu.RUnlock()
-	return db.segments
+	other := make([]WALSegment, len(db.segments))
+	copy(other, db.segments)
+	return other
 }
 
 // init initializes a new database file.
 func (db *DB) init() error {
-
 	if err := db.initMetaPage(); err != nil {
 		return fmt.Errorf("meta: %w", err)
 	} else if err := db.initRootRecordPage(); err != nil {
@@ -713,6 +574,10 @@ func (db *DB) BeginWithExclusiveLock() (_ *Tx, err error) {
 }
 
 func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) {
+	if exclusive {
+		assert(writable) // exclusive transactions must be writable
+	}
+
 	if exclusive {
 		db.exclmu.Lock()
 	} else {
@@ -749,7 +614,7 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) {
 	// Flush all WAL writes to disk before an exclusive writer so that we can
 	// work directly with the on-disk database.
 	if exclusive {
-		if err := db.checkpoint(true); err != nil {
+		if err := db.checkpoint(true, &nopLocker{}); err != nil {
 			cleanup()
 			return nil, err
 		}
@@ -762,10 +627,21 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) {
 		writable:    writable,
 		exclusive:   exclusive,
 	}
+	if writable {
+		tx.wcache = db.wcache[:0]
+	}
+
+	// Copy list of WAL segments so they can be altered by the tx.
+	// Add last segment to the list of segments that will be updated/added.
+	if len(db.segments) != 0 {
+		tx.segments = make([]WALSegment, len(db.segments))
+		copy(tx.segments, db.segments)
+		tx.updatedSegmentPaths = []string{tx.segments[len(tx.segments)-1].Path}
+	}
 
 	// Copy meta page into transaction's buffer.
 	// This page is only written at the end of a dirty transaction.
-	page, err := db.readPage(db.pageMap, 0)
+	page, err := db.readMetaPage()
 	if err != nil {
 		tx.Rollback()
 		return nil, err
@@ -802,8 +678,8 @@ func (db *DB) removeTx(tx *Tx) error {
 	// Write pages from WAL to DB.
 	// TODO(bbj): Move this to an async goroutine.
 	if tx.writable {
-		if err := db.checkpoint(false); err != nil {
-			return err
+		if err := db.checkpoint(false, &nopLocker{}); err != nil {
+			return fmt.Errorf("checkpoint: %w", err)
 		}
 	}
 
@@ -824,21 +700,25 @@ func (db *DB) Check() error {
 	return tx.Check()
 }
 
-// writePage writes a page to the data file.
-func (db *DB) writePage(pgno uint32, page []byte) error {
+// writeDBPage writes a page to the data file.
+func (db *DB) writeDBPage(pgno uint32, page []byte) error {
 	_, err := db.file.WriteAt(page, int64(pgno)*PageSize)
 	return err
 }
 
-func (db *DB) readPage(pageMap *immutable.Map, pgno uint32) ([]byte, error) {
-	// Check if page is currently in WAL.
-	if pageMap != nil {
-		if walID, ok := pageMap.Get(pgno); ok {
-			return db.readWALPage(walID.(int64))
-		}
-	}
-
-	// Otherwise read from the data file.
+func (db *DB) readDBPage(pgno uint32) ([]byte, error) {
 	offset := int64(pgno) * PageSize
 	return db.data[offset : offset+PageSize], nil
 }
+
+func (db *DB) readMetaPage() ([]byte, error) {
+	if walID, ok := db.pageMap.Get(uint32(0)); ok {
+		return readWALPage(db.segments, walID.(int64))
+	}
+	return db.readDBPage(0)
+}
+
+type nopLocker struct{}
+
+func (*nopLocker) Lock()   {}
+func (*nopLocker) Unlock() {}

rbf/db_test.go

@@ -112,10 +112,11 @@ func TestDB_Recovery(t *testing.T) {
 		tx1.Rollback()
 
 		// Close database & truncate WAL to remove commit page & bitmap data page.
-		segment := db.ActiveWALSegment()
+		segments := db.WALSegments()
+		segment := segments[len(segments)-1]
 		if err := db.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Truncate(segment.Path(), segment.Size()-(2*rbf.PageSize)); err != nil {
+		} else if err := os.Truncate(segment.Path, segment.Size()-(2*rbf.PageSize)); err != nil {
 			t.Fatal(err)
 		}
 

rbf/os.go

@@ -19,4 +19,4 @@ package rbf
 // DefaultMaxSize is the default mmap size and therefore the maximum allowed
 // size of the database. The size can be increased by updating the DB.MaxSize
 // and reopening the database. This setting mainly affects virtual space usage.
-const DefaultMaxSize = 100 * (1 << 30) // 100GB
+const DefaultMaxSize = 4 * (1 << 30)

rbf/rbf.go

@@ -91,6 +91,11 @@ var (
 // Debug is just a temporary flag used for debugging.
 var Debug bool
 
+// Testing constants.
+const (
+	SyncEnabled = true
+)
+
 // Magic32 returns the magic bytes as a big endian encoded uint32.
 func Magic32() uint32 {
 	return binary.BigEndian.Uint32([]byte(Magic))
@@ -648,3 +653,26 @@ func RowValues(b []uint64) []uint64 {
 // 	_, file, line, _ := runtime.Caller(skip + 1)
 // 	return fmt.Sprintf("%s:%d", file, line)
 // }
+
+// truncate truncates the file at path to sz bytes. File must exist.
+func truncate(path string, sz int64) error {
+	f, err := os.OpenFile(path, os.O_WRONLY, 0666)
+	if err != nil {
+		return fmt.Errorf("open file: %w", err)
+	}
+	defer f.Close()
+
+	if err := f.Truncate(sz); err != nil {
+		return fmt.Errorf("truncate: %w", err)
+	} else if err := fsync(f); err != nil {
+		return fmt.Errorf("sync: %w", err)
+	}
+	return f.Close()
+}
+
+func fsync(f *os.File) error {
+	if !SyncEnabled {
+		return nil
+	}
+	return f.Sync()
+}

rbf/rbf_norace.go

@@ -1,20 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build !race
-
-package rbf
-
-// RaceEnabled is true if the -race flag is enabled.
-const RaceEnabled = false

rbf/rbf_race.go

@@ -1,20 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build race
-
-package rbf
-
-// RaceEnabled is true if the -race flag is enabled.
-const RaceEnabled = true

rbf/rbf_test.go

@@ -86,6 +86,8 @@ func MustCloseDB(tb testing.TB, db *rbf.DB) {
 	tb.Helper()
 	if err := db.Check(); err != nil && err != rbf.ErrClosed {
 		tb.Fatal(err)
+	} else if n := db.TxN(); n != 0 {
+		tb.Fatalf("db still has %d active transactions; must closed before closing db", n)
 	} else if err := db.Close(); err != nil && err != rbf.ErrClosed {
 		tb.Fatal(err)
 	} else if err := os.RemoveAll(db.Path); err != nil {

rbf/tx.go

@@ -17,9 +17,9 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"os"
+	"path/filepath"
 	"sort"
-
-	//"strconv"
 	"strings"
 	"sync"
 
@@ -33,15 +33,19 @@ var _ = txkey.ToString
 
 // Tx represents a transaction.
 type Tx struct {
-	mu          sync.RWMutex
-	db          *DB            // parent db
-	meta        [PageSize]byte // copy of current meta page
-	walID       int64          // max WAL ID at start of tx
-	rootRecords []*RootRecord  // read-only cache of root records
-	pageMap     *immutable.Map // mapping of database pages to WAL IDs
-	writable    bool           // if true, tx can write
-	exclusive   bool           // if true, tx writes directly to db file (no wal)
-	dirty       bool           // if true, changes have been made
+	mu                  sync.RWMutex
+	db                  *DB            // parent db
+	segments            []WALSegment   // copy of WAL segments
+	updatedSegmentPaths []string       // updated or added segment paths
+	meta                [PageSize]byte // copy of current meta page
+	walID               int64          // max WAL ID at start of tx
+	rootRecords         []*RootRecord  // read-only cache of root records
+	pageMap             *immutable.Map // mapping of database pages to WAL IDs
+	writable            bool           // if true, tx can write
+	exclusive           bool           // if true, tx writes directly to db file (no wal)
+	dirty               bool           // if true, changes have been made
+
+	wcache []byte // write cache
 
 	// If Rollback() has already completed, don't do it again.
 	// Note db == nil means that commit has already been done.
@@ -76,7 +80,7 @@ func (tx *Tx) Commit() error {
 	if tx.dirty {
 		if err := tx.writeMetaPage(MetaPageFlagCommit); err != nil {
 			return err
-		} else if err := tx.db.SyncWAL(); err != nil {
+		} else if err := tx.flushWALWriter(); err != nil {
 			return err
 		}
 
@@ -89,6 +93,11 @@ func (tx *Tx) Commit() error {
 		tx.db.mu.Lock()
 		tx.db.rootRecords = tx.rootRecords
 		tx.db.pageMap = tx.pageMap
+		for _, path := range tx.updatedSegmentPaths {
+			segment := walSegmentByPath(tx.segments, path)
+			assert(segment != nil)
+			tx.db.updateWALSegment(*segment)
+		}
 		tx.db.mu.Unlock()
 	}
 
@@ -112,11 +121,12 @@ func (tx *Tx) Rollback() {
 
 	// TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen?
 
-	// Remove all WAL pages that have been written by this transaction.
 	if tx.dirty {
-		if err := tx.db.truncateWALAfter(tx.walID); err != nil {
-			panic(err)
+		if _, err := truncateWALAfter(tx.segments, tx.walID); err != nil {
+			panicOn(err)
 		}
+		tx.segments = nil
+		tx.updatedSegmentPaths = nil
 	}
 
 	// Disconnect transaction from DB.
@@ -915,11 +925,28 @@ func (tx *Tx) readPage(pgno uint32) ([]byte, error) {
 		return tx.meta[:], nil
 	}
 
+	// Verify page number requested is within current size of database.
 	pageN := readMetaPageN(tx.meta[:])
 	if pgno > pageN {
 		return nil, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN)
 	}
-	return tx.db.readPage(tx.pageMap, pgno)
+
+	// Check if page is remapped.
+	if walID, ok := tx.pageMap.Get(pgno); ok {
+		walID64 := walID.(int64)
+
+		// Read from write cache if not yet flushed to disk.
+		maxWALID := activeWALSegment(tx.segments).MaxWALID()
+		if walID64 > maxWALID {
+			offset := (walID64 - maxWALID - 1) * PageSize
+			return tx.wcache[offset : offset+PageSize], nil
+		}
+
+		// Otherwise return remapped page from WAL segment.
+		return readWALPage(tx.segments, walID64)
+	}
+
+	return tx.db.readDBPage(pgno)
 }
 
 func (tx *Tx) writePage(page []byte) error {
@@ -928,11 +955,11 @@ func (tx *Tx) writePage(page []byte) error {
 
 	// If we are running in exclusive mode, directly write page to database.
 	if tx.exclusive {
-		return tx.db.writePage(readPageNo(page), page)
+		return tx.db.writeDBPage(readPageNo(page), page)
 	}
 
 	// Write page to WAL and obtain position in WAL.
-	walID, err := tx.db.writeWALPage(page, false)
+	walID, err := tx.writeWALPage(page, false)
 	if err != nil {
 		return err
 	}
@@ -948,11 +975,11 @@ func (tx *Tx) writeBitmapPage(pgno uint32, page []byte) error {
 
 	// If we are running in exclusive mode, directly write page to database.
 	if tx.exclusive {
-		return tx.db.writePage(pgno, page)
+		return tx.db.writeDBPage(pgno, page)
 	}
 
 	// Write bitmap to WAL and obtain WAL position of the actual page data (not the prefix page).
-	walID, err := tx.db.writeBitmapPage(pgno, page)
+	walID, err := tx.writeBitmapWALPage(pgno, page)
 	if err != nil {
 		return err
 	}
@@ -968,11 +995,11 @@ func (tx *Tx) writeMetaPage(flag uint32) error {
 
 	// If we are running in exclusive mode, directly write page to database.
 	if tx.exclusive {
-		return tx.db.writePage(0, tx.meta[:])
+		return tx.db.writeDBPage(0, tx.meta[:])
 	}
 
 	// Write page to WAL and obtain position in WAL.
-	walID, err := tx.db.writeWALPage(tx.meta[:], true)
+	walID, err := tx.writeWALPage(tx.meta[:], true)
 	if err != nil {
 		return err
 	}
@@ -1562,3 +1589,112 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
 	}
 	return
 }
+
+func (tx *Tx) flushWALWriter() error {
+	// Ignore if we have no data in the write cache.
+	if len(tx.wcache) == 0 {
+		return nil
+	}
+
+	// Determine active WAL segment.
+	assert(len(tx.segments) != 0)
+	segment := &tx.segments[len(tx.segments)-1]
+
+	// Open write handle to active segment.
+	w, err := os.OpenFile(segment.Path, os.O_WRONLY, 0666)
+	if err != nil {
+		return fmt.Errorf("open wal segment write handle: %w", err)
+	}
+	defer w.Close()
+
+	// Flush cache to writer.
+	if _, err := w.WriteAt(tx.wcache, int64(segment.PageN)*PageSize); err != nil {
+		return fmt.Errorf("write wal segment: %w", err)
+	} else if err := fsync(w); err != nil {
+		return fmt.Errorf("sync wal segment: %w", err)
+	} else if err := w.Close(); err != nil {
+		return fmt.Errorf("close wal segment: %w", err)
+	}
+
+	// Increase the size of the last WAL segment & clear cache.
+	assert(len(tx.wcache)%PageSize == 0)
+	segment.PageN += len(tx.wcache) / PageSize
+	tx.wcache = tx.wcache[:0]
+
+	return nil
+}
+
+func (tx *Tx) writeWALPage(page []byte, isMeta bool) (walID int64, err error) {
+	if err := tx.ensureWritableWALSegment(); err != nil {
+		return 0, err
+	}
+
+	// Determine next WAL ID from cached meta page.
+	walID = readMetaWALID(tx.meta[:]) + 1
+
+	// Update WAL ID on cached meta page.
+	writeMetaWALID(tx.meta[:], walID)
+
+	// Append write to write buffer.
+	tx.wcache = append(tx.wcache, page...)
+
+	return walID, nil
+}
+
+func (tx *Tx) writeBitmapWALPage(pgno uint32, page []byte) (walID int64, err error) {
+	if err := tx.ensureWritableWALSegment(); err != nil {
+		return 0, err
+	}
+
+	// Write header page for next bitmap page.
+	buf := make([]byte, PageSize)
+	writePageNo(buf[:], pgno)
+	writeFlags(buf[:], PageTypeBitmapHeader)
+	// TODO(BBJ): Write checksum.
+	if _, err := tx.writeWALPage(buf, false); err != nil {
+		return 0, fmt.Errorf("write bitmap header: %w", err)
+	}
+
+	// Write the bitmap page and return its WALID.
+	return tx.writeWALPage(page, false)
+}
+
+func (tx *Tx) ensureWritableWALSegment() error {
+	// Ignore if we still have space in the write cache.
+	writeCacheSize := int64(len(tx.wcache))
+	if len(tx.segments) != 0 && activeWALSegment(tx.segments).Size()+writeCacheSize < MaxWALSegmentFileSize {
+		return nil
+	}
+
+	// Flush write cache out to file before adding new segment.
+	if err := tx.flushWALWriter(); err != nil {
+		return err
+	}
+
+	// If we have a current active WAL segment then close it and start the
+	// next segment from the next WAL ID. If there is no existing WAL segments,
+	// read the last checkpointed WAL ID from the DB and start after that.
+	var base int64
+	if len(tx.segments) != 0 {
+		base = activeWALSegment(tx.segments).MaxWALID() + 1
+	} else {
+		page, err := tx.readPage(0)
+		if err != nil {
+			return err
+		}
+		base = readMetaWALID(page) + 1
+	}
+
+	// Create new segment file.
+	s := NewWALSegment(filepath.Join(tx.db.WALPath(), FormatWALSegmentPath(base)))
+	if err := s.Open(); err != nil {
+		return fmt.Errorf("add wal segment: %w", err)
+	}
+
+	// Track all segments that need to be added back to DB.
+	// The DB can remove segments in the background so we don't want to replace.
+	tx.segments = append(tx.segments, s)
+	tx.updatedSegmentPaths = append(tx.updatedSegmentPaths, s.Path)
+
+	return nil
+}

rbf/tx_test.go

@@ -218,8 +218,6 @@ func TestTx_Add_Quick(t *testing.T) {
 		t.Skip("-short enabled, skipping")
 	} else if is32Bit() {
 		t.Skip("32-bit build, skipping quick check tests")
-	} else if rbf.RaceEnabled {
-		t.Skip("race detection enabled, skipping")
 	}
 
 	QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
@@ -257,8 +255,6 @@ func TestTx_AddRemove_Quick(t *testing.T) {
 		t.Skip("-short enabled, skipping")
 	} else if is32Bit() {
 		t.Skip("32-bit build, skipping quick check tests")
-	} else if rbf.RaceEnabled {
-		t.Skip("race detection enabled, skipping")
 	}
 
 	QuickCheck(t, func(t *testing.T, rand *rand.Rand) {

rbf/wal.go

@@ -16,9 +16,9 @@ package rbf
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
-	"sync"
 	"syscall"
 
 	"github.com/pilosa/pilosa/v2/syswrap"
@@ -26,66 +26,40 @@ import (
 
 // WALSegment represents a single file in the WAL.
 type WALSegment struct {
-	mu         sync.RWMutex
-	minWALID   int64    // base WALID; calculated from path
-	path       string   // path to file
-	w          *os.File // write handle
-	data       []byte   // read-only mmap data
-	writeCache []byte   // write buffer
-	pageN      int      // number of written pages
+	Path     string // path to file
+	MinWALID int64  // base WALID; calculated from path
+	PageN    int    // number of written pages
+
+	data []byte // read-only mmap data
 }
 
 // NewWALSegment returns a new instance of WALSegment for a given path.
-func NewWALSegment(path string) *WALSegment {
-	return &WALSegment{
-		path: path,
+func NewWALSegment(path string) WALSegment {
+	return WALSegment{
+		Path: path,
 	}
 }
 
-// Path returns the path the segment was initialized with.
-func (s *WALSegment) Path() string { return s.path }
-
-// MinWALID returns the initial WAL ID of the segment. Only available after Open().
-func (s *WALSegment) MinWALID() int64 {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.minWALID
-}
-
 // MaxWALID returns the maximum WAL ID of the segment. Only available after Open().
-func (s *WALSegment) MaxWALID() int64 {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.minWALID + int64(s.pageN) - 1
-}
-
-// PageN returns the number of pages in the segment.
-func (s *WALSegment) PageN() int {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.pageN
+func (s WALSegment) MaxWALID() int64 {
+	return s.MinWALID + int64(s.PageN) - 1
 }
 
 // Size returns the current size of the segment, in bytes.
-func (s *WALSegment) Size() int64 {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return int64(s.pageN) * PageSize
+func (s WALSegment) Size() int64 {
+	return int64(s.PageN) * PageSize
 }
 
 func (s *WALSegment) Open() (err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	// Extract base WAL ID and validate path.
-	if s.minWALID, err = ParseWALSegmentPath(s.path); err != nil {
+	if s.MinWALID, err = ParseWALSegmentPath(s.Path); err != nil {
 		return err
 	}
 
 	// Determine file size & create if necessary.
 	var sz int64
-	if fi, err := os.Stat(s.path); os.IsNotExist(err) {
-		if f, err := os.OpenFile(s.path, os.O_RDWR|os.O_CREATE, 0666); err != nil {
+	if fi, err := os.Stat(s.Path); os.IsNotExist(err) {
+		if f, err := os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE, 0666); err != nil {
 			return fmt.Errorf("touch wal segment file: %w", err)
 		} else if err := f.Close(); err != nil {
 			return fmt.Errorf("close touched wal segment file: %w", err)
@@ -97,11 +71,11 @@ func (s *WALSegment) Open() (err error) {
 	}
 
 	// Determine page count & truncate if a partial page is written.
-	s.pageN = int(sz / PageSize)
+	s.PageN = int(sz / PageSize)
 	if sz%PageSize != 0 {
-		sz = int64(s.pageN * PageSize)
-		if err := os.Truncate(s.path, sz); err != nil {
-			return fmt.Errorf("truncate wal segment file: %w", err)
+		sz = int64(s.PageN * PageSize)
+		if err := truncate(s.Path, sz); err != nil {
+			return fmt.Errorf("truncate wal file: %w", err)
 		}
 	}
 
@@ -113,7 +87,7 @@ func (s *WALSegment) Open() (err error) {
 	}
 
 	// Open file as a read-only memory map.
-	if f, err := os.OpenFile(s.path, os.O_RDONLY, 0666); err != nil {
+	if f, err := os.OpenFile(s.Path, os.O_RDONLY, 0666); err != nil {
 		return fmt.Errorf("open wal segment file: %w", err)
 	} else if s.data, err = syswrap.Mmap(int(f.Fd()), 0, int(mmapSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
 		f.Close()
@@ -127,12 +101,6 @@ func (s *WALSegment) Open() (err error) {
 
 // Close closes the write handle and the read-only mmap.
 func (s *WALSegment) Close() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if err := s.closeForWrite(); err != nil {
-		return err
-	}
 	if s.data != nil {
 		if err := syswrap.Munmap(s.data); err != nil {
 			return err
@@ -142,144 +110,148 @@ func (s *WALSegment) Close() error {
 	return nil
 }
 
-// CloseForWrite closes the write handle, if initialized.
-func (s *WALSegment) CloseForWrite() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.closeForWrite()
-}
-
-func (s *WALSegment) closeForWrite() error {
-	// Ensure write buffer is flushed out.
-	if err := s.sync(); err != nil {
-		return err
-	}
-	s.writeCache = nil
-
-	// Close underlying file writer.
-	if s.w != nil {
-		if err := s.w.Close(); err != nil {
-			return err
-		}
-		s.w = nil
-	}
-	return nil
-}
-
 // ReadWALPage reads a single page at the given WAL ID.
 func (s *WALSegment) ReadWALPage(walID int64) ([]byte, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
 	// Ensure requested ID is contained in this file.
-	if walID < s.minWALID || walID > s.minWALID+int64(s.pageN) {
-		return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.minWALID, s.pageN)
+	if walID < s.MinWALID || walID > s.MinWALID+int64(s.PageN) {
+		return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.MinWALID, s.PageN)
 	}
 
-	offset := (walID - s.minWALID) * PageSize
-
-	// If offset is within write buffer, return from write buffer.
-	writeBufferOffset := int64((s.pageN * PageSize) - len(s.writeCache))
-	if offset >= writeBufferOffset {
-		buf := s.writeCache[offset-writeBufferOffset:]
-		return buf[:PageSize:PageSize], nil
-	}
-
-	// Otherwise return from on-disk mmap.
+	offset := (walID - s.MinWALID) * PageSize
 	return s.data[offset : offset+PageSize], nil
 }
 
-// WriteWALPage writes a single page to the WAL segment and returns its WAL identifier.
-func (s *WALSegment) WriteWALPage(page []byte, isMeta bool) (walID int64, err error) {
-	assert(len(page) == PageSize) // invalid page size
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	// Initialize write file handle if not yet initialized.
-	if s.w == nil {
-		if s.w, err = os.OpenFile(s.path, os.O_WRONLY, 0666); err != nil {
-			return 0, fmt.Errorf("open wal segment write handle: %w", err)
+func walSegmentByPath(segments []WALSegment, path string) *WALSegment {
+	for i := range segments {
+		if segments[i].Path == path {
+			return &segments[i]
 		}
 	}
-
-	// Determine current WAL position.
-	walID = s.minWALID + int64(s.pageN)
-
-	// Write WAL ID if this is a meta page.
-	if isMeta {
-		writeMetaWALID(page, walID)
-		// TODO: Write meta page checksum
-	}
-
-	// Append write to write buffer & increment page count.
-	if s.writeCache == nil {
-		s.writeCache = make([]byte, 0, MaxWALSegmentFileSize+PageSize)
-	}
-	s.writeCache = append(s.writeCache, page...)
-	s.pageN++
-
-	return walID, nil
-}
-
-// TruncateAfter removes all pages after a given WAL ID.
-func (s *WALSegment) TruncateAfter(walID int64) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	// Ensure this is a partial truncation. Full truncation of a segment
-	// should be performed by the DB since it needs to remove the segment.
-	assert(walID > s.minWALID)
-
-	// Update to new page size.
-	newPageN := int((walID - s.minWALID) + 1) // new page count of segment
-	truncPageN := s.pageN - newPageN          // number of pages removed
-	s.pageN = newPageN
-
-	// Check to see if we are only truncating from the write cache.
-	writeCachePageN := len(s.writeCache) / PageSize
-	if truncPageN <= int(writeCachePageN) {
-		s.writeCache = s.writeCache[:(writeCachePageN-truncPageN)*PageSize]
-		return nil
-	}
-
-	// Clear write cache.
-	s.writeCache = s.writeCache[:0]
-
-	// Remove on disk pages.
-	return os.Truncate(s.path, int64(s.pageN)*PageSize)
-}
-
-// Flush flushes the write buffer to the OS cache.
-func (s *WALSegment) Flush() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.flush()
-}
-
-func (s *WALSegment) flush() error {
-	if _, err := s.w.WriteAt(s.writeCache, int64((s.pageN*PageSize)-len(s.writeCache))); err != nil {
-		return fmt.Errorf("wal segment write: %w", err)
-	}
-	s.writeCache = s.writeCache[:0]
 	return nil
 }
 
-// Sync flushes the write buffer and invokes a file sync to flush data to disk.
-func (s *WALSegment) Sync() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.sync()
+func activeWALSegment(segments []WALSegment) WALSegment {
+	if len(segments) == 0 {
+		return WALSegment{}
+	}
+	return segments[len(segments)-1]
 }
 
-func (s *WALSegment) sync() error {
-	if s.w == nil {
-		return nil
+func minWALID(segments []WALSegment) int64 {
+	if len(segments) == 0 {
+		return 0
 	}
-	if err := s.flush(); err != nil {
-		return err
+	return segments[0].MinWALID
+}
+
+func maxWALID(segments []WALSegment) int64 {
+	if len(segments) == 0 {
+		return 0
+	}
+	s := segments[len(segments)-1]
+	return s.MaxWALID()
+}
+
+func walSize(segments []WALSegment) int64 {
+	var sz int64
+	for _, s := range segments {
+		sz += s.Size()
+	}
+	return sz
+}
+
+// readWALPage reads a single page at the given WAL ID.
+func readWALPage(segments []WALSegment, walID int64) ([]byte, error) {
+	// TODO(BBJ): Binary search for segment.
+	for _, s := range segments {
+		if walID >= s.MinWALID && walID <= s.MaxWALID() {
+			return s.ReadWALPage(walID)
+		}
+	}
+	return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID)
+}
+
+func findNextWALMetaPage(segments []WALSegment, walID int64) (metaWALID int64, err error) {
+	maxWALID := maxWALID(segments)
+
+	for ; walID <= maxWALID; walID++ {
+		// Read page data from WAL and return if it is a meta page.
+		page, err := readWALPage(segments, walID)
+		if err != nil {
+			return walID, err
+		} else if IsMetaPage(page) {
+			return walID, nil
+		}
+
+		// Skip over next page if this is a bitmap header.
+		if IsBitmapHeader(page) {
+			walID++
+		}
+	}
+
+	return -1, io.EOF
+}
+
+func findLastWALMetaPage(segments []WALSegment) (walID int64, err error) {
+	if len(segments) == 0 {
+		return 0, nil
+	}
+
+	var maxMetaWALID int64
+	maxWALID := maxWALID(segments)
+	for walID := minWALID(segments); walID <= maxWALID; walID++ {
+		if page, err := readWALPage(segments, walID); err != nil {
+			return walID, err
+		} else if IsBitmapHeader(page) {
+			walID++ // skip next page for bitmap headers
+		} else if IsMetaPage(page) {
+			maxMetaWALID = walID // save max meta WAL ID
+		}
+	}
+	return maxMetaWALID, nil
+}
+
+// truncateWALAfter removes all pages in the WAL after walID.
+func truncateWALAfter(segments []WALSegment, walID int64) ([]WALSegment, error) {
+	var newSegments []WALSegment
+
+	for i := range segments {
+		segment := &segments[i]
+
+		// Append entire segment if WAL range entirely before target WAL ID.
+		if walID > segment.MaxWALID() {
+			newSegments = append(newSegments, *segment)
+			continue
+		}
+
+		// If we only remove some of the WAL pages then truncate and append.
+		if segment.MinWALID < walID {
+			newSegment := *segment
+			newSegment.PageN = int((walID - newSegment.MinWALID) + 1)
+
+			if err := truncate(newSegment.Path, int64(newSegment.PageN)*PageSize); err != nil {
+				return segments, err
+			}
+			newSegments = append(newSegments, newSegment)
+			continue
+		}
+
+		// Drop entire segment if all pages are after WAL ID.
+		if err := segment.Close(); err != nil {
+			return segments, err
+		} else if err := os.Remove(segment.Path); err != nil {
+			return segments, err
+		}
+	}
+
+	return newSegments, nil
+}
+
+func DumpWALSegments(segments []WALSegment) {
+	fmt.Printf("WAL (%d segments)\n", len(segments))
+	for i, s := range segments {
+		fmt.Printf("[%d] WALIDs=(%d-%d) PageN=%d\n", i, s.MinWALID, s.MaxWALID(), s.PageN)
 	}
-	return s.w.Sync()
 }
 
 // FormatWALSegmentPath returns a path for a WAL segment using a WAL ID.

rbf/wal_test.go

@@ -15,10 +15,10 @@
 package rbf_test
 
 import (
-	"bytes"
-	"encoding/hex"
+	// "bytes"
+	// "encoding/hex"
 	"io/ioutil"
-	"math/rand"
+	// "math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -30,9 +30,9 @@ func TestWALSegment_Open(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		s := MustOpenWALSegment(t, 10)
 		defer MustCloseWALSegment(t, s)
-		if got, want := s.MinWALID(), int64(10); got != want {
+		if got, want := s.MinWALID, int64(10); got != want {
 			t.Fatalf("Base()=%d, want %d", got, want)
-		} else if got, want := s.PageN(), 0; got != want {
+		} else if got, want := s.PageN, 0; got != want {
 			t.Fatalf("PageN()=%d, want %d", got, want)
 		}
 	})
@@ -40,6 +40,7 @@ func TestWALSegment_Open(t *testing.T) {
 	// TODO(BBJ): Test open w/ partially written pages.
 }
 
+/*
 func TestWALSegment_WritePage(t *testing.T) {
 	rand := rand.New(rand.NewSource(0))
 	s := MustOpenWALSegment(t, 10)
@@ -84,6 +85,7 @@ func TestWALSegment_WritePage(t *testing.T) {
 		t.Fatal("unexpected second page")
 	}
 }
+*/
 
 func TestFormatWALSegmentPath(t *testing.T) {
 	if got, want := rbf.FormatWALSegmentPath(1234), "00000000000004d2.wal"; got != want {
@@ -107,6 +109,7 @@ func TestParseWALSegmentPath(t *testing.T) {
 	})
 }
 
+/*
 func BenchmarkWALSegment_WriteWALPage(b *testing.B) {
 	b.Run("8KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 8*(1<<10)) })
 	b.Run("16KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 16*(1<<10)) })
@@ -147,9 +150,10 @@ func benchmarkWALSegment_WriteWALPage(b *testing.B, flushSize int) {
 
 	b.SetBytes(rbf.MaxWALSegmentFileSize)
 }
+*/
 
 // MustOpenWALSegment opens a WAL segment in a temporary path. Fails on error.
-func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment {
+func MustOpenWALSegment(tb testing.TB, walID int64) rbf.WALSegment {
 	tb.Helper()
 
 	dir, err := ioutil.TempDir("", "")
@@ -169,11 +173,11 @@ func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment {
 }
 
 // MustCloseWALSegment closes s. Fails on error.
-func MustCloseWALSegment(tb testing.TB, s *rbf.WALSegment) {
+func MustCloseWALSegment(tb testing.TB, s rbf.WALSegment) {
 	tb.Helper()
 	if err := s.Close(); err != nil {
 		tb.Fatal(err)
-	} else if err := os.Remove(s.Path()); err != nil {
+	} else if err := os.Remove(s.Path); err != nil {
 		tb.Fatal(err)
 	}
 }

server.go

@@ -269,6 +269,14 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption {
 	}
 }
 
+// OptServerClusterName sets the human-readable cluster name.
+func OptServerClusterName(name string) ServerOption {
+	return func(s *Server) error {
+		s.cluster.Name = name
+		return nil
+	}
+}
+
 // OptServerSerializer is a functional option on Server
 // used to set the serializer.
 func OptServerSerializer(ser Serializer) ServerOption {
@@ -534,10 +542,12 @@ func (s *Server) Close() error {
 	s.wg.Wait()
 
 	var errh error
+	var errhs error
 	var errc error
 	if s.cluster != nil {
 		errc = s.cluster.close()
 	}
+	errhs = s.syncer.stopTranslationSync()
 	if s.holder != nil {
 		errh = s.holder.Close()
 	}
@@ -553,6 +563,9 @@ func (s *Server) Close() error {
 	if errh != nil {
 		return errors.Wrap(errh, "closing holder")
 	}
+	if errhs != nil {
+		return errors.Wrap(errhs, "terminating holder translation sync")
+	}
 	if errc != nil {
 		return errors.Wrap(errc, "closing cluster")
 	}
@@ -566,7 +579,7 @@ func (s *Server) loadNodeID() string {
 	if s.nodeID != "" {
 		return s.nodeID
 	}
-	nodeID, err := s.holder.loadNodeID()
+	nodeID, err := s.holder.LoadNodeID()
 	if err != nil {
 		s.logger.Printf("loading NodeID: %v", err)
 		return s.nodeID

server/cluster_test.go

@@ -25,9 +25,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pilosa/pilosa/v2/server"
-
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pilosa/pilosa/v2/test"
 	"golang.org/x/sync/errgroup"
 )

server/config.go

@@ -122,6 +122,7 @@ type Config struct {
 		Coordinator bool     `toml:"coordinator"`
 		ReplicaN    int      `toml:"replicas"`
 		Hosts       []string `toml:"hosts"`
+		Name        string   `toml:"name"`
 		// TODO(2.0) move this out of cluster. (why is it here??)
 		LongQueryTime toml.Duration `toml:"long-query-time"`
 	} `toml:"cluster"`

server/grpc.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/logger"
 	pb "github.com/pilosa/pilosa/v2/proto"
+	vdsm_pb "github.com/pilosa/pilosa/v2/proto/vdsm"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
@@ -125,48 +126,6 @@ func errToStatusError(err error) error {
 	return status.Error(codes.Unknown, err.Error())
 }
 
-// GetVDSs returns a single VDS given a name
-func (h *GRPCHandler) GetVDS(ctx context.Context, req *pb.GetVDSRequest) (*pb.GetVDSResponse, error) {
-	// TODO: Return all schema information associated with the VDS.
-	// It's obviously not very useful to return the same data as given.
-	schema := h.api.Schema(ctx)
-	for _, index := range schema {
-		if req.Name == index.Name {
-			return &pb.GetVDSResponse{Vds: &pb.VDS{Name: index.Name}}, nil
-		}
-	}
-	return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", req.Name))
-}
-
-// GetVDSs returns a list of all VDSs
-func (h *GRPCHandler) GetVDSs(ctx context.Context, req *pb.GetVDSsRequest) (*pb.GetVDSsResponse, error) {
-	schema := h.api.Schema(ctx)
-	vdss := make([]*pb.VDS, len(schema))
-	for i, index := range schema {
-		vdss[i] = &pb.VDS{Name: index.Name}
-	}
-	return &pb.GetVDSsResponse{Vdss: vdss}, nil
-}
-
-// PostVDS creates a new VDS
-func (h *GRPCHandler) PostVDS(ctx context.Context, req *pb.PostVDSRequest) (*pb.PostVDSResponse, error) {
-	opts := pilosa.IndexOptions{Keys: req.Keys, TrackExistence: req.TrackExistence}
-	_, err := h.api.CreateIndex(ctx, req.Name, opts)
-	if err != nil {
-		return nil, errToStatusError(err)
-	}
-	return &pb.PostVDSResponse{}, nil
-}
-
-// DeleteVDS deletes a VDS
-func (h *GRPCHandler) DeleteVDS(ctx context.Context, req *pb.DeleteVDSRequest) (*pb.DeleteVDSResponse, error) {
-	err := h.api.DeleteIndex(ctx, req.Name)
-	if err != nil {
-		return nil, errToStatusError(err)
-	}
-	return &pb.DeleteVDSResponse{}, nil
-}
-
 func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) {
 	h.stats.Count(pilosa.MetricSqlQueries, 1, 1)
 	return execSQL(ctx, h.api, h.logger, queryStr)
@@ -292,6 +251,103 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest
 	return table, errToStatusError(nil)
 }
 
+// VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM.
+type VDSMGRPCHandler struct {
+	grpcHandler *GRPCHandler
+	api         *pilosa.API
+	logger      logger.Logger
+	stats       stats.StatsClient
+}
+
+func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler {
+	return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient}
+}
+
+func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler {
+	h.logger = logger
+	return h
+}
+
+func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler {
+	h.stats = stats
+	return h
+}
+
+// GetVDSs returns a single VDS given a name
+func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) {
+	typedIdOrName := req.GetIdOrName()
+	switch idOrName := typedIdOrName.(type) {
+	case *vdsm_pb.GetVDSRequest_Id:
+		return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported")
+	case *vdsm_pb.GetVDSRequest_Name:
+		schema := h.api.Schema(ctx)
+		for _, index := range schema {
+			if idOrName.Name == index.Name {
+				return &vdsm_pb.GetVDSResponse{Vds: &vdsm_pb.VDS{Name: index.Name}}, nil
+			}
+		}
+		return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", idOrName.Name))
+	default:
+		return nil, status.Error(codes.NotFound, "VDS not found")
+	}
+}
+
+// GetVDSs returns a list of all VDSs
+func (h *VDSMGRPCHandler) GetVDSs(ctx context.Context, req *vdsm_pb.GetVDSsRequest) (*vdsm_pb.GetVDSsResponse, error) {
+	schema := h.api.Schema(ctx)
+	vdss := make([]*vdsm_pb.VDS, len(schema))
+	for i, index := range schema {
+		vdss[i] = &vdsm_pb.VDS{Name: index.Name}
+	}
+	return &vdsm_pb.GetVDSsResponse{Vdss: vdss}, nil
+}
+
+// PostVDS creates a new VDS
+func (*VDSMGRPCHandler) PostVDS(ctx context.Context, req *vdsm_pb.PostVDSRequest) (*vdsm_pb.PostVDSResponse, error) {
+	// Pilosa doesn't implement VDSD files, so this is unimplemented
+	return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented")
+}
+
+// DeleteVDS deletes a VDS
+func (h *VDSMGRPCHandler) DeleteVDS(ctx context.Context, req *vdsm_pb.DeleteVDSRequest) (*vdsm_pb.DeleteVDSResponse, error) {
+	typedIdOrName := req.GetIdOrName()
+	switch idOrName := typedIdOrName.(type) {
+	case *vdsm_pb.DeleteVDSRequest_Id:
+		return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported")
+	case *vdsm_pb.DeleteVDSRequest_Name:
+		err := h.api.DeleteIndex(ctx, idOrName.Name)
+		if err != nil {
+			return nil, errToStatusError(err)
+		}
+		return &vdsm_pb.DeleteVDSResponse{}, nil
+	default:
+		return nil, status.Error(codes.NotFound, "")
+	}
+}
+
+func (h *VDSMGRPCHandler) QuerySQL(req *pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error {
+	return h.grpcHandler.QuerySQL(req, srv)
+}
+
+func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLRequest) (*pb.TableResponse, error) {
+	return h.grpcHandler.QuerySQLUnary(ctx, req)
+}
+
+func (h *VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error {
+	preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql}
+	return h.grpcHandler.QueryPQL(preq, srv)
+}
+
+func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*pb.TableResponse, error) {
+	preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql}
+	return h.grpcHandler.QueryPQLUnary(ctx, preq)
+}
+
+func (h *VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error {
+	preq := &pb.InspectRequest{Index: req.Vds, Columns: req.Records, FilterFields: req.FilterFields, Limit: req.Limit, Offset: req.Offset, Query: req.Query}
+	return h.grpcHandler.Inspect(preq, srv)
+}
+
 // ResultUint64 is a wrapper around a uint64 result type
 // so that we can implement the ToTabler and ToRowser
 // interfaces.
@@ -829,6 +885,8 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 					return nil
 				}
 				cols = limitedCols
+			} else {
+				return errors.Errorf("expected 1 result for inspect query; got %d from %q", len(resp.Results), req.Query)
 			}
 		}
 
@@ -870,7 +928,13 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 							rowResp.Columns = append(rowResp.Columns,
 								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}})
 							colAdded++
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
 						}
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
 					}
 
 				case "mutex":
@@ -902,6 +966,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 							rowResp.Columns = append(rowResp.Columns,
 								&pb.ColumnResponse{ColumnVal: nil})
 						}
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
 					}
 
 				case "int":
@@ -938,7 +1005,13 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 										value = vals[0]
 										exists = true
 									}
+								} else {
+									rowResp.Columns = append(rowResp.Columns,
+										&pb.ColumnResponse{ColumnVal: nil})
 								}
+							} else {
+								rowResp.Columns = append(rowResp.Columns,
+									&pb.ColumnResponse{ColumnVal: nil})
 							}
 						} else {
 							value, exists, err = field.StringValue(tx, id)
@@ -975,6 +1048,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 								rowResp.Columns = append(rowResp.Columns,
 									&pb.ColumnResponse{ColumnVal: nil})
 							}
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
 						}
 					}
 
@@ -999,6 +1075,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 							rowResp.Columns = append(rowResp.Columns,
 								&pb.ColumnResponse{ColumnVal: nil})
 						}
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
 					}
 
 				case "bool":
@@ -1030,9 +1109,12 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 							rowResp.Columns = append(rowResp.Columns,
 								&pb.ColumnResponse{ColumnVal: nil})
 						}
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
 					}
 
-				case "time":
+				default:
 					rowResp.Columns = append(rowResp.Columns,
 						&pb.ColumnResponse{ColumnVal: nil})
 				}
@@ -1144,7 +1226,9 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
 	// create grpc server
 	s.mu.Lock()
 	s.grpcServer = grpc.NewServer(opts...)
-	pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats))
+	grpcHandler := NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)
+	pb.RegisterPilosaServer(s.grpcServer, grpcHandler)
+	vdsm_pb.RegisterMoleculaServer(s.grpcServer, NewVDSMGRPCHandler(grpcHandler, s.api).WithLogger(s.logger).WithStats(s.stats))
 
 	// register the server so its services are available to grpc_cli and others
 	reflection.Register(s.grpcServer)

server/handler_test.go

@@ -381,6 +381,20 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 	})
 
+	t.Run("UI/usage", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+		ret := mustJSONDecode(t, w.Body)
+		usage := ret["bytesOnDisk"].(map[string]interface{})
+		indexes := usage["indexes"].(map[string]interface{})
+		if len(indexes) != 2 {
+			t.Fatalf("wrong length index size list: %#v", indexes)
+		}
+	})
+
 	t.Run("Metrics", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/metrics", nil))

server/server.go

@@ -37,6 +37,7 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"github.com/pelletier/go-toml"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
 	"github.com/pilosa/pilosa/v2/encoding/proto"
@@ -406,6 +407,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerGRPCURI(advertiseGRPCURI),
 		pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
 		pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
+		pilosa.OptServerClusterName(m.Config.Cluster.Name),
 		pilosa.OptServerSerializer(proto.Serializer{}),
 		pilosa.OptServerTxsrc(m.Config.Txsrc),
 		coordinatorOpt,
@@ -621,3 +623,10 @@ func (f *filteredWriter) Write(p []byte) (n int, err error) {
 	}
 	return len(p), nil
 }
+
+// ParseConfig parses s into a Config.
+func ParseConfig(s string) (Config, error) {
+	var c Config
+	err := toml.Unmarshal([]byte(s), &c)
+	return c, err
+}

server/server_test.go

@@ -31,7 +31,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pelletier/go-toml"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
 	"github.com/pilosa/pilosa/v2/pql"
@@ -338,7 +337,7 @@ func TestMain_MinMaxFloat(t *testing.T) {
 
 // Ensure the host can be parsed.
 func TestConfig_Parse_Host(t *testing.T) {
-	if c, err := ParseConfig(`bind = "local"`); err != nil {
+	if c, err := server.ParseConfig(`bind = "local"`); err != nil {
 		t.Fatal(err)
 	} else if c.Bind != "local" {
 		t.Fatalf("unexpected host: %s", c.Bind)
@@ -347,7 +346,7 @@ func TestConfig_Parse_Host(t *testing.T) {
 
 // Ensure the data directory can be parsed.
 func TestConfig_Parse_DataDir(t *testing.T) {
-	if c, err := ParseConfig(`data-dir = "/tmp/foo"`); err != nil {
+	if c, err := server.ParseConfig(`data-dir = "/tmp/foo"`); err != nil {
 		t.Fatal(err)
 	} else if c.DataDir != "/tmp/foo" {
 		t.Fatalf("unexpected data dir: %s", c.DataDir)
@@ -600,13 +599,6 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
 	return cmds
 }
 
-// ParseConfig parses s into a Config.
-func ParseConfig(s string) (server.Config, error) {
-	var c server.Config
-	err := toml.Unmarshal([]byte(s), &c)
-	return c, err
-}
-
 // MustMarshalJSON marshals v into a string. Panic on error.
 func MustMarshalJSON(v interface{}) string {
 	buf, err := json.Marshal(v)

test/cluster.go

@@ -248,7 +248,7 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Clust
 			commandOpts = opts[i%len(opts)]
 		}
 		m := NewCommandNode(tb, i == 0, commandOpts...)
-		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600)
+		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"__"+strconv.Itoa(i)), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
 		}

test/pilosa.go

@@ -302,7 +302,7 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse {
 	// set a timeout instead of allowing gohttp.Defaultclient to
 	// potentially hang forever.
 	hc := &gohttp.Client{
-		Timeout: time.Second * 10,
+		Timeout: time.Second * 30,
 	}
 	resp, err := hc.Do(req)
 

tournament.sh

@@ -8,5 +8,6 @@
 for i in rbf lmdb roaring rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb ; do
    echo "$(date) starting ${i}, output to tourna.log.${i}"
    echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i}
-   PILOSA_TXSRC=${i}  make testv-race &>> tourna.log.${i}
+   PILOSA_TXSRC=${i}  make testv-race 2>&1 > tourna.log.${i}
 done
+

translate.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"sync"
@@ -90,7 +91,15 @@ type TranslateStore interface {
 	// the read payload.
 	ReadFrom(io.Reader) (int64, error)
 
-	ComputeTranslatorSummary() (sum *TranslatorSummary, err error)
+	ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error)
+	ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error)
+
+	KeyWalker(walk func(key string, col uint64)) error
+	IDWalker(walk func(key string, col uint64)) error
+
+	RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error)
+
+	GetStorePath() string
 }
 
 // TranslatorSummary is returned, for example from the boltdb string key translators,
@@ -101,6 +110,14 @@ type TranslatorSummary struct {
 	// ParitionID is filled for column keys
 	PartitionID int
 
+	NodeID    string
+	StorePath string
+	IsPrimary bool
+	IsReplica bool
+
+	// PrimaryNodeIndex indexes into the cluster []node array to find the primary
+	PrimaryNodeIndex int
+
 	// Field is filled for row keys
 	Field string
 
@@ -112,6 +129,41 @@ type TranslatorSummary struct {
 
 	// IDCount has the number of ID->Key mappings
 	IDCount int
+
+	// false for RowIDs, true for string-Key column IDs.
+	IsColKey bool
+}
+
+func (s *TranslatorSummary) String() string {
+	return fmt.Sprintf(`
+TranslatorSummary{
+	Index      : %v
+	PartitionID: %v
+	NodeID     : %v
+	StorePath  : %v
+	IsPrimary  : %v
+	IsReplica  : %v
+	PrimaryNodeIndex: %v
+	Field   : %v
+	Checksum: %v
+	KeyCount: %v
+	IDCount : %v
+	IsColKey: %v
+}
+`,
+		s.Index,
+		s.PartitionID,
+		s.NodeID,
+		s.StorePath,
+		s.IsPrimary,
+		s.IsReplica,
+		s.PrimaryNodeIndex,
+		s.Field,
+		s.Checksum,
+		s.KeyCount,
+		s.IDCount,
+		s.IsColKey,
+	)
 }
 
 // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore.
@@ -127,7 +179,7 @@ func GenerateNextPartitionedID(index string, prev uint64, partitionID, partition
 	// Try to use the next ID if it is in the same partition.
 	// Otherwise find ID in next shard that has a matching partition.
 	for id := prev + 1; ; id += ShardWidth {
-		if shardPartition(index, id/ShardWidth, partitionN) == partitionID {
+		if shardToShardPartition(index, id/ShardWidth, partitionN) == partitionID {
 			return id
 		}
 	}
@@ -304,6 +356,30 @@ func NewInMemTranslateStore(index, field string, partitionID, partitionN int) *I
 	}
 }
 
+func (s *InMemTranslateStore) GetStorePath() string {
+	return ""
+}
+
+// KeyWalker executes walk for every pair in the database
+func (s *InMemTranslateStore) KeyWalker(walk func(key string, col uint64)) error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	for id, key := range s.keysByID {
+		walk(key, id)
+	}
+	return nil
+}
+
+// IDWalker executes walk for every pair in the database
+func (s *InMemTranslateStore) IDWalker(walk func(key string, col uint64)) error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	for key, id := range s.idsByKey {
+		walk(key, id)
+	}
+	return nil
+}
+
 var _ OpenTranslateStoreFunc = OpenInMemTranslateStore
 
 // OpenInMemTranslateStore returns a new instance of InMemTranslateStore.
@@ -312,9 +388,18 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition
 	return NewInMemTranslateStore(index, field, partitionID, partitionN), nil
 }
 
-func (s *InMemTranslateStore) ComputeTranslatorSummary() (sum *TranslatorSummary, err error) {
+func (s *InMemTranslateStore) ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error) {
 	panic("TODO")
 }
+
+func (s *InMemTranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error) {
+	panic("TODO")
+}
+
+func (s *InMemTranslateStore) RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error) {
+	panic("TODO")
+}
+
 func (s *InMemTranslateStore) Close() error {
 	return nil
 }

txfactory_internal_test.go

@@ -95,9 +95,9 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) {
 	// allow all goro to finish before Closing the lmdb.env, otherwise
 	// we will crash as the goroutines making Tx will try to use the env
 	// after it is closed. It can take quite a while.
-	// one writer might be blocking the other... so ask for only N-1 at first.
-	barrier.BlockUntil(N - 1)
-	//barrier.BlockUntil(N)
+	// one writer might be blocking the other... so ask for only N-2 at first
+	// to avoid deadlock.
+	barrier.BlockUntil(N - 2)
 	barrier.UnblockReaders()
 	time.Sleep(1 * time.Second)
 }

utils_internal_test.go

@@ -70,7 +70,7 @@ func NewTestCluster(tb testing.TB, n int) *cluster {
 	c.ReplicaN = 1
 	c.Hasher = NewTestModHasher()
 	c.Path = path
-	c.Topology = newTopology()
+	c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
 
 	for i := 0; i < n; i++ {
 		c.nodes = append(c.nodes, &Node{
@@ -137,6 +137,15 @@ func (t *ClusterCluster) CreateIndex(name string) error {
 	return nil
 }
 
+func (t *ClusterCluster) CreateIndexWithOpt(name string, opt IndexOptions) error {
+	for _, c := range t.Clusters {
+		if _, err := c.holder.CreateIndexIfNotExists(name, opt); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error {
 	for _, c := range t.Clusters {
 		idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{})
@@ -272,7 +281,8 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 	c.ReplicaN = 1
 	c.Hasher = NewTestModHasher()
 	c.Path = path
-	c.Topology = newTopology()
+	c.partitionN = DefaultPartitionN
+	c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
 	c.holder = h
 	c.Node = node
 	c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
@@ -515,3 +525,47 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 	node := instr.Coordinator
 	return bcast{t: t}.SendTo(node, complete)
 }
+
+var _ = NewTestClusterWithReplication // happy linter
+
+func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN int) (c *cluster, cleaner func()) {
+	path, err := testhook.TempDir(tb, "pilosa-cluster-")
+	if err != nil {
+		panic(err)
+	}
+
+	// holder
+	h := NewHolder(path, nil)
+
+	// cluster
+	availableShardFileFlushDuration.Set(100 * time.Millisecond)
+	c = newCluster()
+	c.holder = h
+	c.ReplicaN = nReplicas
+	c.Hasher = &Jmphasher{}
+	c.Path = path
+	c.partitionN = partitionN
+	c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
+
+	for i := 0; i < nNodes; i++ {
+		nodeID := fmt.Sprintf("node%d", i)
+		c.nodes = append(c.nodes, &Node{
+			ID:  nodeID,
+			URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
+		})
+		c.Topology.addID(nodeID)
+	}
+
+	c.Node = c.nodes[0]
+	c.Coordinator = c.nodes[0].ID
+	c.SetState(ClusterStateNormal)
+
+	if err := c.holder.Open(); err != nil {
+		panic(err)
+	}
+
+	return c, func() {
+		c.holder.Close()
+		c.close()
+	}
+}