Merge pull request #875 from molecula/remove_debug_tooling

turn on row cache. tx: remove debug tooling

.circleci/config.yml

@@ -1,106 +1,118 @@
-version: 2
+version: 2.1
-defaults: &defaults
+
-  working_directory: /go/src/github.com/pilosa/pilosa
+executors:
-  docker:
+  golang:
-    - image: circleci/golang:1.13
+    parameters:
-  environment:
+      version:
-    GO111MODULE: "on"
+        type: string
-fast-checkout: &fast-checkout
+        default: "1.14"
-  attach_workspace:
+      resource_class:
-    at: .
+        type: string
-jobs:
+        default: medium
-  setup:
+    docker:
-    <<: *defaults
+      - image: circleci/golang:<< parameters.version >>
+    resource_class: << parameters.resource_class >>
+    working_directory: /go/src/github.com/pilosa/pilosa
+
+commands:
+  add-github-auth:
+    steps:
+      - run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com"
+  restore-mod-cache:
     steps:
-      - checkout
       - restore_cache:
-          keys:
+          key: mod-cache-{{ checksum "go.sum" }}
-            - mod-cache-{{ checksum "go.sum" }}
+  save-mod-cache:
-      - run: "go mod download"
+    steps:
       - save_cache:
           key: mod-cache-{{ checksum "go.sum" }}
           paths:
             - /go/pkg/mod/
-      - persist_to_workspace:
+  checkout-plus:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
     steps:
-      - *fast-checkout
+      - add-github-auth
+      - checkout
+      - restore-mod-cache
+
+jobs:
+  setup:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod download
+      - save-mod-cache
+  check-license-headers:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
       - run: make golangci-lint
-  test-build-arm:
+  go-mod-tidy:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - run: go mod tidy
+      - run: git diff --exit-code -- go.mod go.sum
+  test-build-arm:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
+  test:
-    <<: *defaults
+    parameters:
+      resource_class:
+        type: string
+        default: medium
+      golang_version:
+        type: string
+        default: "1.14"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
+      resource_class: << parameters.resource_class >>
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - setup_remote_docker
       - run: make clustertests-build
   prerelease:
-    <<: *base-test
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - run: make prerelease
       - store_artifacts:
           path: build
@@ -108,9 +120,12 @@ jobs:
           root: .
           paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - attach_workspace:
+          at: .
       - run: make release
       - store_artifacts:
           path: build
@@ -121,50 +136,72 @@ jobs:
     docker:
       - image: circleci/python:2.7-jessie
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - checkout-plus
-      - *fast-checkout
+      - attach_workspace:
+          at: .
       - run: sudo pip install awscli
       - run: make prerelease-upload
-  dockerhub-upload:
+  dockerhub-upload-unstable:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - checkout-plus
-      - *fast-checkout
       - setup_remote_docker
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker
+      - run: make docker
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
-  version: 2
+  build:
-  test:
     jobs:
-      - setup
+      - setup:
+          context: molecula
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
           requires:
             - setup
       - check-license-headers:
           requires:
             - setup
+      - go-mod-tidy:
+          requires:
+            - setup
       - test-build-arm:
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          matrix:
+            parameters:
+              golang_version: ["1.14", "1.13"]
+          resource_class: xlarge
           requires:
             - setup
-      - test-golang-1.13-race:
+          filters:
+            tags:
+              only: /^v.*/
+      - test:
+          name: test-race
+          test_make_target: test-race
+          resource_class: xlarge
           requires:
             - setup
-      - test-golang-1.13-386:
+      - test:
-          requires:
+          name: test-shardwidth-22
-            - setup
+          shard_width: "22"
-      - test-golang-1.13:
+          resource_class: large
-          requires:
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
           requires:
             - setup
       - cluster-tests:
@@ -174,22 +211,20 @@ workflows:
           requires:
             - linter
             - check-license-headers
-            - test-golang-1.13
+            - test-golang-1.14
-      - release:
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - linter
+            - setup
-            - check-license-headers
+          filters:
-            - test-golang-1.13
+            branches:
+              only: master
+      - dockerhub-upload-stable:
+          context: molecula
+          requires:
+            - setup
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13

.gitignore

@@ -4,3 +4,5 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+lattice

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,14 +1,19 @@
 FROM golang:1.13.0 as builder
 
+ARG BUILD_FLAGS
+ARG MAKE_FLAGS
+
 COPY . pilosa
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+RUN cd pilosa && make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
 
-FROM alpine:3.9.4
+FROM ubuntu:20.10
 
 LABEL maintainer "dev@pilosa.com"
 
-RUN apk add --no-cache curl jq
+RUN apt-get update
+## debug image: RUN apt-get install -y curl htop vim golang tree jq netcat
+RUN apt-get install -y curl jq
 
 COPY --from=builder /go/bin/pilosa /pilosa
 
@@ -18,5 +23,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
+    && make install FLAGS="-a -mod=vendor"
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,23 +1,30 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
+VARIANT = Molecula
+VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)"
 GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
 RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
+NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+TEST_TAGS = roaringparanoia
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
 
 # Run tests and compile Pilosa
 default: test build
@@ -32,7 +39,41 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v
+
+# Run test suite with race flag
+test-race:
+	go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
+
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
+tour:
+	./tournament.sh
 
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@@ -53,8 +94,7 @@ build:
 # Create a single release build under the build directory
 release-build:
 	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
+	cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -65,14 +105,15 @@ ifndef SKIP_CHECK_CLEAN
 endif
 
 # Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
-release: check-clean
+release: check-clean generate-statik
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
-	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
+# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
+release-sans-ui: check-clean
+	rm -f statik/statik.go
+	$(MAKE) release-build GOOS=darwin GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=amd64
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
 DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
@@ -81,15 +122,15 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
+clustertests-build: vendor
-	docker-compose -f $(DOCKER_COMPOSE) down
+	docker-compose -f $(DOCKER_COMPOSE) down -v
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
 # Create prerelease builds
@@ -104,10 +145,20 @@ prerelease-upload:
 install:
 	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
 
+lattice:
+	git clone git@github.com:molecula/lattice.git
+
+build-lattice: lattice require-yarn
+	cd lattice && git pull && yarn install && yarn build
+
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
 	go generate github.com/pilosa/pilosa/v2/internal
 
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	go generate github.com/pilosa/pilosa/v2/statik
+
 # `go generate` stringers
 generate-stringer:
 	go generate github.com/pilosa/pilosa/v2
@@ -115,25 +166,175 @@ generate-stringer:
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+# dunno if protoc-gen-gofast is actually needed here
+generate-proto-grpc: require-protoc require-protoc-gen-gofast
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
-docker:
+docker: vendor
-	docker build -t "pilosa:$(VERSION)" .
+	docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
 	@echo Created docker image: pilosa:$(VERSION)
 
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
 # Compile Pilosa inside Docker container
 docker-build:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
 
+# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
+pilosa-keydump:
+	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
+
+# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
+pilosa-chk:
+	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
+
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
+
+# run top tests, not subdirs. print summary red/green after.
+# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
+topt:
+	mv log.topt.roar log.topt.roar.prev || true
+	go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.roar
+	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
+	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
+
+topt-badger:
+	mv log.topt.badger log.topt.badger.prev || true
+	PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
+	@echo "   log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
+	@echo "   log.topt.badger   red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
+
+topt-badger-race:
+	mv log.topt.badger-race log.topt.badger-race.prev || true
+	PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger-race
+	@echo "   log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
+	@echo "   log.topt.badger-race   red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
+
+topt-rbf:
+	mv log.topt.rbf log.topt.rbf.prev || true
+	PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.rbf
+	@echo "   log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
+	@echo "   log.topt.rbf   red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
+
+topt-rbf-race:
+	mv log.topt.rbf-race log.topt.rbf-race.prev || true
+	PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m  2>&1 | tee log.topt.rbf-race
+	@echo "   log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l
+	@echo "   log.topt.rbf-race   red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l
+
+topt-lmdb:
+	mv log.topt.lmdb log.topt.lmdb.prev || true
+	PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb
+	@echo "   log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l
+	@echo "   log.topt.lmdb   red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l
+
+topt-lmdb-race:
+	mv log.topt.lmdb log.topt.lmdb.prev || true
+	PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lmdb-race
+	@echo "   log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l
+	@echo "   log.topt.lmdb-race   red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l
+
+topt-race:
+	mv log.topt.race log.topt.race.prev || true
+	go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.race
+	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
+	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
+
+# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
+# and compare each transaction for a result.
+
+bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
+	mv log.bg-rr log.bg-rr.prev || true
+	PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rr
+	@echo "   log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
+	@echo "   log.bg-rr   red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
+
+rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
+	mv log.bg.roar_bg log.bg.roar_bg.prev || true
+	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-bg
+	@echo "   log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
+	@echo "   log.rr-bg   red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
+
+rbf-rr:
+	mv log.rbf-rr log.rbf-rr.prev || true
+	PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-rr
+	@echo "   log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
+	@echo "   log.rbf-rr   red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
+
+rr-rbf:
+	mv log.rr-rbf log.rr-rbf.prev || true
+	PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-rbf
+	@echo "   log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
+	@echo "   log.rr-rbf   red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
+
+rbf-bg:
+	mv log.rbf-bg log.rbf-bg.prev || true
+	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-bg
+	@echo "   log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
+	@echo "   log.rbf-bg   red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
+
+bg-rbf:
+	mv log.bg-rbf log.bg-rbf.prev || true
+	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rbf
+	@echo "   log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
+	@echo "   log.bg-rbf   red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
+
+rbf-lm:
+	mv log.rbf-lm log.rbf-lm.prev || true
+	PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-lm
+	@echo "   log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l
+	@echo "   log.rbf-lm   red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l
+
+lm-rbf:
+	mv log.lm-rbf log.lm-rbf.prev || true
+	PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rbf
+	@echo "   log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l
+	@echo "   log.lm-rbf   red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l
+
+lm-rr:
+	mv log.lm-rr log.lm-rr.prev || true
+	PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.lm-rr
+	@echo "   log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l
+	@echo "   log.lm-rr   red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l
+
+rr-lm:
+	mv log.rr-lm log.rr-lm.prev || true
+	PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-lm
+	@echo "   log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l
+	@echo "   log.rr-lm   red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l
+
+bg-lm:
+	mv log.topt.bg-lm log.topt.bg-lm.prev || true
+	PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.bg-lm
+	@echo "   log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l
+	@echo "   log.topt.bg-lm   red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l
+
+lm-bg:
+	mv log.topt.lm-bg log.topt.lm-bg.prev || true
+	PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.lm-bg
+	@echo "   log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l
+	@echo "   log.topt.lm-bg   red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l
+
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +362,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
 
 ######################
 # Build dependencies #
@@ -175,7 +375,10 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go get -u github.com/rakyll/statik
 
 install-stringer:
 	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer

NOTICE

@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
 Third-party software licenses
 =============================
 
@@ -115,3 +94,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
 
 1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
 
+Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101).
+
 2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
 
     ```shell
@@ -40,7 +42,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
     and verify that it's running:
     
     ```shell
-    curl localhost:10101/nodes
+    curl localhost:10101/status
     ```
 
 3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
@@ -63,13 +65,12 @@ There are supported libraries for the following languages:
 - [Java](https://www.pilosa.com/docs/client-libraries/#java)
 - [Python](https://www.pilosa.com/docs/client-libraries/#python)
 
-## Licenses
+## License
 
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
+Pilosa is licensed under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
+
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
 
 ## Get Support
 

api/client/grpc.go

@@ -0,0 +1,214 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +31,140 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "i"
+		fieldName := "f"
+		attrKey := "k"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, indexName, fieldName)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs0,
+			AttrVals:       attrVals0,
+			Shard:          0,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs1,
+			AttrVals:       attrVals1,
+			Shard:          1,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		m := len(res.ColumnAttrSets)
+		if m != 100 {
+			t.Fatalf("incorrect number of column attrs set; m = %v", m)
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -49,62 +184,79 @@ func TestAPI_Import(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
-	m1 := c[1]
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "rick"
+		indexName := "rick"
-		field := "f"
+		fieldName := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
 
 		rowID := uint64(1)
 		timestamp := int64(0)
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
-		for i := 1; i <= 10; i++ {
+		N := 10
+		for i := 1; i <= N; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		colKeys = colKeys[:N]
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
-			Index:      index,
+			Index:          indexName,
-			Field:      field,
+			IndexCreatedAt: index.CreatedAt(),
-			Shard:      0,
+			Field:          fieldName,
-			RowIDs:     rowIDs,
+			FieldCreatedAt: field.CreatedAt(),
-			ColumnKeys: colKeys,
+			Shard:          0, // import is all on shard 0, why are we making lots of other shards? b/c this is not a restriction.
-			Timestamps: timestamps,
+			RowIDs:         rowIDs,
-		}
+			ColumnKeys:     colKeys,
-		if err := m0.API.Import(ctx, req); err != nil {
+			Timestamps:     timestamps,
-			t.Fatal(err)
 		}
 
-		pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
+		qcx := m0.API.Txf().NewQcx()
+
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 			t.Fatal(err)
 		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
-			t.Fatalf("unexpected column keys: %#v", keys)
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
 		}
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 				return err
 			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 				return fmt.Errorf("unexpected column keys: %#v", keys)
@@ -129,58 +281,6 @@ func TestAPI_Import(t *testing.T) {
 		}
 
 	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
@@ -200,8 +300,8 @@ func TestAPI_ImportValue(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
-	m1 := c[1]
+	m1 := c.GetNode(1)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -219,12 +319,13 @@ func TestAPI_ImportValue(t *testing.T) {
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
@@ -233,9 +334,12 @@ func TestAPI_ImportValue(t *testing.T) {
 			ColumnKeys: colKeys,
 			Values:     values,
 		}
-		if err := m0.API.ImportValue(ctx, req); err != nil {
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
 			t.Fatal(err)
 		}
+		panicOn(qcx.Finish())
 
 		pql := fmt.Sprintf("Row(%s>0)", field)
 
@@ -258,6 +362,125 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+
+		qcx := m1.API.Txf().NewQcx()
+		if err := m1.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		query := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: observerd %+v;  expected '%+v'", ids, colIDs[6:])
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: observerd %+v;  expected '%+v'", ids, []uint64{1})
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.
@@ -266,3 +489,134 @@ type offsetModHasher struct{}
 func (*offsetModHasher) Hash(key uint64, n int) int {
 	return int(key+1) % n
 }
+
+func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	// plan:
+	//  1. set a bit
+	//  2. clear with Import() using the ImportRequest.Clear flag
+	//  3. verifiy the clear is done.
+	//  repeat for ImportValueRequest and ImportValues()
+
+	m0 := c.GetNode(0)
+	m0api := m0.API
+
+	ctx := context.Background()
+	index := "i"
+	fieldAcct0 := "acct0"
+
+	opts := pilosa.OptFieldTypeInt(-1000, 1000)
+
+	_, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = m0api.CreateField(ctx, index, fieldAcct0, opts)
+	if err != nil {
+		t.Fatalf("creating fieldAcct0: %v", err)
+	}
+
+	iraField := "ira" // set field.
+	iraRowID := uint64(3)
+	_, err = m0api.CreateField(ctx, index, iraField)
+	if err != nil {
+		t.Fatalf("creating fieldIRA: %v", err)
+	}
+
+	acctOwnerID := uint64(78) // ColumnID
+	shard := acctOwnerID / ShardWidth
+	acct0bal := int64(500)
+
+	ivr0 := &pilosa.ImportValueRequest{
+		Index:     index,
+		Field:     fieldAcct0,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		Values:    []int64{acct0bal},
+	}
+	ir0 := &pilosa.ImportRequest{
+		Index:     index,
+		Field:     iraField,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		RowIDs:    []uint64{iraRowID},
+	}
+
+	qcx := m0api.Txf().NewQcx()
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bitIsSet := func() bool {
+		query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+		cols := res.Results[0].(*pilosa.Row).Columns()
+		for i := range cols {
+			if cols[i] == acctOwnerID {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !bitIsSet() {
+		panic("IRA bit should have been set")
+	}
+
+	queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
+		query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+
+		if len(res.Results) == 0 {
+			return 0
+		}
+		valCount := res.Results[0].(pilosa.ValCount)
+		return valCount.Val
+	}
+
+	bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+
+	if bal != acct0bal {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
+	}
+
+	// clear the bit
+	qcx = m0api.Txf().NewQcx()
+	ir0.Clear = true
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	if bitIsSet() {
+		panic("IRA bit should have been cleared")
+	}
+
+	// clear the BSI
+	qcx = m0api.Txf().NewQcx()
+	ivr0.Clear = true
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+	if bal != 0 {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
+	}
+}

apimethod_string.go

@@ -19,25 +19,30 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
+	_ = x[apiTranslateData-11]
-	_ = x[apiFieldAttrDiff-12]
+	_ = x[apiField-12]
-	_ = x[apiImport-13]
+	_ = x[apiFieldAttrDiff-13]
-	_ = x[apiImportValue-14]
+	_ = x[apiImport-14]
-	_ = x[apiIndex-15]
+	_ = x[apiImportValue-15]
-	_ = x[apiIndexAttrDiff-16]
+	_ = x[apiIndex-16]
-	_ = x[apiQuery-17]
+	_ = x[apiIndexAttrDiff-17]
-	_ = x[apiRecalculateCaches-18]
+	_ = x[apiQuery-18]
-	_ = x[apiRemoveNode-19]
+	_ = x[apiRecalculateCaches-19]
-	_ = x[apiResizeAbort-20]
+	_ = x[apiRemoveNode-20]
-	_ = x[apiSetCoordinator-21]
+	_ = x[apiResizeAbort-21]
-	_ = x[apiShardNodes-22]
+	_ = x[apiSetCoordinator-22]
-	_ = x[apiViews-23]
+	_ = x[apiShardNodes-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiViews-24]
+	_ = x[apiApplySchema-25]
+	_ = x[apiStartTransaction-26]
+	_ = x[apiFinishTransaction-27]
+	_ = x[apiTransactions-28]
+	_ = x[apiGetTransaction-29]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransaction"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 332, 345, 353, 367, 386, 406, 421, 438}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path)
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

bluegreentx_test.go

@@ -0,0 +1,98 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	cryrand "crypto/rand"
+)
+
+var _ = context.Background
+var _ = os.Open
+var _ = strings.Split
+
+func TestMultiReaderB(t *testing.T) {
+	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
+	// member io.Readers, else it should panic. This should hold for
+	// varying sizes of inputs.
+
+	for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
+		src := io.LimitReader(cryrand.Reader, int64(n))
+
+		a := make([]byte, n)
+		nr := 0
+		for nr < n {
+			na, err := src.Read(a)
+			panicOn(err)
+			nr += na
+		}
+		if nr != n {
+			panic("short read")
+		}
+
+		b := make([]byte, n)
+		copy(b, a)
+		if !bytes.Equal(a, b) {
+			panic("test prep failed")
+		}
+
+		m := &MultiReaderB{
+			a: ioutil.NopCloser(bytes.NewBuffer(a)),
+			b: ioutil.NopCloser(bytes.NewBuffer(b)),
+		}
+
+		// should not trigger the internal panic of MultiReadB
+		ncp, err := io.Copy(ioutil.Discard, m)
+		panicOn(err)
+		if ncp != int64(n) {
+			panic("short copy")
+		}
+
+		for victim := 0; victim < n; victim += 7 {
+
+			copy(b, a)
+			if victim%2 == 0 {
+				// corrupt b
+				b[victim] = (b[victim] + 1) % 255
+			} else {
+				// corrupt a
+				a[victim] = (a[victim] + 1) % 255
+			}
+			m = &MultiReaderB{
+				a: ioutil.NopCloser(bytes.NewBuffer(a)),
+				b: ioutil.NopCloser(bytes.NewBuffer(b)),
+			}
+			helperShouldPanicOnCopy(m)
+		}
+	}
+}
+
+func helperShouldPanicOnCopy(m *MultiReaderB) {
+	// differences in bytes read should be noticed
+	defer func() {
+		r := recover()
+		if r == nil {
+			panic("expected panic on byte difference but didn't see it")
+		}
+	}()
+	_, _ = io.Copy(ioutil.Discard, m)
+}

boltdb/translate.go

@@ -11,10 +11,13 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
@@ -23,17 +26,32 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	"github.com/zeebo/blake3"
 )
 
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
+
+	// ErrTranslateKeyNotFound is returned when translating key
+	// and the underlying store returns an empty set
+	ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set")
+
+	bucketKeys = []byte("keys")
+	bucketIDs  = []byte("ids")
+)
+
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+
+	errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found"
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +63,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
-	index string
+	index       string
-	field string
+	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +91,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -82,9 +112,9 @@ func (s *TranslateStore) Open() (err error) {
 
 	// Initialize buckets.
 	if err := s.db.Update(func(tx *bolt.Tx) error {
-		if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil {
+		if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
 			return err
-		} else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil {
+		} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
 			return err
 		}
 		return nil
@@ -108,6 +138,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,105 +170,82 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKey converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
+// If key does not have an associated id then one is created, unless writable is false,
-func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
-	// Find id by key under read lock.
+func (s *TranslateStore) TranslateKey(key string, writable bool) (uint64, error) {
-	if err := s.db.View(func(tx *bolt.Tx) error {
+	ids, err := s.translateKeys([]string{key}, writable)
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
+	if err != nil {
-		return nil
-	}); err != nil {
-		return 0, err
-	} else if id != 0 {
-		return id, nil
-	}
-
-	if s.ReadOnly() {
-		return 0, pilosa.ErrTranslateStoreReadOnly
-	}
-
-	// Find or create id under write lock.
-	var written bool
-	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
-			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
-			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
-			return err
-		}
-		written = true
-		return nil
-	}); err != nil {
 		return 0, err
 	}
-
+	if len(ids) == 0 {
-	if written {
+		return 0, ErrTranslateKeyNotFound
-		s.notifyWrite()
 	}
-
+	return ids[0], nil
-	return id, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
-// If key does not have an associated id then one is created.
+// If a key does not have an associated id then one is created, unless writable is false,
-func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
-	if len(keys) == 0 {
+func (s *TranslateStore) TranslateKeys(keys []string, writable bool) ([]uint64, error) {
-		return nil, nil
+	return s.translateKeys(keys, writable)
-	}
+}
 
-	// Allocate slice for ID mapping.
+func (s *TranslateStore) translateKeys(keys []string, writable bool) ([]uint64, error) {
-	ids = make([]uint64, len(keys))
+	ids := make([]uint64, 0, len(keys))
 
-	// Find ids by key under read lock.
+	if s.ReadOnly() || !writable {
-	var found int
+		found := 0
-	if err := s.db.View(func(tx *bolt.Tx) error {
+		if err := s.db.View(func(tx *bolt.Tx) error {
-		bkt := tx.Bucket([]byte("keys"))
+			bkt := tx.Bucket(bucketKeys)
-		for i, key := range keys {
+			if bkt == nil {
-			if id := findIDByKey(bkt, key); id != 0 {
+				return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
-				ids[i] = id
-				found++
 			}
+			for _, key := range keys {
+				if id, _ := findIDByKey(bkt, key); id != 0 {
+					ids = append(ids, id)
+					found++
+				}
+			}
+			return nil
+		}); err != nil {
+			return nil, err
 		}
-		return nil
+		if found == len(keys) {
-	}); err != nil {
+			return ids, nil
-		return nil, err
+		}
-	} else if found == len(keys) {
+		if s.ReadOnly() {
-		return ids, nil
+			return ids, pilosa.ErrTranslateStoreReadOnly
-	}
+		}
-
+		if !writable {
-	if s.ReadOnly() {
+			return nil, pilosa.ErrTranslatingKeyNotFound
-		return ids, pilosa.ErrTranslateStoreReadOnly
+		}
+		return nil, nil
 	}
 
 	// Find or create ids under write lock if any keys were not found.
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
+		bkt := tx.Bucket(bucketKeys)
-		for i, key := range keys {
+		for _, key := range keys {
-			if ids[i] != 0 {
+			id, boltKey := findIDByKey(bkt, key)
+			if id != 0 {
+				ids = append(ids, id)
 				continue
 			}
-
+			id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
+			if err := bkt.Put(boltKey, u64tob(id)); err != nil {
-				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
+			} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), boltKey); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
 				return err
 			}
+			ids = append(ids, id)
 			written = true
 		}
 		return nil
 	}); err != nil {
 		return nil, err
 	}
-
 	if written {
 		s.notifyWrite()
 	}
@@ -248,7 +260,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) {
 		return "", err
 	}
 	defer func() { _ = tx.Rollback() }()
-	return findKeyByID(tx.Bucket([]byte("ids")), id), nil
+	return findKeyByID(tx.Bucket(bucketIDs), id), nil
 }
 
 // TranslateIDs converts a list of integer IDs to a list of string keys.
@@ -263,9 +275,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 	}
 	defer func() { _ = tx.Rollback() }()
 
+	bucket := tx.Bucket(bucketIDs)
+
 	keys := make([]string, len(ids))
 	for i, id := range ids {
-		keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id)
+		keys[i] = findKeyByID(bucket, id)
 	}
 	return keys, nil
 }
@@ -273,9 +287,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 // ForceSet writes the id/key pair to the store even if read only. Used by replication.
 func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil {
+		if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
 			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil {
 			return err
 		}
 		return nil
@@ -286,7 +300,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +325,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		max = maxID(tx)
-			max = btou64(key)
-		}
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +333,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -353,7 +420,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 		var found bool
 		if err := r.store.db.View(func(tx *bolt.Tx) error {
 			// Find ID/key lookup at offset or later.
-			cur := tx.Bucket([]byte("ids")).Cursor()
+			cur := tx.Bucket(bucketIDs).Cursor()
 			key, value := cur.Seek(u64tob(r.offset))
 			if key == nil {
 				return nil
@@ -387,13 +454,73 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
+// emptyKey is a sentinel byte slice which stands for "" as a key.
-	if value := bkt.Get([]byte(key)); value != nil {
+var emptyKey = []byte{
-		return btou64(value)
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
+}
+
+func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) {
+	sum = &pilosa.TranslatorSummary{}
+	hasher := blake3.New()
+
+	err = s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			panic("bucketKeys not found")
+		}
+
+		cur := bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			input := append(k, v...)
+			_, _ = hasher.Write(input)
+			sum.KeyCount++
+		}
+
+		bkt = tx.Bucket(bucketIDs)
+		if bkt == nil {
+			panic("bucketIDs not found")
+		}
+
+		cur = bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			input := append(k, v...)
+			_, _ = hasher.Write(input)
+			sum.IDCount++
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+	sum.Checksum = string(buf[:])
+	return sum, nil
 }

boltdb/translate_test.go

@@ -14,9 +14,12 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"testing"
 	"time"
 
@@ -28,24 +31,23 @@ func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
+	// Ensure initial key translates to first ID for shard
-	if id, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
-	if id, err := s.TranslateKey("bar"); err != nil {
+	if id, err := s.TranslateKey("bar", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	if id, err := s.TranslateKey("foo", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
@@ -54,59 +56,142 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial keys translate to incrementing IDs.
+	ids, err := s.TranslateKeys([]string{"abc", "abc"}, true)
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[1], ids[0]; got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
+		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	}
+
+	// Ensure initial keys translate to incrementing IDs.
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
-	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
 
+func TestTranslateStore_ReadKey(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	id, err := s.TranslateKey("foo", false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+
+	s.SetReadOnly(true)
+	id, err = s.TranslateKey("foo", true)
+	if err == nil {
+		t.Fatalf("got error: %+v, want: 'translate store read only'", err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+	s.SetReadOnly(false)
+
+	// Ensure next key autoincrements.
+	if id, err = s.TranslateKey("foo", true); err != nil {
+		t.Fatal(err)
+	}
+	id1, err := s.TranslateKey("foo", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id1 != id {
+		t.Fatalf("TranslateKey()=%d, want %d", id1, id)
+	}
+}
+
+func TestTranslateStore_ReadKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	for _, id := range ids {
+		if id != 0 {
+			t.Fatalf("TranslateKeys()=%d, want %d", id, 0)
+		}
+	}
+
+	// Ensure next key autoincrements.
+	if ids, err = s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, true); err != nil {
+		t.Fatal(err)
+	}
+	ids1, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range ids1 {
+		if ids1[i] != ids[i] {
+			t.Fatalf("TranslateKeys()=%d, want %d", ids1[i], ids[i])
+		}
+	}
+}
 func TestTranslateStore_TranslateID(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("", true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
@@ -114,12 +199,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -136,7 +222,8 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +238,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +247,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz", true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -198,20 +286,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo", true)
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -302,11 +395,76 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, true); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
 func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 	s := MustNewTranslateStore()
@@ -324,3 +482,54 @@ func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 		panic(err)
 	}
 }
+
+func TestCryptoHashPerKey(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	// hash one translation
+
+	expect := map[int]string{
+		1: string([]byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12}),
+		2: string([]byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0}),
+		3: string([]byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9}),
+	}
+	for n := 1; n < 4; n++ {
+		var batch0 []string
+		for i := 0; i < n; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		_ = batch0IDs
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// done with setup
+
+		sum, err := s.ComputeTranslatorSummary()
+		if err != nil {
+			panic(err)
+		}
+		nkey := sum.KeyCount
+		nid := sum.IDCount
+		observedChecksum := sum.Checksum
+		if nkey != n {
+			panic("wrong key count")
+		}
+		if nkey != nid {
+			panic("key count should match id count")
+		}
+
+		// shardwidth 22 has different hashes, of course.
+		if pilosa.ShardWidth == 20 {
+			expectedChecksum := expect[n]
+			if observedChecksum != expectedChecksum {
+				panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum))
+			}
+		}
+	}
+
+}

broadcast.go

@@ -69,6 +69,7 @@ const (
 	messageTypeRecalculateCaches
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -116,6 +117,8 @@ func getMessage(typ byte) Message {
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -155,6 +158,8 @@ func getMessageType(m Message) byte {
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +121,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -134,9 +138,11 @@ var _ cache = &lruCache{}
 
 // rankCache represents a cache with sorted entries.
 type rankCache struct {
-	mu       sync.Mutex
+	mu           sync.Mutex
-	entries  map[uint64]uint64
+	entries      map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -168,10 +174,16 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	// Ignore if the column count is below the threshold,
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -184,7 +196,13 @@ func (c *rankCache) Add(id uint64, n uint64) {
 func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -209,12 +227,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
+	if len(c.entries) == 0 {
-	for id := range c.entries {
+		return nil
-		a = append(a, id)
 	}
-	sort.Sort(uint64Slice(a))
+	ids := make([]uint64, 0, len(c.entries))
-	return a
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +249,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -236,27 +257,40 @@ func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,11 +306,14 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
+
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // SetStats defines the stats client used in the cache.
@@ -285,7 +322,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +358,68 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
+func (p *bitmapPairs) Len() int           { return len(*p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
+	}
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +495,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
+	}
+	copy(r.Pairs, p.Pairs)
+	return
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 

cache_test.go

@@ -15,13 +15,14 @@
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
+// Ensure cache stays constrained to its configured size.
-func TestCache_Rank(t *testing.T) {
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +32,54 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
 }

catcher.go

@@ -0,0 +1,318 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// catcher is useful to report error locations with a
+// stack dump before the complexity
+// of the executor_test swallows up
+// the location of a panic.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
+	c.b.IncrementOpN(index, field, view, shard, changedN)
+}
+
+func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	return c.b.NewTxIterator(index, field, view, shard)
+}
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
+}
+
+func (c *catcherTx) Dump(short bool, shard uint64) {
+	c.b.Dump(short, shard)
+}
+
+func (c *catcherTx) Readonly() bool {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Readonly()
+}
+
+func (tx *catcherTx) Pointer() string {
+	return fmt.Sprintf("%p", tx)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) UseRowCache() bool {
+	return c.b.UseRowCache()
+}
+
+func (c *catcherTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, batched, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEach(index, field, view, shard, fn)
+}
+
+func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.UnionInPlace(index, field, view, shard, others...)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.SliceOfShards(index, field, view, optionalViewPath)
+}
+
+func (c *catcherTx) Group() *TxGroup {
+	return c.b.Group()
+}
+
+func (c *catcherTx) Options() Txo {
+	return c.b.Options()
+}
+
+// Sn retreives the serial number of the Tx.
+func (c *catcherTx) Sn() int64 {
+	return c.b.Sn()
+}

client.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -43,7 +44,11 @@ type FieldValue struct {
 // something hasn't been architected correctly.
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
+// Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client.
+// Doing that may obviate the need to refactor this.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
@@ -51,7 +56,6 @@ type InternalClient interface {
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
 	Nodes(ctx context.Context) ([]*Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,6 +63,7 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@@ -68,7 +73,14 @@ type InternalClient interface {
 	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
 	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error)
 	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
 }
 
 //===============
@@ -76,6 +88,10 @@ type InternalClient interface {
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
 	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+
+	// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
+	TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
 }
 
 type nopInternalQueryClient struct{}
@@ -84,6 +100,14 @@ func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index
 	return nil, nil
 }
 
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
 func newNopInternalQueryClient() *nopInternalQueryClient {
 	return &nopInternalQueryClient{}
 }
@@ -123,15 +147,30 @@ func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest
 func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
+func (n nopInternalClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	return nil, nil
+}
+func (n nopInternalClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
+	return nil
+}
+
 func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -172,3 +211,19 @@ func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte
 func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
 	return nil, nil
 }
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}

cluster_internal_test.go

@@ -17,7 +17,6 @@ package pilosa
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -35,6 +34,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -91,12 +91,17 @@ func TestFragCombos(t *testing.T) {
 }
 
 // newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	h := NewHolder(path, nil)
+	panicOn(h.Open())
+	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -105,7 +110,6 @@ func newIndexWithTempPath(name string) *Index {
 
 // Ensure that fragSources creates the correct fragment mapping.
 func TestFragSources(t *testing.T) {
-
 	uri0, err := NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
@@ -157,27 +161,53 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 101, nil)
+
+	// Obtain transaction.
+	var shard uint64
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 1
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*2+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 2
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*3+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 3
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	panicOn(tx.Commit())
 
 	tests := []struct {
 		from     *cluster
@@ -351,7 +381,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := c.shardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -391,7 +421,7 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
 	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
 
@@ -537,7 +567,7 @@ func TestCluster_Coordinator(t *testing.T) {
 }
 
 func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
+	c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"}
 
 	uri0 := NewTestURIFromHostPort("host0", 0)
 	uri1 := NewTestURIFromHostPort("host1", 0)
@@ -585,7 +615,7 @@ func TestCluster_Topology(t *testing.T) {
 func TestCluster_ResizeStates(t *testing.T) {
 
 	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
+		tc := NewClusterCluster(t, 1)
 
 		// Open TestCluster.
 		if err := tc.Open(); err != nil {
@@ -615,7 +645,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -647,7 +677,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -676,7 +706,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -718,7 +748,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -767,7 +797,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -782,6 +812,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		// Each tc.SetBit starts and commits its own Tx.
 		if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
 			t.Fatalf("setting bit: %v", err)
 		}
@@ -794,12 +825,21 @@ func TestCluster_ResizeStates(t *testing.T) {
 		node0Field := node0.holder.Field("i", "f")
 		node0View := node0Field.view("standard")
 		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
+		node0Checksum, err := node0Fragment.Checksum()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		idx0 := node0.holder.Index("i")
+		if idx0 == nil {
+			t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
+		}
 
 		// addNode needs to block until the resize process has completed.
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
+
 		node1 := tc.Clusters[1]
 
 		// Ensure that nodes come up in state NORMAL.
@@ -808,6 +848,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 		} else if node1.State() != ClusterStateNormal {
 			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
 		}
+		// INVAR: after node1.State() is normal, the rebalancing should have been done.
 
 		expectedTop := &Topology{
 			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
@@ -826,8 +867,15 @@ func TestCluster_ResizeStates(t *testing.T) {
 		node1View := node1Field.view("standard")
 		node1Fragment := node1View.Fragment(1)
 
+		idx1 := node1.holder.Index("i")
+		if idx1 == nil {
+			t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
+		}
+
 		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
+		if chksum, err := node1Fragment.Checksum(); err != nil {
+			t.Fatal(err)
+		} else if !bytes.Equal(chksum, node0Checksum) {
 			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
 		}
 
@@ -846,6 +894,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -857,11 +906,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@@ -899,7 +950,7 @@ func TestAE(t *testing.T) {
 // Ensures that coordinator can be changed.
 func TestCluster_UpdateCoordinator(t *testing.T) {
 	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
+		c := NewTestCluster(t, 2)
 
 		oldNode := c.nodes[0]
 		newNode := c.nodes[1]
@@ -942,15 +993,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c := newCluster()
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be up")
 	}
 
 }
 func TestCluster_confirmNodeDownTimeout(t *testing.T) {
+	sleep := 50 * time.Millisecond
+	retries := 5
+	if testing.Short() {
+		t.Skip()
+	}
 	r := mux.NewRouter()
 	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
+		time.Sleep(sleep * time.Duration(retries))
 		fmt.Fprintln(w, "ignored")
 	}))
 	server := httptest.NewServer(r)
@@ -969,21 +1027,29 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-
+	c := newCluster()
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c.confirmDownSleep = sleep
+	c.confirmDownRetries = retries
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
 }
 
 func TestCluster_confirmNodeDownDown(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	uri := URI{}
 	uri.Scheme = "http"
 	uri.Host = "DoesntMatter"
 	uri.Port = 6666
+	c := newCluster()
+	c.confirmDownSleep = 50 * time.Millisecond
+	c.confirmDownRetries = 5
+	c.logger = logger.NewVerboseLogger(os.Stdout)
 
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
 }

cmd/badloader/badloader.go

@@ -0,0 +1,154 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+	//"fmt"
+	"fmt"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	//"log"
+	"os"
+	//"path/filepath"
+	//"sort"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *http.InternalClient) error {
+	t0 := time.Now()
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	//vv("top of tar loop")
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				panic("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				panicOn(err)
+				//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+			}
+			return nil
+		}
+		//vv("got header '%v'", header.Name)
+		n++
+		if n%500 == 0 {
+			vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		//vv("parts = '%#v'", parts)
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+// badloader reproduce a union in place issue for us. slurp is
+// the new "good" loader, and should always be preferred now
+// when not trying to repro that bug. pulled from 85fa67e8
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	panicOn(UploadTar(tarSrcPath, c))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pilosa.URI
+
+func init() {
+	var err error
+	globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
+	panicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
+	return globURI
+}

cmd/badloader/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/convert.go

@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
 			return inspector.Run(context.Background())
 		},
 	}
+	flags := inspectCmd.Flags()
+	flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+	flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+	flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+	flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+	flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+	flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
 	return inspectCmd
 }

cmd/convert/main.go

@@ -0,0 +1,43 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+func main() {
+
+	if len(os.Args) != 3 {
+		log.Fatal("USAGE convert srcPath destPath")
+
+	}
+	holder := pilosa.NewHolder(os.Args[1], nil)
+	err := holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+
+	}
+	c := &pilosa.RBFConverter{
+		Dbs:  make(map[string]*rbf.DB),
+		Base: os.Args[2],
+	}
+	holder.ConvertToRBF(c)
+}

cmd/demo-lmdb/lmdb.go

@@ -0,0 +1,267 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !386
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/glycerine/lmdb-go/lmdb"
+)
+
+// note: use runtime.LockOSThread on any write goroutine; must create and write the txn from
+// the same goroutine.
+
+// This example demonstrates a complete workflow for a simple application
+// working with LMDB.  First, an Env is configured and mapped to memory.  Once
+// mapped, database handles are opened and normal database operations may
+// begin.
+func main() {
+	// Create an environment and make sure it is eventually closed.
+	env, err := lmdb.NewEnv()
+	panicOn(err)
+	defer env.Close()
+
+	// Configure and open the environment.  Most configuration must be done
+	// before opening the environment.  The go documentation for each method
+	// should indicate if it must be called before calling env.Open()
+	err = env.SetMaxDBs(1)
+	panicOn(err)
+	err = env.SetMapSize(1 << 30)
+	panicOn(err)
+	path := "./db-lmdb"
+	panicOn(os.MkdirAll(path, 0755))
+	err = env.Open(path, 0, 0644) // lmdb.Create ?
+	panicOn(err)
+
+	// In any real application it is important to check for readers that were
+	// never closed by their owning process, and for which the owning process
+	// has exited.  See the documentation on transactions for more information.
+	staleReaders, err := env.ReaderCheck()
+	panicOn(err)
+	if staleReaders > 0 {
+		log.Printf("cleared %d reader slots from dead processes", staleReaders)
+	}
+
+	// Open a database handle that will be used for the entire lifetime of this
+	// application.  Because the database may not have existed before, and the
+	// database may need to be created, we need to get the database handle in
+	// an update transacation.
+	var dbi lmdb.DBI
+	_ = dbi
+	err = env.Update(func(txn *lmdb.Txn) (err error) {
+		dbi, err = txn.CreateDBI("example")
+		return err
+	})
+	panicOn(err)
+
+	// The database referenced by our DBI handle is now ready for the
+	// application to use.  Here the application just opens a readonly
+	// transaction and reads the data stored in the "hello" key and prints its
+	// value to the application's standard output.
+	err = env.View(func(txn *lmdb.Txn) (err error) {
+		v, err := txn.Get(dbi, []byte("hello"))
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(v))
+		return nil
+	})
+	_ = err
+	//panicOn(err) // mdb_get: MDB_NOTFOUND: No matching key/data pair found
+
+	err = env.Update(func(txn *lmdb.Txn) (err error) {
+		panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0))
+		panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0))
+		panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0))
+		panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0))
+		panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0))
+		panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0))
+		return nil
+	})
+	_ = err
+
+	// find max in [000,100) and get 099
+	// find max in [100,200) and get 199
+	// find max in [300,400) and get 399
+	// find max in [400,500) and get nothing back
+	err = env.View(func(txn *lmdb.Txn) (err error) {
+		v, err := txn.Get(dbi, []byte("hello"))
+		if err != nil {
+			return err
+		}
+		fmt.Printf("key 'hello' retreived value: '%v'\n", string(v))
+		return nil
+	})
+	_ = err
+
+	err = env.View(func(txn *lmdb.Txn) (err error) {
+		cur, err := txn.OpenCursor(dbi)
+		panicOn(err)
+		defer cur.Close()
+
+		var cur2 *lmdb.Cursor
+		var err2 error
+		var k, k2, v, v2 []byte
+		i := 0
+		for {
+			if i == 0 {
+				// lmdb.SetRange : The first key no less than the specified key.
+				k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange)
+
+				// cur2 should start at 'a'
+				cur2, err2 = txn.OpenCursor(dbi)
+				panicOn(err2)
+				defer cur2.Close()
+				k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next)
+				_ = err2
+			} else {
+				k, v, err = cur.Get(nil, nil, lmdb.Next)
+				k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next)
+				_ = err2
+			}
+			if lmdb.IsNotFound(err) {
+				return nil
+			}
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("i=%v, %s %s\n", i, k, v)
+			fmt.Printf("i=%v, k2:%s v2:%s\n", i, k2, v2)
+			i++
+		}
+		// return nil // unreachable
+	})
+	_ = err
+
+	// panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0))
+	// panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0))
+	// panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0))
+	// panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0))
+	// panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0))
+	// panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0))
+	//
+	// find max in [300,400) and get 399
+	// find max in [000,100) and get 099
+	// find max in [100,200) and get 199
+	// find max in [400,500) and get nothing back
+	// find max in [201,300) and get nothing back
+
+	err = env.View(func(txn *lmdb.Txn) (err error) {
+		cur, err := txn.OpenCursor(dbi)
+		panicOn(err)
+		defer cur.Close()
+
+		var k, v []byte
+
+		// find max in [300,400) and get 399
+
+		// lmdb.SetRange : The first key no less than the specified key.
+		k, v, err = cur.Get([]byte("400"), nil, lmdb.SetRange)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("400 not found, as expected\n") // happens on starting empty db
+		} else {
+			fmt.Printf("Get 400 => %v: %v\n", string(k), string(v))
+		}
+
+		k, v, err = cur.Get(nil, nil, lmdb.Prev)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("Get 400 then Get Prev => not found\n")
+		} else {
+			fmt.Printf("Get 400 then Get Prev => %v: %v\n", string(k), string(v)) // 399: F, so wraps backwards from beginning.
+		}
+		panicOn(err)
+
+		// now try for 199 in [100,200)
+
+		k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange)
+		if lmdb.IsNotFound(err) {
+			panic("200 not found, not expected")
+		} else {
+			fmt.Printf("Get 200 => %v: %v\n", string(k), string(v)) // Get 200 => 200: D
+		}
+
+		k, v, err = cur.Get(nil, nil, lmdb.Prev)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("Get 200 then Get Prev => not found\n")
+		} else {
+			fmt.Printf("Get 200 then Get Prev => %v: %v\n", string(k), string(v)) // Get 200 then Get Prev => 199: C
+		}
+		panicOn(err)
+
+		k, v, err = cur.Get([]byte("500"), nil, lmdb.SetRange)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("500 not found, as expected\n") // 500 not found, as expected
+		} else {
+			panic(fmt.Sprintf("Get 500 => %v: %v\n", string(k), string(v)))
+		}
+
+		k, v, err = cur.Get(nil, nil, lmdb.Prev)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("Get 500 then Get Prev => not found\n")
+		} else {
+			fmt.Printf("Get 500 then Get Prev => %v: %v\n", string(k), string(v)) // Get 500 then Get Prev => 399: F
+		}
+		panicOn(err)
+
+		k, v, err = cur.Get([]byte("100"), nil, lmdb.SetRange)
+		if lmdb.IsNotFound(err) {
+			panic("100 not found, not expected")
+		} else {
+			fmt.Printf("Get 100 => %v: %v\n", string(k), string(v)) // Get 100 => 101: B
+		}
+
+		k, v, err = cur.Get(nil, nil, lmdb.Prev)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("Get 100 then Get Prev => not found\n")
+		} else {
+			fmt.Printf("Get 100 then Get Prev => %v: %v\n", string(k), string(v)) // Get 100 then Get Prev => 099: A
+		}
+		panicOn(err)
+
+		// find max in [201,300) and get nothing back
+
+		k, v, err = cur.Get([]byte("300"), nil, lmdb.SetRange)
+		if lmdb.IsNotFound(err) {
+			panic("300 not found, not expected")
+		} else {
+			fmt.Printf("Get 300 => %v: %v\n", string(k), string(v)) // Get 300 => 300: E
+		}
+
+		k, v, err = cur.Get(nil, nil, lmdb.Prev)
+		if lmdb.IsNotFound(err) {
+			fmt.Printf("Get 300 then Get Prev => not found\n")
+		} else {
+			fmt.Printf("Get 300 then Get Prev => %v: %v\n", string(k), string(v)) // Get 300 then Get Prev => 200: D
+		}
+		cmp := bytes.Compare(k, []byte("201"))
+		if cmp >= 0 {
+			fmt.Printf("key k = '%v' was >= 201", string(k))
+		} else {
+			fmt.Printf("key k = '%v' was < 201", string(k)) // key k = '200' was < 201
+		}
+		panicOn(err)
+
+		return nil
+	})
+	panicOn(err)
+
+	vv("done")
+}

cmd/demo-lmdb/vprint.go

@@ -0,0 +1,179 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// +build !386
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/export.go

@@ -50,7 +50,7 @@ The file does not contain any headers.
 	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
 	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
-	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
 
 	return exportCmd
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
@@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
-	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
 
 	return importCmd
 }

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
+					Max:       pql.NewDecimal(100, 0),
-					Min:       -10,
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/pilosa-chk/chk.go

@@ -0,0 +1,109 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/hash"
+	"github.com/zeebo/blake3"
+)
+
+// pilosa-chk : read boltdb files and print checksums and counts on the keys. With
+// -v and -ops and -bits you can display every last bit if you want.
+//
+// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
+// forcing a customer to upgrade or downgrade their installed version.
+
+func main() {
+
+	var dir string
+	var showOpsLog bool
+	var showBits bool
+	var showFrags bool
+	var dirChecksum bool
+	home := os.Getenv("HOME")
+	flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
+	flag.BoolVar(&showFrags, "v", false, "show the checksum hash for each fragment in each index. Warning: long output")
+	flag.BoolVar(&showOpsLog, "ops", false, "show the ops log for each fragment. Warning: very long output. Implies -v")
+	flag.BoolVar(&showBits, "bits", false, "show the hot bits for each fragment. Warning: very, very long output. Implies -v")
+	flag.BoolVar(&dirChecksum, "dirsum", false, "compute a directory hash")
+	flag.Parse()
+
+	if showBits {
+		showFrags = true
+	}
+	if showOpsLog {
+		showFrags = true
+	}
+	fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
+
+	if dirChecksum {
+		fmt.Printf("path '%v' has dirhash %v\n", dir, hash.HashOfDir(dir))
+		return
+	}
+
+	fmt.Printf("    the blake-3 hash includes the value of each mapping and the field or partitionID.\n")
+
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+
+	err := holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("\ncalculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
+	var indexes []*pilosa.Index
+
+	final := pilosa.NewAllTranslatorSummary()
+	const verbose = true
+	for _, idx := range holder.Indexes() {
+		asum, err := idx.ComputeTranslatorSummary(verbose)
+		if err != nil {
+			log.Fatal(err)
+		}
+		final.Append(asum)
+		indexes = append(indexes, idx)
+	}
+	final.Sort()
+
+	hasher := blake3.New()
+	fmt.Printf("\nsummary of col/row translations%v:\n", dir)
+	for _, sum := range final.Sums {
+		//fmt.Printf("index: %v  partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		_, _ = hasher.Write([]byte(sum.Checksum))
+	}
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	fmt.Printf("all-checksum = blake3-%x\n", buf)
+
+	if showFrags {
+		for _, idx := range indexes {
+			fmt.Printf("==============================\n")
+			fmt.Printf("index: %v\n", idx.Name())
+			fmt.Printf("==============================\n")
+			idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog)
+		}
+	}
+}

cmd/pilosa-keydump/keydump.go

@@ -0,0 +1,180 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// +build amd64
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/glycerine/lmdb-go/lmdb"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/hash"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/txkey"
+)
+
+// pilosa-keydump is a diagnostic tool that simply prints all the
+// database keys in the database directory path specified
+// as the first argument on the command line.
+func main() {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "must supply path to database directory as only arg\n")
+		os.Exit(1)
+	}
+
+	path := os.Args[1]
+	if !DirExists(path) {
+		fmt.Fprintf(os.Stderr, "directory path '%v' does not exist.\n", path)
+		os.Exit(1)
+	}
+
+	maxr := 1
+	env, err := lmdb.NewEnvMaxReaders(maxr)
+	panicOn(err)
+	defer env.Close()
+
+	panicOn(env.SetMapSize(256 << 30))
+
+	err = env.SetMaxDBs(10)
+	panicOn(err)
+
+	//var myflags uint = NoReadahead | NoSubdir
+	var myflags uint = 0 //lmdb.NoSubdir
+	err = env.Open(path, myflags, 0664)
+	panicOn(err)
+
+	// In any real application it is important to check for readers that were
+	// never closed by their owning process, and for which the owning process
+	// has exited.  See the documentation on transactions for more information.
+	staleReaders, err := env.ReaderCheck()
+	panicOn(err)
+	if staleReaders > 0 {
+		vv("cleared %d reader slots from dead processes", staleReaders)
+	}
+
+	dbnames := []string{}
+
+	var dbiRoot lmdb.DBI
+	var dbi lmdb.DBI
+	env.UseSphynxReader()
+	err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) {
+		//txn.RawRead = true
+
+		dbiRoot, err = txn.OpenRoot(0)
+		panicOn(err)
+
+		cur, err := txn.OpenCursor(dbiRoot)
+		panicOn(err)
+		defer cur.Close()
+
+		for i := 0; true; i++ {
+			var k, v []byte
+			var err error
+			if i == 0 {
+				// must give it at least a zero byte here to start.
+				k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange)
+				panicOn(err)
+			} else {
+				k, v, err = cur.Get([]byte(nil), nil, lmdb.Next)
+				if lmdb.IsNotFound(err) {
+					break
+				} else {
+					panicOn(err)
+				}
+			}
+			dbnames = append(dbnames, string(k))
+			_ = v
+		}
+		cur.Close()
+		return
+	})
+	panicOn(err)
+
+	for _, dbn := range dbnames {
+		fmt.Printf(`
+=========================
+database '%v':
+=========================
+
+`, dbn)
+		err = env.SphynxReader(func(txn *lmdb.Txn, readslot int) (err error) {
+			//txn.RawRead = true
+
+			dbi, err = txn.OpenDBI(dbn, 0)
+			panicOn(err)
+
+			cur, err := txn.OpenCursor(dbi)
+			panicOn(err)
+			defer cur.Close()
+
+			for i := 0; true; i++ {
+				var k, v []byte
+				var err error
+				if i == 0 {
+					// must give it at least a zero byte here to start.
+					k, v, err = cur.Get([]byte{0}, nil, lmdb.SetRange)
+					panicOn(err)
+				} else {
+					k, v, err = cur.Get([]byte(nil), nil, lmdb.Next)
+					if lmdb.IsNotFound(err) {
+						break
+					} else {
+						panicOn(err)
+					}
+				}
+
+				ckey := txkey.KeyExtractContainerKey(k)
+
+				n := len(v)
+
+				hash := hash.Blake3sum16(v[0:(n - 1)])
+				ct := pilosa.ToContainer(v[n-1], v[0:(n-1)])
+				cts := roaring.NewSliceContainers()
+				cts.Put(ckey, ct)
+				rbm := &roaring.Bitmap{Containers: cts}
+				srbm := pilosa.BitmapAsString(rbm)
+
+				fmt.Printf("%04v  %v -> %v (%v hot)\n", i, txkey.ToString(k), hash, ct.N())
+				fmt.Printf("          .......%v\n", srbm)
+			}
+			return
+		})
+		panicOn(err)
+	} // for dbnames
+
+	fmt.Printf("=================== done.\n")
+}

cmd/pilosa-keydump/vprint.go

@@ -0,0 +1,179 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// +build amd64
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/root.go

@@ -26,23 +26,18 @@ import (
 )
 
 func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	productName := "Pilosa " + pilosa.Version
-	if pilosa.EnterpriseEnabled {
-		productName = "Pilosa Enterprise " + pilosa.Version
-	}
 	rc := &cobra.Command{
-		Use:   "pilosa",
+		Use: "pilosa",
+		// TODO: These short/long descriptions could use some updating.
 		Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
-		// TODO - is documentation actually there?
 		Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
+tools for administering Pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
 at https://www.pilosa.com/docs/.
 
-` + productName + `
+` + pilosa.VersionInfo() + "\n",
-Build Time: ` + pilosa.BuildTime + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
 			err := setAllConfig(v, cmd.Flags(), "PILOSA")

cmd/root_test.go

@@ -55,6 +55,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +67,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -53,6 +53,7 @@ func TestServerConfig(t *testing.T) {
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	max-writes-per-request = 3000
 
 	[cluster]
@@ -96,6 +97,7 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		disabled = true
@@ -122,6 +124,7 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		hosts = [

cmd/slurp/slurp.go

@@ -0,0 +1,212 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"flag"
+	"time"
+
+	//"fmt"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+
+	//"log"
+	"os"
+	//"path/filepath"
+	//"sort"
+	"strconv"
+	"strings"
+)
+
+// slurp: slurp is a load-tester for importing bulk data.
+//        It allows us to measure write performance.
+
+type stateMachine struct {
+	viewData  map[string][]byte
+	lastIndex string
+	lastField string
+	lastShard uint64
+	state     string
+	client    *http.InternalClient
+	start     time.Time
+	direct    bool
+}
+
+func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
+	parts := strings.Split(h.Name, "/")
+	switch parts[0] {
+	case "roaring":
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		panicOn(err)
+		if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return err
+		}
+		if _, already := r.viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		r.viewData[view] = roaringData
+		r.lastIndex = index
+		r.lastField = field
+		r.lastShard = shard
+	case "bolt":
+		if r.state == "roaring" {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+			vv("Finished import %v", time.Since(r.start))
+		}
+		//
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+
+		switch v := parts[len(parts)-1]; v {
+		case "keys":
+			index := parts[1]
+			fieldName := parts[2]
+			if fieldName == "_keys" {
+				//skip index keys are not not real fields so will have no need for field keys
+				return nil
+
+			}
+
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
+			if err != nil {
+				return err
+			}
+		default:
+			pilosa.VV("%v", h.Name)
+			index := parts[1]
+			partition, err := strconv.ParseUint(v, 10, 64)
+			if err != nil {
+				return err
+			}
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	r.state = parts[0]
+	return nil
+}
+func (r *stateMachine) Upload() error {
+	if len(r.viewData) > 0 {
+		request := &pilosa.ImportRoaringRequest{
+			Views:  r.viewData,
+			Direct: r.direct,
+		}
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+		err := r.client.ImportRoaring(context.Background(), uri, r.lastIndex, r.lastField, r.lastShard, false, request)
+		if err != nil {
+			return err
+		}
+		r.viewData = make(map[string][]byte)
+	}
+	return nil
+}
+
+func UploadTar(srcFile string, direct bool, client *http.InternalClient) error {
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	runner := &stateMachine{
+		viewData: make(map[string][]byte),
+		start:    time.Now(),
+		direct:   direct,
+	}
+	runner.client = client
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			_ = runner.Upload()
+			break
+		}
+		if err != nil {
+			panicOn(err)
+		}
+		err = runner.NewHeader(header, tarReader)
+		panicOn(err)
+	}
+	return nil
+}
+
+func main() {
+	var host string
+	var direct bool
+	var tarSrcPath string
+	flag.StringVar(&host, "host", "127.0.0.1:10101", "host to import into")
+	flag.BoolVar(&direct, "direct", false, "direct write to database (unsafe)")
+	flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import")
+	flag.Parse()
+
+	uri, err := pilosa.NewURIFromAddress(host)
+	panicOn(err)
+	globURI = uri
+
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	t0 := time.Now()
+	println("uploading", tarSrcPath)
+	panicOn(UploadTar(tarSrcPath, direct, c))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pilosa.URI
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
+	return globURI
+}

cmd/slurp/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

ctl/common.go

@@ -31,12 +31,12 @@ type CommandWithTLSSupport interface {
 }
 
 // SetTLSConfig creates common TLS flags
-func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
+func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
-	flags.StringVarP(certificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
+	flags.StringVarP(certificatePath, prefix+"tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
-	flags.StringVarP(certificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
+	flags.StringVarP(certificateKeyPath, prefix+"tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
-	flags.StringVarP(caCertPath, "tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
+	flags.StringVarP(caCertPath, prefix+"tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
-	flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
+	flags.BoolVarP(skipVerify, prefix+"tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
-	flags.BoolVarP(enableClientVerification, "tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
+	flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
 }
 
 // commandClient returns a pilosa.InternalHTTPClient for the command

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
+			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-				return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-			}
 		}
 
-		// Parse FieldValue.
+		// Parse value.
-		value, err := strconv.ParseInt(record[1], 10, 64)
+		if parseAsFloat {
-		if err != nil {
+			value, err := strconv.ParseFloat(record[1], 64)
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
+			if err != nil {
-		}
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
-		val.Value = value
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
 			}
-			a = a[:0]
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
+			value, err := strconv.ParseInt(record[1], 10, 64)
+			if err != nil {
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
+			}
+			req.Values = append(req.Values, value)
+		}
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
@@ -72,7 +73,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -101,7 +102,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -134,7 +135,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -176,7 +177,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -218,7 +219,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -270,8 +271,8 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	c := test.MustRunCluster(t, 2)
 	defer c.Close()
-	cmd0 := c[0]
+	cmd0 := c.GetNode(0)
-	cmd1 := c[1]
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
-		qry := "Count(Row(f=foo0))"
+		if err := test.RetryUntil(2*time.Second, func() error {
-		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
+			qry := "Count(Row(f=foo0))"
-		if err != nil {
+			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
-			t.Fatalf("Querying data for validation: %s", err)
+			if err != nil {
-		}
+				return fmt.Errorf("Querying data for validation: %s", err)
+			}
 
-		// Read body and unmarshal response.
+			// Read body and unmarshal response.
-		exp := `{"results":[100]}` + "\n"
+			exp := `{"results":[100]}` + "\n"
-		if body, err := ioutil.ReadAll(resp.Body); err != nil {
+			if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
+				return fmt.Errorf("reading: %s", err)
-		} else if !reflect.DeepEqual(body, []byte(exp)) {
+			} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.
@@ -332,7 +339,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -358,7 +365,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -446,7 +453,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -522,7 +529,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))

ctl/inspect.go

@@ -16,15 +16,23 @@ package ctl
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
+	"hash/fnv"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
-	"unsafe"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -33,6 +41,12 @@ import (
 type InspectCommand struct {
 	// Path to data file
 	Path string
+	// don't list details of objects
+	Quiet bool
+	// list only this many objects
+	Max int
+	// Filters:
+	InspectOpts pilosa.InspectRequest
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
+type pointerContext struct {
+	from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(cmd.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+		if cmd.Max != 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(cmd.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+		if cmd.Max > 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
 // Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
 	if err != nil {
@@ -58,7 +183,172 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "statting file")
 	}
+	if fi.IsDir() {
+		total := 0
+		infos, err := f.Readdir(0)
+		if err != nil {
+			return err
+		}
+		if len(infos) == 0 {
+			return errors.New("directory contains no files")
+		}
 
+		names := make([]string, len(infos))
+		nameToInfo := make(map[string]os.FileInfo, len(infos))
+		// find numeric-only names; we'll operate on
+		// either those, or the whole holder if we find
+		// a .topology file.
+		n := 0
+		for _, fi := range infos {
+			name := fi.Name()
+			if name == ".topology" {
+				return cmd.InspectHolder(ctx, cmd.Path)
+			}
+			if _, err := strconv.Atoi(name); err == nil {
+				names[n] = name
+				nameToInfo[name] = fi
+				n++
+			}
+		}
+		if n == 0 {
+			return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+		}
+		names = names[:n]
+		fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+		for _, name := range names {
+			f2, err := os.Open(filepath.Join(cmd.Path, name))
+			if err != nil {
+				return fmt.Errorf("opening %q: %v", name, err)
+			}
+			fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+			err = cmd.InspectFile(f2, nameToInfo[name])
+			total++
+			f2.Close()
+			if err != nil {
+				return fmt.Errorf("inspecting %q: %v", name, err)
+			}
+		}
+		return nil
+	}
+	return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology internal.Topology, myID string, err error) {
+	buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, errors.Wrap(err, "reading file")
+	}
+	if err := proto.Unmarshal(buf, &topology); err != nil {
+		return topology, myID, errors.Wrap(err, "unmarshalling")
+	}
+	sort.Slice(topology.NodeIDs,
+		func(i, j int) bool {
+			return topology.NodeIDs[i] < topology.NodeIDs[j]
+		})
+	buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, nil
+	}
+	myID = strings.TrimSpace(string(buf))
+	return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+	var shardMap map[uint64]int
+	var ok bool
+	if shardMap, ok = partitions[index]; !ok {
+		shardMap = make(map[uint64]int)
+		partitions[index] = shardMap
+	}
+	if partition, ok = shardMap[shard]; !ok {
+		var buf [8]byte
+		binary.BigEndian.PutUint64(buf[:], shard)
+
+		// Hash the bytes and mod by partition count.
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write(buf[:])
+		partition = int(h.Sum64() % uint64(partitionN))
+		shardMap[shard] = partition
+	}
+	return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+	parts := strings.Split(path, "/")
+	shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+	holder := pilosa.NewHolder(path, nil)
+	holder.Opts.Inspect = true
+	holder.Opts.ReadOnly = true
+	err := holder.Open()
+	if err != nil {
+		return fmt.Errorf("%s: holder open: %v", path, err)
+	}
+	holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+	if err != nil {
+		return fmt.Errorf("%s: inspect: %v", path, err)
+	}
+	myPartition := 0
+	topology, myID, err := loadTopology(path)
+	if err == nil {
+		fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+		if len(topology.NodeIDs) > 1 {
+			fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+		}
+		found := false
+		for i := range topology.NodeIDs {
+			if topology.NodeIDs[i] == myID {
+				found = true
+				myPartition = i
+				break
+			}
+		}
+		if !found {
+			fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+		}
+	} else {
+		fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+	}
+	for _, name := range holderInfo.FragmentNames {
+		partition, err := findPartitionPath(name, len(topology.NodeIDs))
+		if err != nil {
+			fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+		} else {
+			if partition == myPartition {
+				fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+			} else {
+				fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+			}
+		}
+		details := holderInfo.FragmentInfo[name]
+		cmd.DisplayInfo(details.BitmapInfo)
+		if details.BlockChecksums != nil {
+			fmt.Fprintf(cmd.Stdout, "  Checksums [%d total]:\n", len(details.BlockChecksums))
+			for _, block := range details.BlockChecksums {
+				fmt.Fprintf(cmd.Stdout, "   %8d: %x\n", block.ID, block.Checksum)
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
 	// Memory map the file.
 	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
@@ -72,39 +362,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	}()
 	// Attach the mmap file to the bitmap.
 	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
+	fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
-	bm := roaring.NewBitmap()
+	var info roaring.BitmapInfo
-	if err := bm.UnmarshalBinary(data); err != nil {
+	_, _, err = roaring.InspectBinary(data, true, &info)
-		return errors.Wrap(err, "unmarshalling")
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	cmd.DisplayInfo(info)
+	if err != nil {
+		return errors.Wrap(err, "inspecting")
 	}
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	return nil
+}
 
-	// Retrieve stats.
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
-	t = time.Now()
+	pC := pointerContext{
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
+		from: info.From,
-	info := bm.Info()
+		to:   info.To,
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	}
 
 	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
+	fmt.Fprintf(cmd.Stdout, "  Bitmap Info:\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Bits: %d\n", info.BitCount)
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintf(cmd.Stdout, "    Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Operations: %d (%d bits)\n", info.Ops, info.OpN)
 	fmt.Fprintln(cmd.Stdout, "")
 
 	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
+	if !cmd.Quiet {
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+		if info.ContainerCount > 0 {
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
+			cmd.PrintContainers(info, pC)
-	for _, ci := range info.Containers {
+		}
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
+		if info.Ops > 0 {
-			ci.Key,
+			cmd.PrintOps(info)
-			ci.Type,
+		}
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
 	}
-	tw.Flush()
-
-	return nil
 }

ctl/inspect_test.go

@@ -41,7 +41,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	file.Close()
 	cm.Path = file.Name()
 	err = cm.Run(context.Background())
-	expectedError := "unmarshalling: "
+	expectedError := "inspecting: "
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Fatalf("expected error '%s', got '%v'", expectedError, err)
 	}
@@ -52,7 +52,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	if err != nil {
 		t.Fatalf("copying data: %v", err)
 	}
-	if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+	if !strings.Contains(buf.String(), "inspecting bitmap...") {
 		t.Fatalf("Inspect doesn't work: %s", err)
 	}
 

ctl/server.go

@@ -26,7 +26,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
+	flags.StringVar(&srv.Config.AdvertiseGRPC, "advertise-grpc", srv.Config.AdvertiseGRPC, "Address to advertise externally for gRPC.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
@@ -34,10 +36,10 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.Uint64Var(&srv.Config.MaxFileCount, "max-file-count", srv.Config.MaxFileCount, "Soft limit on the maximum number of fragment files Pilosa keeps open simultaneously.")
 
 	// TLS
-	SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
+	SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
 
 	// Handler
-	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).")
+	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
 
 	// Cluster
 	flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
@@ -70,17 +72,29 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
+	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
 	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
 	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
 	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
+	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
 	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling
 	flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
 	flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
+
+	// Transactional storage engine
+	flags.StringVarP(&srv.Config.Txsrc, "tx", "", "", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger (default roaring)")
+
+	// Postgres endpoint
+	flags.StringVar(&srv.Config.Postgres.Bind, "postgres.bind", srv.Config.Postgres.Bind, "Address to which to bind a postgres endpoint (leave blank to disable)")
+	SetTLSConfig(flags, "postgres.", &srv.Config.Postgres.TLS.CertificatePath, &srv.Config.Postgres.TLS.CertificateKeyPath, &srv.Config.Postgres.TLS.CACertPath, &srv.Config.Postgres.TLS.SkipVerify, &srv.Config.Postgres.TLS.EnableClientVerification)
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.StartupTimeout), "postgres.startup-timeout", time.Duration(srv.Config.Postgres.StartupTimeout), "Timeout for postgres connection startup. (set 0 to disable)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.ReadTimeout), "postgres.read-timeout", time.Duration(srv.Config.Postgres.ReadTimeout), "Timeout for reads on a postgres connection. (set 0 to disable; does not include connection idling)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
+	flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
+	flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
 }

dbshard.go

@@ -0,0 +1,868 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var _ = sort.Sort
+
+// types to support a database file per shard
+
+type DBHolder struct {
+	Index map[string]*DBIndex
+}
+
+func NewDBHolder() *DBHolder {
+	return &DBHolder{
+		Index: make(map[string]*DBIndex),
+	}
+}
+
+type DBIndex struct {
+	Shard map[uint64]*DBShard
+}
+
+type DBWrapper interface {
+	NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error)
+	DeleteDBPath(dbs *DBShard) error
+	Close() error
+	DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
+	DeleteField(index, field, fieldPath string) error
+	OpenListString() string
+	OpenSnList() (sns []int64)
+	Path() string
+	HasData() (has bool, err error)
+	SetHolder(h *Holder)
+}
+
+type DBRegistry interface {
+	OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error)
+}
+
+type DBShard struct {
+	HolderPath string
+
+	Index string
+	Shard uint64
+	Open  bool
+
+	// With RWMutex, the
+	// writer who calls Lock() automatically gets priority over
+	// any reader who arrives later, even if the lock is held
+	// by a reader to start with.
+	mut sync.RWMutex
+
+	types      []txtype
+	hasRoaring bool // if either of the types is roaringTxn
+
+	W             []DBWrapper
+	ParentDBIndex *DBIndex
+
+	idx *Index
+	per *DBPerShard
+
+	useOpenList int
+	closed      bool
+}
+
+func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) {
+	for _, w := range dbs.W {
+		err = w.DeleteFragment(index, field, view, shard, frag)
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
+	for _, w := range dbs.W {
+		err = w.DeleteField(index, field, fieldPath)
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+func (dbs *DBShard) Close() (err error) {
+	for _, w := range dbs.W {
+		err = w.Close()
+		if err != nil {
+			return err
+		}
+	}
+	dbs.closed = true
+	return
+}
+
+func (dbs *DBShard) HolderString() string {
+	return dbs.HolderPath
+}
+
+// Cleanup must be called at every commit/rollback of a Tx, in
+// order to release the read-write mutex that guarantees a single
+// writer at a time. Each tx must take care to call cleanup()
+// exactly once. examples:
+//   tx.o.dbs.Cleanup(tx)
+//   tx.Options().dbs.Cleanup(tx)
+//
+func (dbs *DBShard) Cleanup(tx Tx) {
+	if dbs == nil {
+		return // some tests are using Tx only, no dbs available.
+	}
+	if useRWLock {
+		if !dbs.hasRoaring {
+			if tx.Readonly() {
+				dbs.mut.RUnlock()
+			} else {
+				dbs.mut.Unlock()
+			}
+		}
+	}
+}
+
+// experimental feature, off for now.
+const useRWLock = false
+
+func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
+	if useRWLock {
+		// enforce only one writer at a time. The dbs.mut is held until
+		// the Tx finishes.
+		if !dbs.hasRoaring {
+			if write {
+				dbs.mut.Lock()
+			} else {
+				dbs.mut.RLock()
+			}
+		}
+	}
+	if o.dbs != dbs {
+		panic(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs))
+	}
+	if o.Shard != dbs.Shard {
+		panic(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)))
+	}
+	var txns []Tx
+
+	for _, w := range dbs.W {
+		tx, err = w.NewTx(write, initialIndexName, o)
+		if err != nil {
+			return nil, err
+		}
+		txns = append(txns, tx)
+	}
+	if len(txns) == 1 {
+		return
+	}
+	// blue green
+	return dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil
+}
+
+func (dbs *DBShard) DeleteDBPath() (err error) {
+	for _, w := range dbs.W {
+		err = w.DeleteDBPath(dbs)
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+type flatkey struct {
+	index string
+	shard uint64
+}
+
+type DBPerShard struct {
+	Mu sync.Mutex
+
+	HolderDir string
+
+	dbh *DBHolder
+
+	// just flat, not buried within the Node heirarchy.
+	// Easily see how many we have.
+	Flatmap map[flatkey]*DBShard
+
+	types      []txtype
+	hasRoaring bool
+
+	txf    *TxFactory
+	holder *Holder
+
+	// which of our types is not-roaring, since
+	// roaring doesn't keep a list of open Tx sn.
+	// or default to the 2nd.
+	useOpenList int
+}
+
+// HasData returns true if the database has at least one key.
+// For roaring it returns true if we a fragment stored.
+// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green.
+// If you pass 1, be sure you have a blue-green configuration.
+func (per *DBPerShard) HasData(which int) (hasData bool, err error) {
+	// has to aggregate across all available DBShard for each index and shard.
+
+	for _, v := range per.Flatmap {
+		hasData, err = v.W[which].HasData()
+		if err != nil {
+			return
+		}
+		if hasData {
+			return
+		}
+	}
+	return
+}
+
+func (per *DBPerShard) ListOpenString() (r string) {
+	for _, v := range per.Flatmap {
+		r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n"
+	}
+	return
+}
+
+func (per *DBPerShard) LoadExistingDBs() (err error) {
+	idxs := per.holder.Indexes()
+
+	for _, idx := range idxs {
+
+		sos, err := per.txf.GetShardsForIndex(idx, "", true)
+		if err != nil {
+			return err
+		}
+		for _, shard := range sos {
+			_, err := per.GetDBShard(idx.name, shard, idx)
+			if err != nil {
+				return errors.Wrap(err, "DBPerShard.LoadExistingDBs GetDBShard()")
+			}
+		}
+	}
+	return
+}
+
+func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) {
+
+	useOpenList := 0
+	hasRoaring := false
+	if types[0] == roaringTxn {
+		hasRoaring = true
+	}
+	if len(types) == 2 {
+		// blue-green, avoid the empty roaring Tx open list.
+		// Prefer B's open list if neither is roaring.
+		if types[0] == roaringTxn || types[1] != roaringTxn {
+			useOpenList = 1
+		}
+		if types[1] == roaringTxn {
+			hasRoaring = true
+		}
+	}
+
+	d = &DBPerShard{
+		types:       types,
+		HolderDir:   holderDir,
+		holder:      holder,
+		dbh:         NewDBHolder(),
+		Flatmap:     make(map[flatkey]*DBShard),
+		txf:         txf,
+		useOpenList: useOpenList,
+		hasRoaring:  hasRoaring,
+	}
+	return
+}
+
+func (per *DBPerShard) DeleteIndex(index string) (err error) {
+
+	per.Mu.Lock()
+	defer per.Mu.Unlock()
+
+	dbi, ok := per.dbh.Index[index]
+	if !ok {
+		// since we lazily make indexes upon use by a Tx now, we won't
+		// have an index for server/ TestQuerySQLUnary/test-20 to delete.
+		// Don't freak out. Just return nil.
+		return nil
+	}
+	for _, dbs := range dbi.Shard {
+		err = dbs.Close()
+		if err != nil {
+			return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()")
+		}
+		for _, ty := range per.types {
+			path := dbs.pathForType(ty)
+			err = os.RemoveAll(path)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path))
+			}
+		}
+	}
+	return
+}
+
+func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
+	per.Mu.Lock()
+	defer func() {
+		if fieldPath != "" {
+			panicOn(os.RemoveAll(fieldPath))
+		}
+		per.Mu.Unlock()
+	}()
+
+	dbi, ok := per.dbh.Index[index]
+	if !ok {
+		// TestIndex_Existence_Delete in index_internal_test.go
+		// will call us without having ever created a Tx or DB,
+		// so we can't complain here.
+		return nil
+	}
+	for _, dbs := range dbi.Shard {
+		for _, w := range dbs.W {
+			err := w.DeleteField(index, field, fieldPath)
+			panicOn(err)
+		}
+	}
+	return
+}
+
+func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
+
+	idx := per.txf.holder.Index(index)
+	dbs, err := per.GetDBShard(index, shard, idx)
+	if err != nil {
+		return err
+	}
+	return dbs.DeleteFragment(index, field, view, shard, frag)
+}
+
+func (dbs *DBShard) DumpAll() {
+	short := false
+	fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard))
+	for i, ty := range dbs.types {
+		_ = i
+		tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx})
+		panicOn(err)
+		defer tx.Rollback()
+		fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty)
+		tx.Dump(short, dbs.Shard)
+
+		switch ty {
+		case roaringTxn:
+		case rbfTxn:
+		case lmdbTxn:
+		case badgerTxn:
+		default:
+			panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
+		}
+	}
+	fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard))
+}
+
+func (per *DBPerShard) DumpAll() {
+	per.Mu.Lock()
+	defer per.Mu.Unlock()
+
+	found1 := false
+	for _, dbi := range per.dbh.Index {
+		for _, dbs := range dbi.Shard {
+			if dbs.Open {
+				found1 = true
+				dbs.DumpAll()
+			}
+		}
+	}
+	if !found1 {
+		AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.HolderDir)
+	}
+}
+
+// if you know the shard, you can use this
+// pathForType and prefixForType must be kept in sync!
+func (dbs *DBShard) pathForType(ty txtype) string {
+	// top level paths will end in "@@"
+
+	// what here for roaring? well, roaringRegistrar.OpenDBWrapper()
+	// is a no-op anyhow. so doesn't need to be correct atm.
+
+	return dbs.HolderPath + sep + dbs.Index + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix())
+}
+
+// if you don't know the shard, you have to use this.
+// prefixForType and pathForType must be kept in sync!
+func (per *DBPerShard) prefixForType(idx *Index, ty txtype) string {
+	// top level paths will end in "@@"
+	return per.HolderDir + sep + idx.name + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep
+}
+
+var ErrNoData = fmt.Errorf("no data")
+
+func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
+
+	per.Mu.Lock()
+	defer per.Mu.Unlock()
+
+	dbi, ok := per.dbh.Index[index]
+	if !ok {
+		dbi = &DBIndex{
+			Shard: make(map[uint64]*DBShard),
+		}
+		per.dbh.Index[index] = dbi
+	}
+	dbs, ok = dbi.Shard[shard]
+	if dbs != nil && dbs.closed {
+		if len(per.types) == 1 && per.types[0] == roaringTxn {
+			// roaring txn are nil/fake anyway. Don't freak out.
+		} else {
+			panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs))
+		}
+	}
+	if !ok {
+		dbs = &DBShard{
+			types:         per.types,
+			ParentDBIndex: dbi,
+			Index:         index,
+			Shard:         shard,
+			HolderPath:    per.HolderDir,
+			idx:           idx,
+			per:           per,
+			useOpenList:   per.useOpenList,
+			hasRoaring:    per.hasRoaring,
+		}
+		dbi.Shard[shard] = dbs
+	}
+	if !dbs.Open {
+		var registry DBRegistry
+		for _, ty := range dbs.types {
+			switch ty {
+			case roaringTxn:
+				registry = globalRoaringReg
+			case rbfTxn:
+				registry = globalRbfDBReg
+			case lmdbTxn:
+				registry = globalLMDBReg
+			case badgerTxn:
+				registry = globalBadgerReg
+			default:
+				panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
+			}
+			path := dbs.pathForType(ty)
+			w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx)
+			panicOn(err)
+			h := idx.Holder()
+			w.SetHolder(h)
+			dbs.Open = true
+			if w != nil && len(dbs.W) == 0 {
+				per.Flatmap[flatkey{index: index, shard: shard}] = dbs
+			}
+			dbs.W = append(dbs.W, w)
+		}
+	}
+	return
+}
+
+func (per *DBPerShard) Del(dbs *DBShard) (err error) {
+	per.Mu.Lock()
+	defer per.Mu.Unlock()
+
+	err = dbs.Close()
+	if err != nil {
+		return
+	}
+	panicOn(dbs.DeleteDBPath())
+	delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard})
+
+	// delete from the heirarchy
+	delete(dbs.ParentDBIndex.Shard, dbs.Shard)
+	return nil
+}
+
+func (per *DBPerShard) Close() (err error) {
+	per.Mu.Lock()
+	defer per.Mu.Unlock()
+
+	for _, dbi := range per.dbh.Index {
+		for _, dbs := range dbi.Shard {
+			err = dbs.Close()
+			panicOn(err)
+		}
+	}
+	return
+}
+
+// DBPerShardGetShardsForIndex returns the shards for idx.
+// If requireData, we open the database and see that it has a key, rather
+// than assume that the database file presence is enough.
+func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (sliceOfShards []uint64, err error) {
+
+	var shards [][]uint64
+	for _, ty := range f.types {
+		var slc []uint64
+		slc, err = f.dbPerShard.TypedDBPerShardGetShardsForIndex(ty, idx, roaringViewPath, requireData)
+		if err != nil {
+			return
+		}
+		shards = append(shards, slc)
+	}
+	n := len(f.types)
+	if n != 1 && n != 2 {
+		panic(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n))
+	}
+
+	// Note: we don't actually know when the blue call and when the green call comes
+	// through here. So if we are deleting a shard, we will see a difference earlier
+	// in one than the other. TestAPI_ClearFlagForImportAndImportValues for example.
+	// Hence we cannot do a blue-green check here for matching shards.
+
+	// If we are populating blue from green, it does matter that we return green.
+	return shards[n-1], nil
+}
+
+// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover
+// all the view paths under idx for type ty.
+// requireData means open the database file and verify that at least one key is set.
+func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (sliceOfShards []uint64, err error) {
+
+	if ty == roaringTxn {
+		rx := &RoaringTx{
+			Index: idx,
+		}
+		if roaringViewPath == "" {
+			fields := idx.Fields()
+			for _, field := range fields {
+				for _, view := range field.views() {
+					sos, err := rx.SliceOfShards("", "", "", view.path)
+					if err != nil {
+						return nil,
+							errors.Wrap(err, fmt.Sprintf(
+								"TypedDBPerShardGetLocalShardsForIndex roaringTxn view.path='%v'", view.path))
+					}
+					sliceOfShards = append(sliceOfShards, sos...)
+				}
+			}
+			return dedupShardSlice(sliceOfShards), nil
+		}
+		return rx.SliceOfShards("", "", "", roaringViewPath)
+	}
+	requiredSuffix := ty.FileSuffix()
+	path := per.prefixForType(idx, ty)
+
+	ignoreEmpty := false
+	includeRoot := true
+	dbf, err := listDirUnderDir(path, includeRoot, requiredSuffix, ignoreEmpty)
+	panicOn(err)
+
+	for _, nm := range dbf {
+
+		base := filepath.Base(nm)
+
+		splt := strings.Split(base, requiredSuffix)
+		if len(splt) != 2 {
+			panic(fmt.Sprintf("should have 2 parts: nm='%v', base(nm)='%v'; requiredSuffix='%v'", nm, base, requiredSuffix))
+		}
+		prefix := splt[0]
+		const shardPrefix = "shard."
+		const lenOfShardPrefix = len(shardPrefix)
+		if !strings.HasPrefix(prefix, shardPrefix) {
+			continue
+		}
+
+		// Parse filename into integer.
+		shard, err := strconv.ParseUint(prefix[lenOfShardPrefix:], 10, 64)
+		if err != nil {
+			panicOn(err)
+			continue
+		}
+
+		// exclude those without data?
+		hasData := false
+
+		if requireData {
+			hasData, err = per.TypedIndexShardHasData(ty, idx, shard)
+			if err != nil {
+				return nil, err
+			}
+			if hasData {
+				sliceOfShards = append(sliceOfShards, shard)
+			}
+		} else {
+			// file presence is enough
+			sliceOfShards = append(sliceOfShards, shard)
+		}
+	}
+	return
+}
+
+func (per *DBPerShard) TypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) {
+	whichty := 0
+	if len(per.types) == 2 {
+		if ty == per.types[1] {
+			whichty = 1
+		}
+	}
+	if ty != per.types[whichty] {
+		return
+	}
+
+	// make the dbs if it doesn't get exist
+	dbs, err := per.GetDBShard(idx.name, shard, idx)
+	if err != nil {
+		return false, errors.Wrap(err, fmt.Sprintf("DBPerShard.TypedIndexShardHasData() "+
+			"per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String()))
+	}
+
+	return dbs.W[whichty].HasData()
+}
+
+func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
+
+	if !dirExists(root) {
+		return
+	}
+	n := len(root) + 1
+	if includeRoot {
+		n = 0
+	}
+	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if len(path) < n {
+			// ignore
+		} else {
+			if info == nil {
+				// re-opening an RBF database hit this, racing with a directory rename.
+				// Don't freak out.
+				return nil
+			}
+			if !info.IsDir() {
+				// ignore files
+			} else {
+				if ignoreEmpty && info.Size() == 0 {
+					return nil
+				}
+				if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
+					files = append(files, path[n:])
+				}
+			}
+		}
+		return nil
+	})
+	return
+}
+
+// populateBlueFromGreen prepares for a blue_green run at startup time.
+//
+// It is called at the end of Holder.Open(). This allows the application
+// of blue-green checking to pilosa instances that
+// were previously run only with a single (solo) backend.
+//
+// PRE: This operation requires, at its start, either:
+//
+// (1) an empty blue database -- this allows transitioning from
+//     a solo database to blue_green checking where the solo
+//     becomes the green; or
+//
+// (2) that the blue data, if present, be logically
+//     identical to the green data -- this allows one to restart
+//     a pilosa that was already running in blue_green mode
+//     and remain in blue_green mode.
+//
+// In either case, the goal to to finish populateBlueFromGreen()
+// and have the exact same logical set of data in both backends.
+//
+// Why must the data be identical after Holder.Open() finishes?
+// Otherwise subsequent blue-green checks have no hope of
+// being accurate.
+//
+// The blue is the destination -- this is always types[0].
+// The green source is always types[1]. The mnemonic is blue_geen.
+// The blue is first, so it is in types[0]. The green
+// is second, in types[1]. For example, with PILOSA_TXSRC=lmdb_roaring
+// we have lmdb as blue, and roaring as green. The contents of
+// lmdb must be empty or exactly match roaring. If lmdb
+// starts empty, it will be populated from roaring by
+// populateBlueFromGreen().
+//
+func (dbs *DBShard) populateBlueFromGreen() (err error) {
+
+	n := len(dbs.W)
+	if n != 2 {
+		panic(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n))
+	}
+
+	dest := dbs.W[0] // blue
+	src := dbs.W[1]  // green
+
+	// copy all the key/container pairs.
+	// Since a shard is fairly small, we think one Tx will suffice.
+
+	readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer readtx.Rollback()
+
+	writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer writetx.Rollback()
+
+	for _, fld := range dbs.idx.Fields() {
+		field := fld.Name()
+		for _, vw := range fld.views() {
+			view := vw.name
+			citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				// might be an empty fragment. If so, let's not freak out.
+				if strings.Contains(err.Error(), "fragment not found") {
+					continue
+				} else {
+					return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator")
+				}
+			}
+
+			for citer.Next() {
+				ckey, rc := citer.Value()
+				err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc)
+				if err != nil {
+					citer.Close()
+					return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer")
+				}
+			}
+			citer.Close()
+		}
+	}
+	err = writetx.Commit()
+	if err != nil {
+		return errors.Wrap(err, "writetx.Commit()")
+	}
+	return nil
+}
+
+// verifyBlueEqualsGreen checks that blue and green are identical.
+func (dbs *DBShard) verifyBlueEqualsGreen() (err error) {
+
+	n := len(dbs.W)
+	if n != 2 {
+		panic(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n))
+	}
+
+	blue := dbs.W[0]
+	green := dbs.W[1]
+
+	greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer greentx.Rollback()
+
+	bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
+	panicOn(err)
+	defer bluetx.Rollback()
+
+	for _, fld := range dbs.idx.Fields() {
+		field := fld.Name()
+		for _, vw := range fld.views() {
+
+			view := vw.name
+			gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				if strings.Contains(err.Error(), "fragment not found") {
+					continue
+				} else {
+					return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator")
+				}
+			}
+
+			bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
+			if err != nil {
+				gCiter.Close()
+				if bCiter != nil {
+					bCiter.Close()
+				}
+				return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator")
+			}
+
+			for gCiter.Next() {
+				greenCkey, greenc := gCiter.Value()
+
+				if !bCiter.Next() {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees missing blue container at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' the greenCkey: '%v'",
+						dbs.Index, field, view, dbs.Shard, greenCkey))
+				}
+				blueCkey, bluec := bCiter.Value()
+
+				if blueCkey != greenCkey {
+					bCiter.Close()
+					gCiter.Close()
+					return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+
+						"difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v'",
+						blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard)
+				}
+				nGreen := greenc.N()
+				nBlue := bluec.N()
+				if nBlue != nGreen {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v",
+						dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue))
+				}
+				err = bluec.BitwiseCompare(greenc)
+				if err != nil {
+					bCiter.Close()
+					gCiter.Close()
+					return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+						"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
+						"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'",
+						dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err))
+				}
+			}
+			if bCiter.Next() {
+				blueCkey, _ := bCiter.Value()
+				bCiter.Close()
+				gCiter.Close()
+				return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
+					"sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+
+					"shard: '%v' the ckey: '%v'",
+					dbs.Index, field, view, dbs.Shard, blueCkey))
+			}
+			bCiter.Close()
+			gCiter.Close()
+		}
+	}
+
+	return nil
+}
+
+func dedupShardSlice(sos []uint64) (r []uint64) {
+	m := make(map[uint64]struct{})
+	for _, s := range sos {
+		m[s] = struct{}{}
+	}
+	for k := range m {
+		r = append(r, k)
+	}
+	return
+}

dbshard_internal_test.go

@@ -0,0 +1,256 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// Shard per db evaluation
+func TestShardPerDB_SetBit(t *testing.T) {
+	f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
+	_ = idx
+	defer f.Clean(t)
+
+	// Set bits on the fragment.
+	if _, err := f.setBit(tx, 120, 1); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 120, 6); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 121, 0); err != nil {
+		t.Fatal(err)
+	}
+	// should have two containers set in the fragment.
+
+	// Verify counts on rows.
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count: %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count: %d", n)
+	}
+
+	// commit the change, and verify it is still there
+	panicOn(tx.Commit())
+
+	// Close and reopen the fragment & verify the data.
+	err := f.Reopen() // roaring data not being flushed? red on roaring
+	if err != nil {
+		t.Fatal(err)
+	}
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	defer tx.Rollback()
+
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	}
+}
+
+// test that we find all *local* shards
+func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetShardsForIndex_LocalOnly")
+	panicOn(err)
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+
+	for _, src := range []string{"lmdb", "roaring", "badger", "rbf"} {
+
+		os.Setenv("PILOSA_TXSRC", src)
+
+		// must make Holder AFTER setting src.
+		holder := NewHolder(tmpdir, nil)
+
+		makeSampleRoaringDir(tmpdir, src, 1, holder)
+
+		idx, err := NewIndex(holder, tmpdir, "rick")
+		panicOn(err)
+		estd := "rick/_exists/views/standard"
+		std := "rick/f/views/standard"
+
+		sos, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
+		panicOn(err)
+
+		for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+			if !inSlice(sos, shard) {
+				panic(fmt.Sprintf("missing shard=%v from sos='%#v'", shard, sos))
+			}
+		}
+		if src == "roaring" {
+			// check estd too
+			sos, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
+			panicOn(err)
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				if !inSlice(sos, shard) {
+					panic(fmt.Sprintf("missing shard=%v from sos='%#v'", shard, sos))
+				}
+			}
+		}
+		holder.Close()
+	}
+}
+
+func inSlice(sos []uint64, shard uint64) bool {
+	for i := range sos {
+		if shard == sos[i] {
+			return true
+		}
+	}
+	return false
+}
+
+// data for Test_DBPerShard_GetShardsForIndex
+//
+var sampleRoaringDirList = map[string]string{"roaring": `
+rick/f/views/standard/fragments/215.cache
+rick/f/views/standard/fragments/221.cache
+rick/f/views/standard/fragments/223.cache
+rick/f/views/standard/fragments/93.cache
+rick/f/views/standard/fragments/217.cache
+rick/f/views/standard/fragments/219.cache
+rick/f/views/standard/fragments/217
+rick/f/views/standard/fragments/219
+rick/f/views/standard/fragments/215
+rick/f/views/standard/fragments/221
+rick/f/views/standard/fragments/223
+rick/f/views/standard/fragments/93
+rick/_exists/views/standard/fragments/221
+rick/_exists/views/standard/fragments/215
+rick/_exists/views/standard/fragments/217
+rick/_exists/views/standard/fragments/93
+rick/_exists/views/standard/fragments/219
+rick/_exists/views/standard/fragments/223
+`,
+	"lmdb": `
+rick.index.txstores@@@/store-lmdb@@/shard.0093-lmdb@
+rick.index.txstores@@@/store-lmdb@@/shard.0215-lmdb@
+rick.index.txstores@@@/store-lmdb@@/shard.0217-lmdb@
+rick.index.txstores@@@/store-lmdb@@/shard.0219-lmdb@
+rick.index.txstores@@@/store-lmdb@@/shard.0221-lmdb@
+rick.index.txstores@@@/store-lmdb@@/shard.0223-lmdb@
+`,
+	"badger": `
+rick.index.txstores@@@/store-badgerdb@@/shard.0093-badgerdb@
+rick.index.txstores@@@/store-badgerdb@@/shard.0215-badgerdb@
+rick.index.txstores@@@/store-badgerdb@@/shard.0217-badgerdb@
+rick.index.txstores@@@/store-badgerdb@@/shard.0219-badgerdb@
+rick.index.txstores@@@/store-badgerdb@@/shard.0221-badgerdb@
+rick.index.txstores@@@/store-badgerdb@@/shard.0223-badgerdb@
+`,
+	"rbf": `
+rick.index.txstores@@@/store-rbfdb@@/shard.0093-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0215-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0217-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0219-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0221-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0223-rbfdb@
+`,
+}
+
+func makeSampleRoaringDir(root, txsrc string, minBytes int, h *Holder) {
+
+	index := "rick"
+	shards := []uint64{0, 93, 215, 217, 219, 221, 223}
+	fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
+	for i, fn := range fns {
+		if fn == "" {
+			continue
+		}
+		var shard uint64
+		if txsrc != "roaring" {
+			// only have shards for the non-roaring
+			shard = shards[i]
+		}
+		switch txsrc {
+		case "lmdb":
+			makeLMDBtestDB(root+sep+fn, h, shard)
+			// also have to make the DBShard in our in-memory tree,
+			// or else the search won't find it because
+			// DBPerShard won't know anything about it.
+			helperCreateDBShard(h, index, shard)
+			continue
+		case "badger":
+			makeBadgertestDB(root+sep+fn, h, shard)
+			helperCreateDBShard(h, index, shard)
+			continue
+		case "rbf":
+			makeRBFtestDB(root+sep+fn, h, shard)
+			helperCreateDBShard(h, index, shard)
+			continue
+		}
+
+		path := root + sep + filepath.Dir(fn)
+		panicOn(os.MkdirAll(path, 0755))
+		fd, err := os.Create(root + sep + fn)
+		panicOn(err)
+		if minBytes > 0 {
+			_, err := fd.Write(make([]byte, minBytes))
+			panicOn(err)
+		}
+		fd.Close()
+	}
+}
+
+func helperCreateDBShard(h *Holder, index string, shard uint64) {
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	panicOn(err)
+	dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
+	panicOn(err)
+	_ = dbs
+}
+
+func makeLMDBtestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+	w, _ := mustOpenEmptyLMDBWrapper(path)
+	LMDBMustSetBitvalue(w, "index", "field", "view", shard, i)
+	w.Close()
+
+}
+
+func makeBadgertestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+	w, _ := mustOpenEmptyBadgerWrapper(path)
+	badgerDBMustSetBitvalue(w, "index", "field", "view", shard, i)
+	w.Close()
+}
+
+func makeRBFtestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+
+	db := rbf.NewDB(path)
+	err := db.Open()
+	panicOn(err)
+	defer db.Close()
+
+	tx, err := db.Begin(true)
+	panicOn(err)
+
+	err = tx.CreateBitmap("x")
+	panicOn(err)
+
+	_, err = tx.Add("x", i)
+	panicOn(err)
+
+	err = tx.Commit()
+	panicOn(err)
+}

dbshard_test.go

@@ -0,0 +1,173 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func skipForNonLMDB(t *testing.T) {
+	src := os.Getenv("PILOSA_TXSRC")
+	if src != "lmdb" {
+		t.Skip("skip if not lmdb")
+	}
+}
+
+var _ = skipForNonLMDB   // happy linter
+var _ = skipForNonBadger // happy linter
+
+func skipForNonBadger(t *testing.T) {
+	src := os.Getenv("PILOSA_TXSRC")
+	if src != "badger" {
+		t.Skip("skip if not badger")
+	}
+}
+
+// Can't write it all to one shard like we do (did).
+func Test_DBPerShard_multiple_shards_used(t *testing.T) {
+	skipForNonLMDB(t)
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+	hldr := c.GetHolder(0)
+	index := "i"
+	hldr.SetBit(index, "general", 10, 0)
+	hldr.SetBit(index, "general", 10, ShardWidth+1)
+	hldr.SetBit(index, "general", 10, ShardWidth+2)
+
+	hldr.SetBit(index, "general", 11, 2)
+	hldr.SetBit(index, "general", 11, ShardWidth+2)
+
+	types := pilosa.MustTxsrcToTxtype("lmdb")
+	idx := hldr.Index(index)
+	shardsU := []uint64{0, 1, 2}
+	pathShard := []string{}
+
+	// check that 3 different shard databases/files were made
+	for i := 0; i < 2; i++ {
+
+		path, err := hldr.Txf().GetDBShardPath(index, shardsU[i], idx, types[0], !writable)
+		panicOn(err)
+		pathShard = append(pathShard, path)
+
+		if !DirExists(pathShard[i]) {
+			panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i]))
+		}
+		sz, err := pilosa.DiskUse(pathShard[i], "")
+		panicOn(err)
+
+		if sz < 100 {
+			panic(fmt.Sprintf("shard %v was too small", i))
+		}
+	}
+
+	if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: `Union(Row(general=10), Row(general=11))`}); err != nil {
+		t.Fatal(err)
+	} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) {
+		t.Fatalf("unexpected columns: %+v", columns)
+	}
+}
+
+func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+
+	t.Run("RowIDColumnKey", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "rick"
+		fieldName := "f"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+
+		rowID := uint64(1)
+		timestamp := int64(0)
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		for i := 1; i <= 10; i++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
+		}
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		//select {}
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("unexpected column keys: %#v", keys)
+		}
+	})
+
+}

diagnostics.go

@@ -229,11 +229,11 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 	timeQuantumEnabled := false
 
 	for _, index := range d.server.holder.Indexes() {
-		numShards += index.AvailableShards().Count()
+		numShards += index.AvailableShards(includeRemote).Count()
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {

diagnostics_internal_test.go

@@ -29,6 +29,7 @@ import (
 func TestDiagnosticsClient(t *testing.T) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)
@@ -121,6 +122,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 			t.Fatalf("couldn't encode version response: %v", err)
 		}
 	}))
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector("localhost:10101")
@@ -158,6 +160,7 @@ func compareJSON(a, b []byte) (bool, error) {
 func BenchmarkDiagnostics(b *testing.B) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)

docs/api-reference.md

@@ -15,7 +15,7 @@ Is equivalent to `GET /schema` and returns the same response.
 
 ### List index schema
 
-`GET /index/<index-name>`
+`GET /index/{index-name}`
 
 Returns the schema of the specified index in JSON.
 
@@ -24,27 +24,31 @@ curl -XGET localhost:10101/index/user
 ```
 ``` response
 {
-    "fields": [
+  "name": "user",
-        {
+  "createdAt": 1591178953061239000,
-            "name": "event",
+  "options": {
-            "options": {
+    "keys": false,
-                "keys": false,
+    "trackExistence": true
-                "timeQuantum": "YMD",
+  },
-                "type": "time"
+  "fields": [
-            }
+    {
-        }
+      "name": "event",
-    ],
+      "createdAt": 1591178962332452000,
-    "name": "user",
+      "options": {
-    "options": {
+        "type": "set",
-        "keys": false,
+        "cacheType": "ranked",
-        "trackExistence": true
+        "cacheSize": 50000,
+        "keys": false
+      }
     }
+  ],
+  "shardWidth": 1048576
 }
 ```
 
 ### Create index
 
-`POST /index/<index-name>`
+`POST /index/{index-name}`
 
 Creates an index with the given name.
 
@@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option
 curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"user","createdAt":1591179042178854000}
 ```
 
 ### Remove index
@@ -75,7 +79,7 @@ curl -XDELETE localhost:10101/index/user
 
 ### Query index
 
-`POST /index/<index-name>/query`
+`POST /index/{index-name}/query`
 
 Sends a [query](../query-language/) to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
 
@@ -133,7 +137,7 @@ By default, all bits and attributes (*for `Row` queries only*) are returned. In
 
 ### Import Data
 
-`POST /index/<index-name>/field/<field-name>/import`
+`POST /index/{index-name}/field/{field-name}/import`
 
 Supports high-rate data ingest to a particular shard of a particular field. The
 official client libraries use this endpoint for their import functionality - it
@@ -151,23 +155,37 @@ represents a particular bit to be set. Timestamps are optional, but if they
 exist must also contain the same number of items as rows and columns. The
 column IDs must all be in the shard specified in the request.
 
+Some endpoints and data structures include a `CreatedAt` fields.
+This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
+but to serve as a unique identifier for use in cache invalidation.
+
+The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
+can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
+This is true except in cases where an index or field gets deleted and then recreated,
+or if Pilosa is restored from a backup.
+So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
+and the ingester will know that it needs to drop its cache.
+
 ```
 message ImportRequest {
-	string Index = 1;
+    string Index = 1;
-	string Field = 2;
+    string Field = 2;
-	uint64 Shard = 3;
+    uint64 Shard = 3;
-	repeated uint64 RowIDs = 4;
+    repeated uint64 RowIDs = 4;
-	repeated uint64 ColumnIDs = 5;
+    repeated uint64 ColumnIDs = 5;
-	repeated string RowKeys = 7;
+    repeated int64 Timestamps = 6;
-	repeated string ColumnKeys = 8;
+    repeated string RowKeys = 7;
-	repeated int64 Timestamps = 6;
+    repeated string ColumnKeys = 8;
+    int64 IndexCreatedAt = 9;
+    int64 FieldCreatedAt = 10;
 }
 ```
 
 
+
 ### Create field
 
-`POST /index/<index-name>/field/<field-name>`
+`POST /index/{index-name}/field/{field-name}`
 
 Creates a field in the given index with the given name.
 
@@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \
      -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"quantity","createdAt":1591180110914425000}
 ```
 
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
@@ -209,21 +227,21 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit,
 curl localhost:10101/index/user/field/language -X POST
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"language","createdAt":1591180128294321000}
 ```
 
 ``` request
 curl localhost:10101/index/repository/field/stats \
     -X POST \
-    -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
+    -d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"stats","createdAt":1591180737881627000}
 ```
 
 ### Remove field
 
-`DELETE /index/<index-name>/field/<field-name>`
+`DELETE /index/{index-name}/field/{field-name}`
 
 Removes the given field.
 
@@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema
 ```
 ``` response
 {
-    "indexes": [
+  "indexes": [
+    {
+      "name": "user",
+      "createdAt": 1591178953061239000,
+      "options": {
+        "keys": false,
+        "trackExistence": true
+      },
+      "fields": [
         {
-            "fields": [
+          "name": "event",
-                {
+          "createdAt": 1591178962332452000,
-                    "name": "event",
+          "options": {
-                    "options": {
+            "type": "set",
-                        "keys": false,
+            "cacheType": "ranked",
-                        "timeQuantum": "YMD",
+            "cacheSize": 50000,
-                        "type": "time"
+            "keys": false
-                    }
+          }
-                },
+        },
-                {
+        {
-                    "name": "language",
+          "name": "language",
-                    "options": {
+          "createdAt": 1591180128294321000,
-                        "cacheSize": 50000,
+          "options": {
-                        "cacheType": "ranked",
+            "type": "set",
-                        "keys": false,
+            "cacheType": "ranked",
-                        "type": "set"
+            "cacheSize": 50000,
-                    }
+            "keys": false
-                }
+          }
-            ],
+        },
-            "name": "user",
+        {
-            "options": {
+          "name": "quantity",
-                "keys": false,
+          "createdAt": 1591180110914425000,
-                "trackExistence": true
+          "options": {
-            }
+            "type": "int",
+            "base": 0,
+            "bitDepth": 0,
+            "min": -1000,
+            "max": 2000,
+            "keys": false,
+            "foreignIndex": ""
+          }
         }
-    ]
+      ],
+      "shardWidth": 1048576
+    }
+  ]
 }
 ```
 
@@ -304,7 +340,7 @@ Returns the version of the Pilosa server.
 curl -XGET localhost:10101/version
 ```
 ``` response
-{"version":"v0.6.0"}
+{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
 ```
 
 ### Get status
@@ -318,22 +354,49 @@ curl -XGET localhost:10101/status
 ```
 ```response
 {
-    "localID": "d3369125-29d8-4305-a351-b4474d14a542",
+  "state": "NORMAL",
-    "nodes": [
+  "nodes": [
-        {
+    {
-            "id": "d3369125-29d8-4305-a351-b4474d14a542",
+      "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
-            "isCoordinator": true,
+      "uri": {
-            "uri": {
+        "scheme": "http",
-                "host": "localhost",
+        "host": "localhost",
-                "port": 10101,
+        "port": 10101
-                "scheme": "http"
+      },
-            }
+      "grpc-uri": {
-        }
+        "scheme": "http",
-    ],
+        "host": "localhost",
-    "state": "NORMAL"
+        "port": 20101
+      },
+      "isCoordinator": true,
+      "state": "READY"
+    }
+  ],
+  "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
 }
 ```
 
+### Get active queries
+
+`GET /queries`
+
+Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format.
+Also includes the amount of time that the query has been running (in nanoseconds when using JSON).
+
+```request
+curl -XGET localhost:10101/queries
+```
+```response
+182.412µs  All()
+```
+
+```request
+curl -XGET -H "Accept: application/json" localhost:10101/queries
+```
+```response
+[{"query":"All()","age":135123}]
+```
+
 ### Recalculate Caches
 
 `POST /recalculate-caches`

docs/configuration.md

@@ -302,7 +302,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
     ```toml
     [metric]
-    service = “statsd”
+    service = "statsd"
     ```
 
 #### Metric Host
@@ -319,7 +319,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 #### Metric Poll Interval
 
 * Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
-* Flag: `metric.poll-interval=”0m15s”`
+* Flag: `metric.poll-interval="0m15s"`
 * Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
 * Config:
 
@@ -403,7 +403,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Tracing Sampler Type
 
-* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely.
+* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely. Default is 'off'.
 * Flag: `tracing.sampler-type`
 * Env: `PILOSA_TRACING_SAMPLER_TYPE`
 * Config:

docs/data-model.md

@@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
 #### Set
 
 Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
+Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
+```go
+[]byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+```
+(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
+In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
 
 ``` request
 curl localhost:10101/index/repository/field/info \

docs/examples.md

@@ -81,7 +81,7 @@ lfm := pdk.LinearFloatMapper{
 }
 ```
 
-`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
+`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a "round to nearest integer" behavior. Other predefined mappers have their own specific parameters, usually two or three.
 
 This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). <!-- TODO update so this makes sense -->
 ```go

docs/getting-started.md

@@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
 ``` response
 {"success":true}
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
@@ -325,7 +325,7 @@ Next, let's create the `repository` index:
 	repository := schema.Index("repository")
 ```
 
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -615,7 +615,7 @@ Next, let's create the `repository` index:
 ```
 	Index repository = schema.index("repository");
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -818,7 +818,7 @@ Next, let's create the `repository` index:
 ```
 repository = schema.index("repository")
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```

docs/glossary.md

@@ -58,7 +58,7 @@ nav = []
 
 <strong id="range-bsi">[Row (BSI)](../query-language/#row-bsi):</strong> A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
 
-<strong id="rows">[Rows](../query-language/#rows):<strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
+<strong id="rows">[Rows](../query-language/#rows):</strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
 
 <strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
 

docs/query-language.md

@@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
 * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
 * `UINT` An unsigned integer (e.g. 42839).
 * `BOOL` A boolean value, `true` or `false`.
@@ -52,6 +52,7 @@ curl localhost:10101/index/repository/query \
 * `CALL` Any query.
 * `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not`.
 * `ROWS_CALL` A query that returns a `Rows` result (i.e. a list of row IDs). Currently only the `Rows` query.
+* `ROWSET_CALL` A query that returns a set of rows. Currently only the `Rows` and `TopN` queries.
 * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`).
 
 ### Write Operations
@@ -428,9 +429,9 @@ Union([ROW_CALL ...])
 
 **Description:**
 
-Union performs a logical OR on the results of all `ROW_CALL` queries passed to it.
+Union performs a set union on the column indexes in the results of all `ROW_CALL` queries passed to it. In comparison to a relational query, this is similar to combining clauses in the "OR" sense.
 
-**Result Type:** object with attrs and bits
+**Result Type:** object with attrs and columns
 
 attrs will always be empty
 
@@ -456,7 +457,7 @@ Intersect(<ROW_CALL>, [ROW_CALL ...])
 
 **Description:**
 
-Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it.
+Intersect performs a set intersection on the column indexes in the results of all `ROW_CALL` queries passed to it. In comparison to a relational query, this is similar to combining clauses in the "AND" sense.
 
 **Result Type:** object with attrs and columns
 
@@ -570,6 +571,38 @@ Not(Row(stargazer=1))
 
 * columns are repositories that were not starred by user 1
 
+#### Limit
+
+**Spec:**
+
+```
+Limit(<ROW_CALL>, [limit=<UINT>], [offset=<UINT>])
+```
+
+**Description:**
+
+Limit executes a `ROW_CALL` and returns a subset of the results.
+If a limit of `n` is specified, then this query will return the first `n` results of the row call.
+If an offset of `m` is specified, then this query will skip the first `m` results of the row call.
+If both a limit and offset are specified, the offset is applied before the limit.
+This can be used to implement pagination.
+
+**Result Type:** object with attrs and columns
+
+attrs will always be empty
+
+**Examples:**
+
+Find the second column that has a bit set in the given row.
+```request
+Limit(Row(stargazer=1), limit=1, offset=1)
+```
+```response
+{"results":[{"attrs":{},"columns":[30]}]}
+```
+
+* columns are repositories that were starred by user 1
+
 #### Count
 **Spec:**
 
@@ -595,34 +628,6 @@ Count(Row(stargazer=1))
 
 * Result is the number of repositories that user 1 has starred.
 
-#### Shift
-**Spec:**
-
-```
-Shift(<ROW_CALL>, [n=UINT])
-```
-
-**Description:**
-
-Returns the row specified by `ROW_CALL` shifted by `n` bits.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of the field `stargazer`
-and shift the result by 2:
-```request
-Shift(Row(stargazer=1), n=2)
-```
-```response
-{"attrs":{},"columns":[12, 22]}
-```
-
-* columns are the repositories which user 1 has starred shifted by 2 bits.
-
 #### TopN
 
 **Spec:**
@@ -643,9 +648,15 @@ have the attribute specified by `attrName` with one of the values specified in
 
 **Caveats:**
 
-* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order.
+In general, the order of the resulting row keys is not guaranteed to reflect the true order of bit counts across an index. The exact solution to the problem of computing the TopN counts is prohibitively expensive, so TopN is instead implemented as a heuristic. This provides a significant performance improvement, at the cost of uncertainty in the result order.
-* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
+
-* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
+The implementation is based on a per-shard cache. The accuracy of the results depends on how well the counts for the overall index are reflected in the individual shards (so TopN queries on a single-shard index are exact). If the distribution of bits across shards is uniform, shard counts are representative. This is often a reasonable assumption, especially for the top results for large data sets, in which counts might follow Zipfian, exponential, or other long-tail distributions. However, this assumption may not hold for some applications.
+
+Additional implementation details:
+
+* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. Note that this per-shard tradeoff is independent of the per-index performance/accuracy tradeoff mentioned above.
+* Fields with cache type `ranked` will return the top rows sorted by count in descending order.
+* Fields with cache type `lru` will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
 * Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order.
 * The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
 
@@ -813,12 +824,39 @@ Options(Row(f1=10), shards=[0, 2])
 {"attrs":{},"columns":[100, 2097152]}
 ```
 
+#### Row Constant
+
+**Spec:**
+
+```
+ConstRow(columns=<[]COLUMN>)
+```
+
+**Description:**
+
+`ConstRow` provides a constant bitmap value that can be used in place of a `Row` call.
+The columns can be specified as integer IDs or strings.
+
+**Result Type:** row value columns.
+
+e.g. `{"attrs":{},"columns":[10, 20]}`
+
+**Examples:**
+
+Filter specified columns to only those with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1):
+```request
+Intersect(ConstRow(columns=[10, 20, 30]), Row(stargazer=1))
+```
+```response
+{"attrs":{},"columns":[10, 20]}
+```
+
 #### Rows
 
 **Spec:**
 
 ```
-Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>)
+Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>, like=<STRING>)
 ```
 
 **Description:**
@@ -839,6 +877,10 @@ If the field is of type `time`, the `from` and `to` arguments can be provided
 to restrict the result to a specific time span. If `from` and `to` are
 not provided, the full range of existing data will be queried.
 
+If `like` is given, only keys matching a pattern will be selected.
+A `like` pattern may use `_` as a placeholder to match a single UTF-8 codepoint, and `%` to match 0 or more codepoints.
+All other characters will be matched exactly.
+
 **Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
 
 **Examples:**
@@ -856,7 +898,50 @@ With keys:
 Rows(job)
 ```
 ```response
-{"rows":null,"keys":["engineer","management","student""]}
+{"rows":null,"keys":["engineer","management","student"]}
+```
+
+With `like`:
+```request
+Rows(job, like="%t")
+```
+```response
+{"rows":null,"keys":["management","student"]}
+```
+
+#### Extract
+
+**Spec:**
+```
+Extract(<ROW_CALL>, [<ROWS_CALL>...])
+```
+
+**Description:**
+
+Extract intersects a set of columns with a set of rows in order to extract a subset of the index.
+The result is a table consisting of the matched columns and the rows which they intersect.
+This is similar to a select query in a SQL database.
+
+**Result Type:** Object with an array of the selected fields and an array of the selected columns.
+The column array contains objects containing a column identifier and an array of field values.
+Field values are typed as such:
+- Bool Field - boolean or null
+- Mutex Field (unkeyed) - 64-bit unsigned integer or null
+- Mutex Field (keyed) - string or null
+- Integer Field - 64-bit signed integer or null
+- Decimal Field - Pilosa decimal value or null
+- Set Field (unkeyed) - array of 64-bit unsigned integers
+- Set Field (keyed) - array of strings
+- Time Field - same as the equivalent Set
+
+**Examples:**
+
+List all stargazers who have starred repository 1, and the full set of repositories they have starred:
+```request
+Extract(Row(stargazer=1), Rows(stargazer))
+```
+```response
+{"fields":[{"name":"stargazer","type":"set"}],"columns":[{"column":3,"rows":[[1, 2, 3]]}]}
 ```
 
 #### Group By
@@ -864,7 +949,7 @@ Rows(job)
 **Spec:**
 
 ```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
+GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>, aggregate=<CALL>)
 ```
 
 **Description:**
@@ -874,14 +959,18 @@ taking one row each from the specified `Rows` calls. It returns only those
 combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
- Intersect, etc.) which will be intersected with each result prior to returning
+Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
+the count. This is analagous to a WHERE clause applied to a relational GROUP BY
- query.
+query.
 
 The optional `limit` argument limits the number of results returned. The results
 are ordered, so as long as the data isn't changing, the same query will return
 the same result set.
 
+The optional `aggregate` argument takes a `Sum()` query which will be used to
+calculate the sum & count of each group. This is similar to using a `SUM()` in
+the SELECT clause of a relation GROUP BY query.
+
 Paging through results is supported by passing the `previous` argument to each
 of the `Rows` calls in the GroupBy. Take the last result from your previous
 `GroupBy` query, and pass each row ID in that result as the `previous` argument
@@ -941,3 +1030,31 @@ GroupBy(Rows(age), Rows(job), limit=7, filter=Row(country=USA))
  {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"student"}],"count":3},
  {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"management"}],"count":7}]
 ```
+
+#### UnionRows
+
+**Spec:**
+
+```
+UnionRows([ROWSET_CALL ...])
+```
+
+**Description:**
+
+UnionRows performs a logical OR on the rows matched by the results of all `ROWSET_CALL` queries passed to it.
+
+**Result Type:** object with attrs and columns
+
+attrs will always be empty
+
+**Examples:**
+
+Query columns with a bit set in any row (repositories that are starred by any user):
+```request
+UnionRows(Rows(stargazer))
+```
+```response
+{"attrs":{},"columns":[10, 20, 30]}
+```
+
+* columns are repositories that were starred by any user

enterprise/COPYING

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

enterprise/enterprise.go

@@ -1,23 +0,0 @@
-// Copyright (c) 2018 Pilosa Corp. All rights reserved.
-//
-// This file is part of Pilosa Enterprise Edition.
-//
-// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package enterprise is now deprecated, and the functionality under
-// enterprise/b has been copied into roaring/. It existed to inject enterprise
-// implementations of various Pilosa features when Pilosa was built with
-// "ENTERPRISE=1 make install". These features were dual-licensed separately
-// from Pilosa community edition under the AGPL and Pilosa's commercial license.
-package enterprise

executor_internal_test.go

@@ -15,20 +15,28 @@
 package pilosa
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"strconv"
 	"strings"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
+	path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
+	holder := NewHolder(path, nil)
+	defer holder.Close()
+
+	cluster := NewTestCluster(t, 1)
+
 	e := &executor{
-		Holder: NewHolder(),
+		Holder:  holder,
+		Cluster: cluster,
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
 	err := e.Holder.Open()
 	if err != nil {
 		t.Fatalf("opening holder: %v", err)
@@ -46,12 +54,14 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("creating fields %v, %v, %v", erra, errb, errc)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
+	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`)
 	if err != nil {
 		t.Fatalf("parsing query: %v", err)
 	}
 	c := query.Calls[0]
-	err = e.translateGroupByCall("i", idx, c)
+	// this is writable call just for testing purpose - to test previous argument
+	// generally GroupBy calls are not writable and keys should already exist
+	err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64), true)
 	if err != nil {
 		t.Fatalf("translating call: %v", err)
 	}
@@ -64,6 +74,19 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		}
 	}
 
+	if having, hok := c.Args["having"].(*pql.Call); !hok {
+		t.Fatal("expected having to be a call")
+	} else if cond, cok := having.Args["count"].(*pql.Condition); !cok {
+		t.Fatal("expected condition to be a count")
+	} else if cond.Op != pql.GT {
+		t.Fatal("expected condition op to be >")
+	} else {
+		val, ok := cond.Uint64Value()
+		if !ok || val != uint64(10) {
+			t.Fatal("expected condition val to be uint64(10)")
+		}
+	}
+
 	errTests := []struct {
 		pql string
 		err string
@@ -102,7 +125,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 				t.Fatalf("parsing query: %v", err)
 			}
 			c := query.Calls[0]
-			err = e.translateGroupByCall("i", idx, c)
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64), false)
 			if err == nil {
 				t.Fatalf("expected error, but translated call is '%s", c)
 			}
@@ -113,6 +136,69 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 	}
 }
 
+func TestExecutor_TranslateRowsOnBool(t *testing.T) {
+	path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
+	holder := NewHolder(path, nil)
+	defer holder.Close()
+
+	e := &executor{
+		Holder:  holder,
+		Cluster: NewTestCluster(t, 1),
+	}
+	if err := e.Holder.Open(); err != nil {
+		t.Fatalf("opening holder: %v", err)
+	}
+
+	idx, err := e.Holder.CreateIndex("i", IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+
+	shard := uint64(0)
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	fb, errb := idx.CreateField("b", OptFieldTypeBool())
+	_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
+	if errb != nil || errbk != nil {
+		t.Fatalf("creating fields %v, %v", errb, errbk)
+	}
+
+	_, err1 := fb.SetBit(tx, 1, 1, nil)
+	_, err2 := fb.SetBit(tx, 2, 2, nil)
+	_, err3 := fb.SetBit(tx, 3, 3, nil)
+	if err1 != nil || err2 != nil || err3 != nil {
+		t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
+	}
+
+	if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		pql string
+	}{
+		{pql: "Rows(b)"},
+		{pql: "GroupBy(Rows(b))"},
+		{pql: "Set(4, b=true)"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.pql, func(t *testing.T) {
+			query, err := pql.ParseString(test.pql)
+			if err != nil {
+				t.Fatalf("parsing query: %v", err)
+			}
+
+			c := query.Calls[0]
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64), true)
+			if err != nil {
+				t.Fatalf("translating call: %v", err)
+			}
+		})
+	}
+}
+
 func isInt(a interface{}) bool {
 	switch a.(type) {
 	case int, int64, uint, uint64:
@@ -222,3 +308,268 @@ func TestFieldRowMarshalJSON(t *testing.T) {
 		t.Fatalf("unexpected json: %s", b)
 	}
 }
+
+func TestExecutor_GroupCountCondition(t *testing.T) {
+	t.Run("satisfiesCondition", func(t *testing.T) {
+		type condCheck struct {
+			cond string
+			exp  bool
+		}
+		tests := []struct {
+			groupCount GroupCount
+			checks     []condCheck
+		}{
+			{
+				groupCount: GroupCount{Count: 100},
+				checks: []condCheck{
+					{cond: "count == 99", exp: false},
+					{cond: "count != 99", exp: true},
+					{cond: "count < 99", exp: false},
+					{cond: "count <= 99", exp: false},
+					{cond: "count > 99", exp: true},
+					{cond: "count >= 99", exp: true},
+
+					{cond: "count == 100", exp: true},
+					{cond: "count != 100", exp: false},
+					{cond: "count < 100", exp: false},
+					{cond: "count <= 100", exp: true},
+					{cond: "count > 100", exp: false},
+					{cond: "count >= 100", exp: true},
+
+					{cond: "count == 101", exp: false},
+					{cond: "count != 101", exp: true},
+					{cond: "count < 101", exp: true},
+					{cond: "count <= 101", exp: true},
+					{cond: "count > 101", exp: false},
+					{cond: "count >= 101", exp: false},
+
+					{cond: "98 < count < 100", exp: false},
+					{cond: "98 < count <= 100", exp: true},
+					{cond: "98 < count < 101", exp: true},
+					{cond: "100 <= count < 102", exp: true},
+					{cond: "100 < count < 102", exp: false},
+					{cond: "98 <= count <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: 100},
+				checks: []condCheck{
+					{cond: "sum == 99", exp: false},
+					{cond: "sum != 99", exp: true},
+					{cond: "sum < 99", exp: false},
+					{cond: "sum <= 99", exp: false},
+					{cond: "sum > 99", exp: true},
+					{cond: "sum >= 99", exp: true},
+
+					{cond: "sum == 100", exp: true},
+					{cond: "sum != 100", exp: false},
+					{cond: "sum < 100", exp: false},
+					{cond: "sum <= 100", exp: true},
+					{cond: "sum > 100", exp: false},
+					{cond: "sum >= 100", exp: true},
+
+					{cond: "sum == 101", exp: false},
+					{cond: "sum != 101", exp: true},
+					{cond: "sum < 101", exp: true},
+					{cond: "sum <= 101", exp: true},
+					{cond: "sum > 101", exp: false},
+					{cond: "sum >= 101", exp: false},
+
+					{cond: "98 < sum < 100", exp: false},
+					{cond: "98 < sum <= 100", exp: true},
+					{cond: "98 < sum < 101", exp: true},
+					{cond: "100 <= sum < 102", exp: true},
+					{cond: "100 < sum < 102", exp: false},
+					{cond: "98 <= sum <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: -100},
+				checks: []condCheck{
+					{cond: "sum == -99", exp: false},
+					{cond: "sum != -99", exp: true},
+					{cond: "sum < -99", exp: true},
+					{cond: "sum <= -99", exp: true},
+					{cond: "sum > -99", exp: false},
+					{cond: "sum >= -99", exp: false},
+
+					{cond: "sum == -100", exp: true},
+					{cond: "sum != -100", exp: false},
+					{cond: "sum < -100", exp: false},
+					{cond: "sum <= -100", exp: true},
+					{cond: "sum > -100", exp: false},
+					{cond: "sum >= -100", exp: true},
+
+					{cond: "sum == -101", exp: false},
+					{cond: "sum != -101", exp: true},
+					{cond: "sum < -101", exp: false},
+					{cond: "sum <= -101", exp: false},
+					{cond: "sum > -101", exp: true},
+					{cond: "sum >= -101", exp: true},
+
+					{cond: "-100 < sum < -98", exp: false},
+					{cond: "-100 <= sum < -98", exp: true},
+					{cond: "-101 < sum < -98", exp: true},
+					{cond: "-102 < sum <= -100", exp: true},
+					{cond: "-102 < sum < -100", exp: false},
+					{cond: "-102 <= sum <= -98", exp: true},
+				},
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
+				for j, check := range test.checks {
+					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
+
+						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
+						if err != nil {
+							t.Fatalf("parsing query: %v", err)
+						}
+						c := query.Calls[0]
+						having := c.Args["having"].(*pql.Call)
+
+						var got bool
+						for subj, cond := range having.Args {
+							switch subj {
+							case "count", "sum":
+								condition, ok := cond.(*pql.Condition)
+								if !ok {
+									t.Fatalf("not a valid condition")
+								}
+								got = test.groupCount.satisfiesCondition(subj, condition)
+							}
+						}
+
+						if got != check.exp {
+							t.Fatalf("expected: %v, but got: %v", check.exp, got)
+						}
+					})
+				}
+			})
+		}
+	})
+}
+
+func TestValCountComparisons(t *testing.T) {
+	tests := []struct {
+		name       string
+		vc         ValCount
+		other      ValCount
+		expLarger  ValCount
+		expSmaller ValCount
+	}{
+		{
+			name: "zero",
+		},
+		{
+			name:       "ints",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 3, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 1},
+			expSmaller: ValCount{Val: 3, Count: 2},
+		},
+		{
+			name:       "floats",
+			vc:         ValCount{FloatVal: 10.2, Count: 1},
+			other:      ValCount{FloatVal: 3.4, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
+			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
+		},
+		{
+			name:       "intsEquality",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 10, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 3},
+			expSmaller: ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:       "floatsEquality",
+			vc:         ValCount{FloatVal: 10.7, Count: 1},
+			other:      ValCount{FloatVal: 10.7, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
+			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			gotLarger := test.vc.larger(test.other)
+			if gotLarger != test.expLarger {
+				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
+			}
+
+			gotSmaller := test.vc.smaller(test.other)
+			if gotSmaller != test.expSmaller {
+				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
+			}
+		})
+	}
+}
+
+func TestToNegInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1 << 63),
+			i64: int64(-1 << 63),
+		},
+		{
+			u64: uint64(1<<63) - 1,
+			i64: int64(-1<<63) + 1,
+		},
+		{
+			u64:      uint64(1<<63) + 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toNegInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}
+
+func TestToInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1<<63) - 1,
+			i64: 1<<63 - 1,
+		},
+		{
+			u64: uint64(0),
+			i64: 0,
+		},
+		{
+			u64:      uint64(1 << 63),
+			overflow: true,
+		},
+		{
+			u64:      1<<64 - 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}

field_internal_test.go

@@ -16,15 +16,18 @@ package pilosa
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a bsiGroup can adjust to its baseValue.
@@ -66,12 +69,12 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b0, pql.LT, 5, 105, false},
 			{b0, pql.LT, -8, 92, false},
 			{b0, pql.LT, -108, -8, false},
-			{b0, pql.LT, 1005, 1023, false},
+			{b0, pql.LT, 1005, 1024, false},
 			{b0, pql.LT, 0, 100, false},
 
 			{b1, pql.LT, 5, 5, false},
 			{b1, pql.LT, -8, -8, false},
-			{b1, pql.LT, 1005, 255, false},
+			{b1, pql.LT, 1005, 256, false},
 			{b1, pql.LT, 0, 0, false},
 
 			{b2, pql.LT, 5, -95, false},
@@ -89,6 +92,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, pql.GT, -8, -8, false},
 			{b1, pql.GT, 1005, 0, true},
 			{b1, pql.GT, 0, 0, false},
+			{b1, pql.GT, -300, -256, false},
 
 			{b2, pql.GT, 5, -95, false},
 			{b2, pql.GT, -8, -108, false},
@@ -141,6 +145,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, 5, 20, 5, 20, false},
 			{b1, 20, 1005, 20, 255, false},
 			{b1, 1005, 2000, 0, 0, true},
+			{b1, 0, -1, 0, 0, true},
 
 			{b2, 5, 95, -95, -5, false},
 			{b2, 95, 120, -5, 20, false},
@@ -158,7 +163,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	viewName := viewStandard + "_v"
@@ -192,64 +197,73 @@ func TestField_DeleteView(t *testing.T) {
 // TestField represents a test wrapper for Field.
 type TestField struct {
 	*Field
+	parent *Index
+	tb     testing.TB
 }
 
 // NewTestField returns a new instance of TestField d/0.
-func NewTestField(opts FieldOption) *TestField {
+func NewTestField(t *testing.T, opts FieldOption) *TestField {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
+	path, err := testhook.TempDirInDir(t, *TempDir, "pilosa-field-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := NewHolder(path, nil)
+	panicOn(h.Open())
+
+	idx, err := h.CreateIndex("i", IndexOptions{})
 	if err != nil {
 		panic(err)
 	}
-	field, err := NewField(path, "i", "f", opts)
+	field, err := idx.CreateField("f", opts)
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
-	return &TestField{Field: field}
+	tf := &TestField{Field: field, parent: idx, tb: t}
+	testhook.Cleanup(t, func() {
+		h.Close()
+	})
+	return tf
 }
 
-// MustOpenField returns a new, opened field at a temporary path. Panic on error.
+// OpenField returns a new, opened field at a temporary path.
-func MustOpenField(opts FieldOption) *TestField {
+func OpenField(t *testing.T, opts FieldOption) *TestField {
-	f := NewTestField(opts)
+	f := NewTestField(t, opts)
-	if err := f.Open(); err != nil {
-		panic(err)
-	}
 	return f
 }
 
 // Close closes the field and removes the underlying data.
 func (f *TestField) Close() error {
+	if f.idx != nil {
+		panicOn(f.idx.holder.txf.CloseIndex(f.idx))
+	}
 	defer os.RemoveAll(f.Path())
 	return f.Field.Close()
 }
 
 // Reopen closes the index and reopens it.
 func (f *TestField) Reopen() error {
-	var err error
+	name := f.Field.Name()
-	if err := f.Field.Close(); err != nil {
+	if err := f.parent.Close(); err != nil {
+		f.parent = nil
 		return err
 	}
-
+	if err := f.parent.Open(); err != nil {
-	path, index, name := f.Path(), f.Index(), f.Name()
+		f.parent = nil
-	f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
-	if err != nil {
-		return err
-	}
-
-	if err := f.Open(); err != nil {
 		return err
 	}
+	f.Field = f.parent.Field(name)
 	return nil
 }
 
-func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
+func (f *TestField) MustSetBit(tx Tx, row, col uint64, ts ...time.Time) {
 	if len(ts) == 0 {
-		_, err := f.Field.SetBit(row, col, nil)
+		_, err := f.Field.SetBit(tx, row, col, nil)
 		if err != nil {
 			panic(err)
 		}
 	}
 	for _, t := range ts {
-		_, err := f.Field.SetBit(row, col, &t)
+		_, err := f.Field.SetBit(tx, row, col, &t)
 		if err != nil {
 			panic(err)
 		}
@@ -258,7 +272,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
 
 // Ensure field can open and retrieve a view.
 func TestField_CreateViewIfNotExists(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	// Create view.
@@ -283,7 +297,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
 }
 
 func TestField_SetTimeQuantum(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	// Set & retrieve time quantum.
@@ -302,44 +316,54 @@ func TestField_SetTimeQuantum(t *testing.T) {
 }
 
 func TestField_RowTime(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
+	// Obtain transaction.
+	tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
+	defer tx.Rollback()
+
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
 		t.Fatal(err)
 	}
 
-	f.MustSetBit(1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
 
-	if r, err := f.RowTime(1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
+	panicOn(tx.Commit())
+
+	// obtain 2nd transaction to read it back.
+	tx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
+	defer tx.Rollback()
+
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
@@ -348,7 +372,9 @@ func TestField_RowTime(t *testing.T) {
 }
 
 func TestField_PersistAvailableShards(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -356,6 +382,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
@@ -366,10 +393,74 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap()
@@ -383,12 +474,14 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
-		t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+		t.Fatalf("unexpected available shards (reopen). expected: %v, \n but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+
 	}
 
 	bm1 := roaring.NewBitmap()
@@ -438,3 +531,389 @@ func TestBSIGroup_BaseDefaultValue(t *testing.T) {
 		}
 	}
 }
+
+func TestField_ApplyOptions(t *testing.T) {
+	for i, tt := range []struct {
+		opts    FieldOptions
+		expOpts FieldOptions
+	}{
+		{
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+		},
+	} {
+
+		fld := &Field{}
+		fld.options = *applyDefaultOptions(&FieldOptions{})
+
+		if err := fld.applyOptions(tt.opts); err != nil {
+			t.Fatal(err)
+		}
+
+		if fld.options.CacheType != tt.expOpts.CacheType {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheType value. expected: %s, but got: %s", i, tt.expOpts.CacheType, fld.options.CacheType)
+		} else if fld.options.CacheSize != tt.expOpts.CacheSize {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheSize value. expected: %d, but got: %d", i, tt.expOpts.CacheSize, fld.options.CacheSize)
+		}
+	}
+}
+
+// Ensure that importValue handles requiredDepth correctly.
+// This test sets the same column value to 1, then 8, then 1.
+// A previous bug was incorrectly determining bitDepth based
+// on the values in the import, and not taking existing values
+// into consideration. This would cause an import of 1/8/1
+// to result in a value of 9 instead of 1.
+func TestBSIGroup_importValue(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+		panicOn(qcx.Finish())
+		if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+		panicOn(qcx.Finish())
+	} // loop
+}
+
+func TestIntField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []int64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []int64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []int64{10},
+			expMax:    ValCount{Val: 10, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []int64{10, 20},
+			expMax:    ValCount{Val: 20, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []int64{10, 20, 10, 10, 20},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11, 44, 1},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importValue(qcx, test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+			panicOn(qcx.Finish())
+
+			shard := uint64(0)
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			// Rollback below manually, because we are in a loop.
+
+			maxvc, err := f.MaxForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+			tx.Rollback()
+		})
+	}
+}
+
+// Ensure we get errors when they are expected.
+func TestDecimalField_MinMaxBoundaries(t *testing.T) {
+	th := newTestHolder(t)
+
+	for i, test := range []struct {
+		scale  int64
+		min    pql.Decimal
+		max    pql.Decimal
+		expErr bool
+	}{
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 0),
+			max:    pql.NewDecimal(math.MaxInt64, 0),
+			expErr: true,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 3),
+			max:    pql.NewDecimal(math.MaxInt64, 3),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(-44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(1, 0),
+			max:    pql.NewDecimal(2, 0),
+			expErr: true,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(math.MinInt64, 18),
+			max:    pql.NewDecimal(math.MaxInt64, 18),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, 20),
+			max:    pql.NewDecimal(2, 20),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -1),
+			max:    pql.NewDecimal(2, -1),
+			expErr: false,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -19),
+			max:    pql.NewDecimal(2, -19),
+			expErr: true,
+		},
+	} {
+		t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
+			_, err := NewField(th, "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
+			if err != nil && test.expErr {
+				if !strings.Contains(err.Error(), "is not supported") {
+					t.Fatal(err)
+				}
+			} else if err != nil && !test.expErr {
+				t.Fatalf("did not expect error, but got: %s", err)
+			} else if err == nil && test.expErr {
+				t.Fatal("expected error, but got none")
+			}
+		})
+	}
+}
+
+func TestDecimalField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeDecimal(3))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []float64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []float64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []float64{10.1},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []float64{10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "another shard",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11, 44.39, 0.23},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importFloatValue(qcx, test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			shard := uint64(0)
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			defer tx.Rollback()
+
+			maxvc, err := f.MaxForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if !reflect.DeepEqual(maxvc, test.expMax) {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if !reflect.DeepEqual(minvc, test.expMin) {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}
+
+func TestBSIGroup_TxReopenDB(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+		panicOn(qcx.Finish())
+
+		if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+		panicOn(qcx.Finish())
+	} // loop
+
+	// the test: can we re-open a BSI fragment under badger/rbf.
+	_ = f.Reopen()
+}

field_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"math"
 	"testing"
 
@@ -23,12 +22,13 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a field can set & read a bsiGroup value.
 func TestField_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -36,15 +36,18 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value on field.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Read value.
-		if value, exists, err := f.Value(100); err != nil {
+		if value, exists, err := f.Value(tx, 100); err != nil {
 			t.Fatal(err)
 		} else if value != 21 {
 			t.Fatalf("unexpected value: %d", value)
@@ -53,7 +56,7 @@ func TestField_SetValue(t *testing.T) {
 		}
 
 		// Setting value should return no change.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if changed {
 			t.Fatal("expected no change")
@@ -61,7 +64,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("Overwrite", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -69,22 +72,25 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Set different value.
-		if changed, err := f.SetValue(100, 23); err != nil {
+		if changed, err := f.SetValue(tx, 100, 23); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Read value.
-		if value, exists, err := f.Value(100); err != nil {
+		if value, exists, err := f.Value(tx, 100); err != nil {
 			t.Fatal(err)
 		} else if value != 23 {
 			t.Fatalf("unexpected value: %d", value)
@@ -94,7 +100,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -102,14 +108,17 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound {
+		if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 
 	t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -117,14 +126,17 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow {
+		if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 
 	t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -132,19 +144,22 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
+		if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 }
 
 func TestField_NameRestriction(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
+	field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault())
 	if field != nil {
 		t.Fatalf("unexpected field name %s", err)
 	}
@@ -158,6 +173,7 @@ func TestField_NameValidation(t *testing.T) {
 		"under_score",
 		"abc123",
 		"trailing_",
+		"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
 	}
 	invalidFieldNames := []string{
 		"",
@@ -168,30 +184,32 @@ func TestField_NameValidation(t *testing.T) {
 		"abc def",
 		"camelCase",
 		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
+		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
 	for _, name := range validFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatalf("unexpected field name: %s %s", name, err)
 		}
 	}
 	for _, name := range invalidFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err == nil {
 			t.Fatalf("expected error on field name: %s", name)
 		}
 	}
 }
 
+const includeRemote = false // for calls to Index.AvailableShards(localOnly bool)
+
 // Ensure can update and delete available shards.
 func TestField_AvailableShards(t *testing.T) {
-	idx := test.MustOpenIndex()
+	idx := test.MustOpenIndex(t)
 	defer idx.Close()
 
 	f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -199,12 +217,15 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// It is okay to pass in a nil tx. f.SetBit will lazily instantiate Tx.
+	var tx pilosa.Tx
+
 	// Set values on shards 0 & 2, and verify.
-	if _, err := f.SetBit(0, 100, nil); err != nil {
+	if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
 		t.Fatal(err)
-	} else if _, err := f.SetBit(0, ShardWidth*2, nil); err != nil {
+	} else if _, err := f.SetBit(tx, 0, ShardWidth*2, nil); err != nil {
 		t.Fatal(err)
-	} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	} else if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -212,7 +233,7 @@ func TestField_AvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
 		t.Fatalf("adding remote shards: %v", err)
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 1, 2, 4}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -223,7 +244,50 @@ func TestField_AvailableShards(t *testing.T) {
 			t.Fatalf("removing shard %d: %v", i, err)
 		}
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 }
+
+func TestField_ClearValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex(t)
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatal(err)
+		}
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
+		// Set value on field.
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		if changed, err := f.ClearValue(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal(err)
+		}
+
+		// Read value.
+		if _, exists, err := f.Value(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if exists {
+			t.Fatal("expected value to not exist")
+		}
+	})
+}

filesystem.go

@@ -0,0 +1,42 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// Ensure nopFileSystem implements interface.
+var _ FileSystem = &nopFileSystem{}
+
+// FileSystem represents an interface for file system for serving the Lattice UI.
+type FileSystem interface {
+	New() (http.FileSystem, error)
+}
+
+func init() {
+	NopFileSystem = &nopFileSystem{}
+}
+
+// NopFileSystem represents a FileSystem that returns an error if called.
+var NopFileSystem FileSystem
+
+type nopFileSystem struct{}
+
+// New is a no-op implementation of FileSystem New method.
+func (n *nopFileSystem) New() (http.FileSystem, error) {
+	return nil, fmt.Errorf("file system not implemented")
+}

gendebug_test.go

@@ -0,0 +1,47 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationdebug
+
+package pilosa
+
+import (
+	"errors"
+	"fmt"
+	"runtime"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func examineResults() error {
+	runtime.GC()
+	stats, results := reportGenerations()
+	if len(stats) > 0 {
+		fmt.Printf("generation stats: %s\n", stats)
+	}
+	if len(results) == 0 {
+		return nil
+	}
+	if len(results) > 0 {
+		fmt.Printf("generations:\n")
+		for _, res := range results {
+			fmt.Printf("  %s\n", res)
+		}
+	}
+	return errors.New("outstanding generations detected")
+}
+
+func init() {
+	testhook.RegisterPostTestHook(examineResults)
+}

generation.go

@@ -0,0 +1,463 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	// "runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pkg/errors"
+)
+
+// generation represents one "generation" of opening a data file.
+// This is what determines when it's safe to unmap a data file, if it
+// got mapped, and handles closing/reopening files if we need to
+// manage file handle availability. It's an interface because this
+// lets us write simpler code for specific cases, rather than handling
+// the whole matrix of mapped/unmapped, staying open/being reopened,
+// etcetera.
+//
+// You create a generation by calling newGeneration with a file
+// path. If it succeeds in opening that path, it calls a provided
+// setup function with the data from the generation, and a flag
+// indicating whether the data is mmapped. If the setup function
+// fails, newGeneration cleans things up and closes. Otherwise,
+// it returns a generation.
+//
+// The generation itself uses runtime.SetFinalizer to clean up when
+// the last reference to it goes away. You should store a pointer
+// to the generation in any object which is reliant on the generation.
+//
+// When you anticipate a generation should be done (for instance,
+// opening a new generation), the old one gets marked done, which
+// stashes a timestamp in it. Later operations can check whether
+// the timestamp is a while back, and if so, complain that something
+// might be wrong.
+//
+// In some cases, we don't have enough open file limit to keep every
+// file actually open. To address this, use the `Transaction` function,
+// which ensures that the file is open, stores a reference to it in
+// a provided `*io.Writer`, and then restores the previous value of
+// the io.Writer when it's done. For instance, for a bitmap, this might
+// be used with `&b.OpWriter`.
+//
+// newGeneration takes an optional previous generation; it calls
+// that generation's Done function after running the provided setup,
+// and bumps the generation count.
+type generation interface {
+	// Transaction runs the given transaction with the generation's
+	// file open. If the **os.File parameter is
+	// non-nil, the generation's file will be open, and stored
+	// into that pointer, during the execution of func, after
+	// which the previous contents are restored. Otherwise
+	// the file may or may not be open during the operation.
+	Transaction(*io.Writer, func() error) error
+	// Done() should be called exactly once, to indicate that a
+	// generation is expected not to be in use for long -- for instance,
+	// when a new generation replaces it.
+	Done()
+	// Generation count.
+	Generation() int64
+	// ID indicates the source -- path and generation number -- that
+	// this generation represents.
+	ID() string
+	// Dead indicates whether this generation is Done.
+	Dead() bool
+	// Bytes reports the storage associated with this generation, if any.
+	// DO NOT USE THIS. Except if you're debugging mmap segfaults.
+	Bytes() []byte
+}
+
+type mmapGeneration struct {
+	mu         sync.Mutex // mutex guards modifiers of generation, not of data
+	transMu    sync.Mutex // guards transactions, specifically
+	path       string
+	id         string
+	file       *os.File
+	data       []byte
+	generation int64     // generation counter
+	dead       bool      // we think this generation is dead
+	deadSince  time.Time // when this generation was marked dead
+	retries    int       // for cases where we're retrying
+	logger     logger.Logger
+}
+
+func (m *mmapGeneration) Dead() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.dead
+}
+
+func (m *mmapGeneration) ID() string {
+	return m.id
+}
+
+func (m *mmapGeneration) Generation() int64 {
+	return m.generation
+}
+
+// Transaction runs an exclusive call, ensuring that the file is open if
+// the *io.Writer parameter is present.
+func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
+	m.transMu.Lock()
+	defer m.transMu.Unlock()
+	// HEY LOOK CAREFULLY AT THIS BIT:
+	// We can't just defer this unlock. We specifically want to be
+	// sure to unlock the regular mutex *before* this function is over,
+	// and if we error out trying to open the file, we want to do it
+	// even sooner. If we deferred this, the transaction would block
+	// *everything*, including things like sanity checks against the
+	// generation being Dead(), but also including the deferred
+	// re-close-the-file.
+	m.mu.Lock()
+	// if we've been asked for a file pointer, we need to ensure that
+	// our file is open, and that the file pointer to it is stored in
+	// the requested location, then revert that when we're done.
+	// if we aren't asked for a file pointer, nothing needs the file
+	// open.
+	if m.dead {
+		elapsed := time.Since(m.deadSince)
+		m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
+	}
+	if fileP != nil {
+		if m.file == nil {
+			// we ignore the shouldClose response here; if this
+			// fragment was previously not being kept open, we're
+			// going to stick with that.
+			_, err := m.openFile()
+			if err != nil {
+				m.mu.Unlock()
+				return err
+			}
+			defer func() {
+				// report a close error if we have no other error to report
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				err := m.closeFile()
+				if transactionErr == nil {
+					transactionErr = err
+				}
+			}()
+		}
+		var fileStash io.Writer
+		fileStash, *fileP = *fileP, m.file
+		defer func() {
+			*fileP = fileStash
+		}()
+	}
+	// We are done locking the generation itself for now.
+	m.mu.Unlock()
+	// wouldPanic := debug.SetPanicOnFault(true)
+	//	defer func() {
+	//		debug.SetPanicOnFault(wouldPanic)
+	//		if r := recover(); r != nil {
+	//			if err, ok := r.(error); ok {
+	//				// special case: if we caught a page fault, we diagnose that directly. sadly,
+	//				// we can't see the actual values that were used to generate this, probably.
+	//				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+	//					if transactionErr == nil {
+	//						transactionErr = errors.New("invalid memory access during transaction")
+	//					} else {
+	//						transactionErr = fmt.Errorf("invalid memory access during transaction, previous error %v", transactionErr)
+	//					}
+	//					return
+	//				}
+	//			}
+	//			if transactionErr == nil {
+	//				transactionErr = fmt.Errorf("panic during transaction: %v", r)
+	//			} else {
+	//				transactionErr = fmt.Errorf("panic during erroring transaction: panic %v, previous error %v", r, transactionErr)
+	//			}
+	//		}
+	//	}()
+	return fn()
+}
+
+func (m *mmapGeneration) Bytes() []byte {
+	return m.data
+}
+
+// Done marks the generation done, and closes its file, but may not unmap it.
+// It's still conceptually possible to end up doing a Transaction against a
+// done generation, but it's a red flag.
+func (m *mmapGeneration) Done() {
+	if m == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.dead {
+		oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
+			m.id, time.Now(), m.deadSince)
+		panic(oops)
+	}
+	m.dead = true
+	m.deadSince = time.Now()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error closing generation %s: %v", m.id, err)
+	}
+	// If we're not debugging, the finalizer won't have been enabled
+	// previously. Finalizers have non-zero cost, so having them not be
+	// created until they're needed seems rewarding?
+	if !generationDebug {
+		runtime.SetFinalizer(m, generationFinalizer)
+	}
+	endGeneration(m.id)
+	// note, Done() doesn't close the file; only the finalizer actually
+	// does the shutdown.
+}
+
+// Try to close the file if it's currently open.
+func (m *mmapGeneration) closeFile() error {
+	var lastErr error
+	// report the most serious error encountered, but still close
+	// file even if something else failed.
+	if m.file != nil {
+		if err := m.file.Sync(); err != nil {
+			lastErr = fmt.Errorf("sync: %s", err)
+		}
+		if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
+			lastErr = fmt.Errorf("unlock: %s", err)
+		}
+		if err := syswrap.CloseFile(m.file); err != nil {
+			lastErr = fmt.Errorf("close file: %s", err)
+		}
+		m.file = nil
+	}
+	return lastErr
+}
+
+// openFile ensures the file is open and locked, or fails. If it does
+// open the file, it will also report the "you need to close this file
+// when you're done" flag from syswrap.
+func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
+	if m.file != nil {
+		return false, nil
+	}
+	m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return false, err
+	}
+
+	// do we actually want this in every openFile? I don't know.
+	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		_ = syswrap.CloseFile(m.file)
+		m.file = nil
+		return false, fmt.Errorf("flock: %s", err)
+	}
+	return shouldClose, nil
+}
+
+func generationFinalizer(m *mmapGeneration) {
+	m.mu.Lock()
+	if !m.dead {
+		m.logger.Printf("finalizing generation %s which isn't dead yet\n",
+			m.id)
+	}
+	m.mu.Unlock()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("finalizing generation, closing file: %v\n", err)
+	}
+	if m.data != nil {
+		err := syswrap.Munmap(m.data)
+		if err != nil {
+			m.logger.Printf("finalizing generation, munmap: %v\n", err)
+		}
+		m.data = nil
+	}
+	finalizeGeneration(m.id)
+}
+
+// Cancel closes a generation out entirely. It cancels any finalizer,
+// unmaps any data, ends generation tracking, and closes any files.
+// It does each of these separately whether or not the others need to be done,
+// or succeed. It's used to handle failures from newGeneration; it makes sure
+// the generation isn't holding any resources and doesn't need to be cleaned
+// up otherwise.
+//
+// Mostly a helper function because there's several cases where newGeneration
+// might fail.
+func (m *mmapGeneration) Cancel() {
+	if m.data != nil {
+		_ = syswrap.Munmap(m.data)
+		m.data = nil
+	}
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error cancelling generation %s: %v", m.id, err)
+	}
+	runtime.SetFinalizer(m, nil)
+	m.dead = true
+	m.deadSince = time.Now()
+	cancelGeneration(m.id)
+}
+
+// newGeneration creates a new generation using the given file path. It
+// then calls the provided setup function with the allocated storage, a
+// file handle, the new generation, and a flag indicatting whether the storage
+// is memory-mapped. If the setup function returns a non-nil error, the
+// generation is cleaned up, and newGeneration fails. The setup function
+// also returns a boolean indicating whether it used the mapping; if it
+// didn't, newGeneration discards the mapping and returns a nil generation.
+//
+// If generationDebug is enabled, we track the generation even if no mapping
+// is actually in use, so we can verify that the tracking is working.
+//
+// On failure, newGeneration returns nil values for generation and func,
+// and an error. On success, the func returned is the close func to use
+// when the generation is no longer needed by the caller.
+func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
+	m := mmapGeneration{path: path, logger: logger}
+	if existing != nil {
+		m.generation = existing.Generation() + 1
+		m.retries = existing.(*mmapGeneration).retries
+		// we might keep a previous generation around just for its generation count.
+		if !existing.Dead() {
+			defer existing.Done()
+		}
+	}
+	shouldClose, err := m.openFile()
+	if err != nil {
+		return nil, err
+	}
+	m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
+	// possibly assign new generation ID if this one's been used, which can
+	// happen with reopens, especially during testing.
+	m.id = registerGeneration(m.id)
+	// if debugging, we always want the finalizer on so we notice if a
+	// generation is finalized without being closed. for non-debugging
+	// use, we only need it when the generation is closed.
+	if generationDebug {
+		runtime.SetFinalizer(&m, generationFinalizer)
+	}
+	// Mmap the underlying file so it can be zero copied.
+	var mapped bool
+	var data []byte
+	fi, err := m.file.Stat()
+	if err == nil && fi.Size() > 0 {
+		data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+		if err == syswrap.ErrMaxMapCountReached {
+			// I have no idea where/how to display this message.
+			m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
+		} else if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "mmap failed")
+		} else {
+			mapped = true
+		}
+	}
+	if data == nil && readData {
+		data, err = ioutil.ReadAll(m.file)
+		if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "failure file readall")
+		}
+	}
+	// if we got here, data's the expected data, so let's try to use it
+	mappedAny, err := setup(data, m.file, &m, mapped)
+
+	// if the setup failed, we unmap data if we previously mapped it,
+	// and exit. Note that having no data, or having only trivial
+	// data (like a zero-container Roaring file) isn't "failed".
+	if err != nil {
+		m.Cancel()
+		// Unless, that is, we think the file probably ought to
+		// be truncated: For instance, if a bitmap has a corrupted
+		// ops log, we could truncate that part of it and retry.
+		if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
+			m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
+			// close this generation, then try again. once.
+			m.retries++
+			err := os.Truncate(m.path, err.SuggestedLength())
+			if err != nil {
+				m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
+			}
+			return newGeneration(&m, path, readData, setup, logger)
+		}
+		return nil, err
+	}
+
+	if mapped {
+		// when generationDebug is on, we want to track this even
+		// if it's not being used.
+		if generationDebug || mappedAny {
+			// Advise the kernel that the mmap is accessed randomly.
+			// We don't care much about errors with this.
+			_ = madvise(data, syscall.MADV_RANDOM)
+			// store the data, so we can unmap it when this generation
+			// gets finalized.
+			m.data = data
+		} else {
+			// unmap the data and don't stash the pointer in this
+			// generation. It's not being used. This generation
+			// doesn't need to exist, yay.
+			unmapErr := syswrap.Munmap(data)
+			if unmapErr != nil {
+				m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
+			}
+		}
+	}
+	// shouldClose comes from underlying syswrap.OpenFile, which checks
+	// a count of open files to hint at us when we need to start closing
+	// files to preserve open file descriptor limit.
+	if shouldClose {
+		err := m.closeFile()
+		if err != nil {
+			m.logger.Printf("closing file to preserve open files failed: %v\n", err)
+		}
+	}
+	// It's possible that the generation has no actual data to track,
+	// because nothing's mapped, in which case there won't be any bitmap
+	// sources following this, just the fragment source. (Bitmaps won't
+	// be attached to the source unless they're actually mapped to it,
+	// or generationDebug is true). That's okay. We pay a tiny cost
+	// for the finalizer, but we also get higher confidence that it really
+	// does get cleaned up.
+	return &m, nil
+}
+
+// NopGeneration is used in fragment.openStorage() to short-circuit
+// generation stuff that only applies to RoaringTx; doesn't apply to RBFTx/BadgerTx/etc.
+type NopGeneration struct {
+}
+
+func (g *NopGeneration) Transaction(w *io.Writer, f func() error) error {
+	return f()
+}
+func (g *NopGeneration) Done() {}
+func (g *NopGeneration) Generation() int64 {
+	return 0
+}
+func (g *NopGeneration) ID() string {
+	return "NOP"
+}
+func (g *NopGeneration) Dead() bool {
+	return true
+}
+func (g *NopGeneration) Bytes() (ret []byte) {
+	return
+}

generation_debug.go

@@ -0,0 +1,164 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+const generationDebug = true
+
+type lifespan struct {
+	from, to, finalized time.Time
+	stack               []byte
+}
+
+var knownGenerations map[string]lifespan
+var knownGenerationLock sync.Mutex
+
+var timeZero time.Time
+
+var generationDebugVerbose bool
+
+// History reports the finalized/dead/created status of a span which we think
+// is in some way in error. It's shared between a couple of places.
+func (span *lifespan) History() string {
+	dead := "not dead"
+	finalized := "not finalized"
+	if span.finalized != timeZero {
+		finalized = fmt.Sprintf("finalized at %v", span.finalized)
+	}
+	if span.to != timeZero {
+		dead = fmt.Sprintf("dead at %v", span.to)
+	}
+	return fmt.Sprintf("%s, %s, created at %v at %s", dead, finalized, span.from, span.stack)
+}
+
+func (span *lifespan) reportHistory(reason string, id string) string {
+	return fmt.Sprintf("%s %s: %s", id, reason, span.History())
+}
+
+func registerGeneration(id string) string {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	if knownGenerations == nil {
+		knownGenerations = make(map[string]lifespan)
+	}
+	newSpan := lifespan{from: time.Now(), stack: debug.Stack()}
+	origId := id
+
+	// if you have more than 65k of the same file open, maybe you have bigger
+	// problems than this.
+	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
+		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
+		if generationDebugVerbose {
+			history := span.History()
+			fmt.Printf("new generation: adding suffix %s, previous %s\n",
+				suffix, history)
+		}
+		id = origId + suffix
+	}
+	if generationDebugVerbose {
+		fmt.Printf("new generation %s\n", id)
+	}
+	knownGenerations[id] = newSpan
+	return id
+}
+
+func endGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("ending generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero || span.to != timeZero {
+		panic(span.reportHistory("ending generation", id))
+	}
+	span.to = time.Now()
+	knownGenerations[id] = span
+}
+
+// cancelGeneration marks the generation as finalized. In principle it's
+// only used in cases where we just started a generation but something
+// went wrong. it's not fancier than this because of the weird cases
+// where the same generation shows up again, such as when closing and
+// reopening an index so we don't know about previous instances of the
+// same files.
+func cancelGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if exists {
+		span.finalized = time.Now()
+		span.to = span.finalized
+		knownGenerations[id] = span
+	}
+}
+
+func finalizeGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("finalizing generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero {
+		panic(span.reportHistory("finalizing", id))
+	}
+	span.finalized = time.Now()
+	knownGenerations[id] = span
+}
+
+func reportGenerations() (stats string, surviving []string) {
+	runtime.GC()
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	times := make([]int64, 0, len(knownGenerations))
+	for id, span := range knownGenerations {
+		if span.to == timeZero || span.finalized == timeZero {
+			surviving = append(surviving, span.reportHistory("surviving", id))
+		} else {
+			times = append(times, int64(span.finalized.Sub(span.to)))
+		}
+	}
+	stats = "no recorded finalized spans"
+	if len(times) > 0 {
+		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
+		var total int64
+		for _, d := range times {
+			total += d
+		}
+		var mean, median, p90, p99, worst int64
+		mean = total / int64(len(times))
+		median = times[len(times)/2]
+		p90 = times[(len(times)*9)/10]
+		p99 = times[(len(times)*99)/100]
+		worst = times[len(times)-1]
+		stats = fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst))
+	}
+	return stats, surviving
+}

generation_nodebug.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package pilosa
+
+const generationDebug = false
+
+func registerGeneration(id string) string {
+	return id
+}
+
+func endGeneration(id string) {
+}
+
+func cancelGeneration(id string) {
+}
+
+func finalizeGeneration(id string) {
+}
+
+//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
+func reportGenerations() []string { //nolint:unused,deadcode
+	return nil
+}

generation_test.go

@@ -0,0 +1,70 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationparanoia
+
+package pilosa
+
+import (
+	"runtime"
+	"testing"
+	"unsafe"
+)
+
+func TestGenerationPanic(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "none")
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	_ = f.Snapshot()
+	_ = f.row(0)
+	var prevData []byte
+
+	if f.gen.(*mmapGeneration).data == nil {
+		t.Fatalf("generation code didn't create a mapping, apparently?")
+	}
+	prevData = f.gen.(*mmapGeneration).data
+	f.mu.Lock()
+	_ = defaultSnapshotQueue.Immediate(f)
+	f.mu.Unlock()
+	runtime.GC()
+	for i := 0; i < (f.MaxOpN / 2); i++ {
+		_, _ = f.setBit(0, uint64(i*32)+23)
+	}
+	f.mu.Lock()
+	defaultSnapshotQueue.Await(f)
+	f.mu.Unlock()
+	runtime.GC()
+	newData := f.gen.(*mmapGeneration).data
+	if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
+		t.Fatalf("test can't run usefully, didn't get new data pointer")
+	}
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err := f.gen.Transaction(wp, func() error {
+		prevData[0] = 0x3c
+		return nil
+	})
+	if err == nil {
+		t.Fatalf("expected a panic to get caught, but nothing happened")
+	}
+	if err.Error() != "invalid memory access during transaction" {
+		t.Fatalf("expected \"invalid memory access during transaction\", got %q", err.Error())
+	}
+}

generator/slice.go

@@ -0,0 +1,51 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package generator
+
+import (
+	"math/rand"
+	"sort"
+)
+
+// Uint64Slice generates between [0, n) random uint64 numbers between min and max.
+func Uint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 {
+	a := make([]uint64, rand.Intn(n))
+	for i := range a {
+		a[i] = min + uint64(rand.Int63n(int64(max-min)))
+	}
+
+	if sorted {
+		sort.Sort(uint64Slice(a))
+	}
+
+	return a
+}
+
+// Uint64SetSlice returns the values in a uint64 set.
+func Uint64SetSlice(m map[uint64]struct{}) []uint64 {
+	a := make([]uint64, 0, len(m))
+	for v := range m {
+		a = append(a, v)
+	}
+	sort.Sort(uint64Slice(a))
+	return a
+}
+
+// uint64Slice represents a sortable slice of uint64 numbers.
+type uint64Slice []uint64
+
+func (u uint64Slice) Swap(i, j int)      { u[i], u[j] = u[j], u[i] }
+func (u uint64Slice) Len() int           { return len(u) }
+func (u uint64Slice) Less(i, j int) bool { return u[i] < u[j] }

gid.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2014 The Go Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 	* Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"runtime"
+	"strconv"
+	"sync"
+)
+
+// Sourced https://github.com/bradfitz/http2/blob/dc0c5c000ec33e263612939744d51a3b68b9cece/gotrack.go
+var goroutineSpace = []byte("goroutine ")
+var littleBuf = sync.Pool{
+	New: func() interface{} {
+		buf := make([]byte, 64)
+		return &buf
+	},
+}
+
+var _ = curGID // happy linter
+
+func curGID() uint64 {
+	if true {
+		return 0 // avoid doing too much work during production profiling.
+	}
+
+	bp := littleBuf.Get().(*[]byte)
+	defer littleBuf.Put(bp)
+	b := *bp
+	b = b[:runtime.Stack(b, false)]
+	// Parse the 4707 out of "goroutine 4707 ["
+	b = bytes.TrimPrefix(b, goroutineSpace)
+	i := bytes.IndexByte(b, ' ')
+	if i < 0 {
+		panic(fmt.Sprintf("No space found in %q", b))
+	}
+	b = b[:i]
+	n, err := parseUintBytes(b, 10, 64)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err))
+	}
+	return n
+}
+
+// parseUintBytes is like strconv.ParseUint, but using a []byte.
+func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
+	var cutoff, maxVal uint64
+
+	if bitSize == 0 {
+		bitSize = int(strconv.IntSize)
+	}
+
+	s0 := s
+	switch {
+	case len(s) < 1:
+		err = strconv.ErrSyntax
+		return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+
+	case 2 <= base && base <= 36:
+		// valid base; nothing to do
+
+	case base == 0:
+		// Look for octal, hex prefix.
+		switch {
+		case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
+			base = 16
+			s = s[2:]
+			if len(s) < 1 {
+				err = strconv.ErrSyntax
+				return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+			}
+		case s[0] == '0':
+			base = 8
+		default:
+			base = 10
+		}
+
+	default:
+		err = errors.New("invalid base " + strconv.Itoa(base))
+		return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+	}
+
+	n = 0
+	cutoff = cutoff64(base)
+	maxVal = 1<<uint(bitSize) - 1
+
+	for i := 0; i < len(s); i++ {
+		var v byte
+		d := s[i]
+		switch {
+		case '0' <= d && d <= '9':
+			v = d - '0'
+		case 'a' <= d && d <= 'z':
+			v = d - 'a' + 10
+		case 'A' <= d && d <= 'Z':
+			v = d - 'A' + 10
+		default:
+			n = 0
+			err = strconv.ErrSyntax
+			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+		}
+		if int(v) >= base {
+			n = 0
+			err = strconv.ErrSyntax
+			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+		}
+
+		if n >= cutoff {
+			// n*base overflows
+			n = 1<<64 - 1
+			err = strconv.ErrRange
+			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+		}
+		n *= uint64(base)
+
+		n1 := n + uint64(v)
+		if n1 < n || n1 > maxVal {
+			// n+v overflows
+			n = 1<<64 - 1
+			err = strconv.ErrRange
+			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+		}
+		n = n1
+	}
+
+	return n, nil
+}
+
+// Return the first number n such that n*base >= 1<<64.
+func cutoff64(base int) uint64 {
+	if base < 2 {
+		return 0
+	}
+	return (1<<64-1)/uint64(base) + 1
+}

go.mod

@@ -3,45 +3,51 @@ module github.com/pilosa/pilosa/v2
 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/benbjohnson/immutable v0.2.0
 	github.com/boltdb/bolt v1.3.1
 	github.com/cespare/xxhash v1.1.0
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
+	github.com/glycerine/lmdb-go v1.9.32
 	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/gogo/protobuf v1.2.0
+	github.com/gogo/protobuf v1.2.1
-	github.com/golang/protobuf v1.3.1
+	github.com/golang/protobuf v1.3.3
-	github.com/google/go-cmp v0.2.0
+	github.com/google/go-cmp v0.4.0
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/lib/pq v1.8.0
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v0.9.3
+	github.com/prometheus/client_golang v1.0.0
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/client_model v0.1.0
+	github.com/prometheus/prom2json v1.3.0
+	github.com/rakyll/statik v0.1.7
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/spf13/cobra v0.0.3
+	github.com/spf13/cobra v1.0.0
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.3.1
+	github.com/spf13/viper v1.4.0
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
-	go.uber.org/atomic v1.4.0 // indirect
+	github.com/zeebo/blake3 v0.0.4
-	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
+	golang.org/x/mod v0.3.0
-	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect
+	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
+	golang.org/x/text v0.3.3 // indirect
-	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/grpc v1.28.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
+	vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
 )
 
-go 1.11
+go 1.13

go.sum

@@ -1,56 +1,113 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
+github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/benbjohnson/immutable v0.2.0 h1:t0rW3lNFwfQ85IDO1mhMbumxdVSti4nnVaal4r45Oio=
+github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
+github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
+github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
+github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
+github.com/glycerine/lmdb-go v1.9.32 h1:thLnzCykFcmn2rACYnwpR4ovYauLNKaAuk+xj7YMbS0=
+github.com/glycerine/lmdb-go v1.9.32/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI=
 github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
@@ -69,17 +126,34 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@@ -93,23 +167,40 @@ github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVT
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
+github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
+github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@@ -118,75 +209,160 @@ github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAri
 github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
 github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
 github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
+github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
+github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
+github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
+vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible h1:GWnLrAdetgJM0Co5bwwczO49iFZBSInpyGAT77BP9Y0=
+vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0=

gopsutil/systeminfo.go

@@ -15,6 +15,7 @@
 package gopsutil
 
 import (
+	"math"
 	"runtime"
 	"strings"
 
@@ -114,6 +115,13 @@ func (s *systemInfo) collectPlatformInfo() error {
 		}
 		s.cpuModel = infos[0].ModelName
 		s.cpuMHz = computeMHz(s.cpuModel)
+		if s.cpuMHz < 0 {
+			s.cpuMHz = int(math.Round(infos[0].Mhz))
+		}
+		if s.cpuMHz < 0 {
+			// This is supposed to be unsigned.
+			s.cpuMHz = 0
+		}
 
 		// gopsutil reports core and clock speed info inconsistently
 		// by OS

gossip/gossip.go

@@ -104,6 +104,7 @@ func (g *memberSet) Open() (err error) {
 // Close attempts to gracefully leave the cluster, and finally calls shutdown
 // after (at most) a timeout period.
 func (g *memberSet) Close() error {
+	g.eventReceiver.Close()
 	leaveErr := g.memberlist.Leave(5 * time.Second)
 	shutdownErr := g.memberlist.Shutdown()
 	if leaveErr != nil || shutdownErr != nil {
@@ -324,16 +325,20 @@ func (g *memberSet) LocalState(join bool) []byte {
 		Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
 	}
 	for _, idx := range m.Schema.Indexes {
-		is := &pilosa.IndexStatus{Name: idx.Name}
+		is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
+
 		for _, f := range idx.Fields {
 			availableShards := roaring.NewBitmap()
 			if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
-				availableShards = field.AvailableShards()
+				availableShards = field.AvailableShards(false)
 			}
-			is.Fields = append(is.Fields, &pilosa.FieldStatus{
+
+			fs := &pilosa.FieldStatus{
 				Name:            f.Name,
+				CreatedAt:       f.CreatedAt,
 				AvailableShards: availableShards,
-			})
+			}
+			is.Fields = append(is.Fields, fs)
 		}
 		m.Indexes = append(m.Indexes, is)
 	}
@@ -362,8 +367,9 @@ func (g *memberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type eventReceiver struct {
-	ch   chan memberlist.NodeEvent
+	ch     chan memberlist.NodeEvent
-	papi *pilosa.API
+	closed chan struct{}
+	papi   *pilosa.API
 
 	logger logger.Logger
 }
@@ -372,6 +378,7 @@ type eventReceiver struct {
 func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver {
 	ger := &eventReceiver{
 		ch:     make(chan memberlist.NodeEvent, 1),
+		closed: make(chan struct{}),
 		logger: logger,
 		papi:   papi,
 	}
@@ -385,7 +392,10 @@ func (g *eventReceiver) NotifyJoin(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
@@ -394,7 +404,10 @@ func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
@@ -403,13 +416,25 @@ func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}:
+	case <-g.closed:
+	}
+}
+
+func (g *eventReceiver) Close() {
+	close(g.closed)
 }
 
 func (g *eventReceiver) listen() {
 	var nodeEventType pilosa.NodeEventType
 	for {
-		e := <-g.ch
+		var e memberlist.NodeEvent
+		select {
+		case <-g.closed:
+			return
+		case e = <-g.ch:
+		}
 		switch e.Event {
 		case memberlist.NodeJoin:
 			nodeEventType = pilosa.NodeJoin

handler.go

@@ -16,6 +16,9 @@ package pilosa
 
 import (
 	"encoding/json"
+
+	"github.com/pilosa/pilosa/v2/tracing"
+	"github.com/pkg/errors"
 )
 
 // QueryRequest represent a request to process a query.
@@ -42,12 +45,21 @@ type QueryRequest struct {
 	// If true, indicates that query is part of a larger distributed query.
 	// If false, this request is on the originating node.
 	Remote bool
+
+	// Should we profile this query?
+	Profile bool
+
+	// Additional data associated with the query, in cases where there's
+	// row-style inputs for precomputed values.
+	EmbeddedData []*Row
 }
 
 // QueryResponse represent a response from a processed query.
 type QueryResponse struct {
 	// Result for each top-level query call.
-	// Can be a Bitmap, Pairs, or uint64.
+	// The result type differs depending on the query; types
+	// include: Row, RowIdentifiers, GroupCounts, SignedRow,
+	// ValCount, Pair, Pairs, bool, uint64.
 	Results []interface{}
 
 	// Set of column attribute objects matching IDs returned in Result.
@@ -55,6 +67,9 @@ type QueryResponse struct {
 
 	// Error during parsing or execution.
 	Err error
+
+	// Profiling data, if any
+	Profile *tracing.Profile
 }
 
 // MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
@@ -68,9 +83,11 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(struct {
 		Results        []interface{}    `json:"results"`
 		ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
+		Profile        *tracing.Profile `json:"profile,omitempty"`
 	}{
 		Results:        resp.Results,
 		ColumnAttrSets: resp.ColumnAttrSets,
+		Profile:        resp.Profile,
 	})
 }
 
@@ -96,33 +113,146 @@ var NopHandler Handler = nopHandler{}
 
 // ImportValueRequest describes the import request structure
 // for a value (BSI) import.
+// Note: no RowIDs here. have to convert BSI Values into RowIDs internally.
 type ImportValueRequest struct {
-	Index      string
+	Index          string
-	Field      string
+	IndexCreatedAt int64
-	Shard      uint64
+	Field          string
-	ColumnIDs  []uint64
+	FieldCreatedAt int64
-	ColumnKeys []string
+	// if Shard is MaxUint64 (an impossible shard value), this
-	Values     []int64
+	// indicates that the column IDs may come from multiple shards.
+	Shard        uint64
+	ColumnIDs    []uint64 // e.g. weather stationID
+	ColumnKeys   []string
+	Values       []int64 // e.g. temperature, humidity, barometric pressure
+	FloatValues  []float64
+	StringValues []string
+	Clear        bool
+}
+
+// AtomicRecord applies all its Ivr and Ivr atomically, in a Tx.
+// The top level Shard has to agree with Ivr[i].Shard and the Iv[i].Shard
+// for all i included (in Ivr and Ir). The same goes for the top level Index: all records
+// have to be writes to the same Index. These requirements are checked.
+//
+type AtomicRecord struct {
+	Index string
+	Shard uint64
+
+	Ivr []*ImportValueRequest // BSI values
+	Ir  []*ImportRequest      // other field types, e.g. single bit
+}
+
+func (ivr *ImportValueRequest) Len() int           { return len(ivr.ColumnIDs) }
+func (ivr *ImportValueRequest) Less(i, j int) bool { return ivr.ColumnIDs[i] < ivr.ColumnIDs[j] }
+func (ivr *ImportValueRequest) Swap(i, j int) {
+	ivr.ColumnIDs[i], ivr.ColumnIDs[j] = ivr.ColumnIDs[j], ivr.ColumnIDs[i]
+	if len(ivr.Values) > 0 {
+		ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
+	} else if len(ivr.FloatValues) > 0 {
+		ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
+	} else if len(ivr.StringValues) > 0 {
+		ivr.StringValues[i], ivr.StringValues[j] = ivr.StringValues[j], ivr.StringValues[i]
+	}
+}
+
+// Validate ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) Validate() error {
+	return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt)
+}
+
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if ivr.Index == "" || ivr.Field == "" {
+		return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
+	}
+	if len(ivr.ColumnIDs) != 0 && len(ivr.ColumnKeys) != 0 {
+		return errors.Errorf("must pass either column ids or keys, but not both")
+	}
+	var valueSetCount int
+	if len(ivr.Values) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.FloatValues) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.StringValues) != 0 {
+		valueSetCount++
+	}
+	if valueSetCount > 1 {
+		return errors.Errorf("must pass ints, floats, or strings but not multiple")
+	}
+	if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 {
+		if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
+}
+
+// ImportColumnAttrsRequest describes the import request structure
+// for a ColumnAttr import.
+type ImportColumnAttrsRequest struct {
+	AttrKey        string
+	ColumnIDs      []uint64
+	AttrVals       []string
+	Shard          int64
+	Index          string
+	IndexCreatedAt int64
 }
 
 // ImportRequest describes the import request structure
-// for an import.
+// for an import.  BSIs use the ImportValueRequest instead.
 type ImportRequest struct {
-	Index      string
+	Index          string
-	Field      string
+	IndexCreatedAt int64
-	Shard      uint64
+	Field          string
-	RowIDs     []uint64
+	FieldCreatedAt int64
-	ColumnIDs  []uint64
+	Shard          uint64
-	RowKeys    []string
+	RowIDs         []uint64
-	ColumnKeys []string
+	ColumnIDs      []uint64
-	Timestamps []int64
+	RowKeys        []string
+	ColumnKeys     []string
+	Timestamps     []int64
+	Clear          bool
 }
 
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
+		if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
+}
+
+const (
+	RequestActionSet       = "set"
+	RequestActionClear     = "clear"
+	RequestActionOverwrite = "overwrite"
+)
+
 // ImportRoaringRequest describes the import request structure
 // for an import containing roaring-encoded data.
 type ImportRoaringRequest struct {
-	Clear bool
+	IndexCreatedAt int64
-	Views map[string][]byte
+	FieldCreatedAt int64
+	Clear          bool
+	Action         string // [set, clear, overwrite]
+	Block          int
+	Views          map[string][]byte
+	Direct         bool
+}
+
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 {
+		if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
 }
 
 // ImportResponse is the structured response of an import.
@@ -153,6 +283,9 @@ type TranslateKeysRequest struct {
 	Index string
 	Field string
 	Keys  []string
+
+	// it's a awkward name, just to keep backward compatibility with go-pilosa and idk.
+	NotWritable bool
 }
 
 // TranslateKeysResponse is the structured response of a key
@@ -160,3 +293,45 @@ type TranslateKeysRequest struct {
 type TranslateKeysResponse struct {
 	IDs []uint64
 }
+
+// TranslateIDsRequest describes the structure of a request
+// for a batch of id translations.
+type TranslateIDsRequest struct {
+	Index string
+	Field string
+	IDs   []uint64
+}
+
+// TranslateIDsResponse is the structured response of a id
+// translation request.
+type TranslateIDsResponse struct {
+	Keys []string
+}
+
+// InspectRequestParams represents the parts of an InspectRequest that
+// aren't generic holder filtering attributes.
+type InspectRequestParams struct {
+	Containers bool // include container details
+	Checksum   bool // perform checksums
+}
+
+// InspectRequest represents a request for a possibly-partial
+// holder inspection, using a provided holder filter and inspect-specific
+// parameters.
+type InspectRequest struct {
+	HolderFilterParams
+	InspectRequestParams
+}
+
+// InspectResponse contains the structured results for an InspectRequest.
+// It may some day be expanded to include metadata about views or indexes.
+type InspectResponse struct {
+	Fragments []struct {
+		Index string
+		Field string
+		View  string
+		Shard int64
+		Path  string
+		Info  *FragmentInfo
+	}
+}

hash/blake3.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hash
+
+import (
+	cryptorand "crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"sync"
+
+	"github.com/zeebo/blake3"
+	"golang.org/x/mod/sumdb/dirhash"
+)
+
+// Blake3Hasher is a thread/goroutine safe way to
+// obtain a blake3 cryptographic hash of input []byte.
+// Reference https://github.com/BLAKE3-team/BLAKE3
+// suggests it is 6x faster than BLAKE2B.
+// The Go github.com/zeebo/blake3 version is
+// AVX2 and SSE4.1 accelerated.
+type Blake3Hasher struct {
+	hasher   *blake3.Hasher
+	hasherMu sync.Mutex
+}
+
+// NewBlake3Hasher returns a new Blake3Hasher.
+func NewBlake3Hasher() *Blake3Hasher {
+	return &Blake3Hasher{
+		hasher: blake3.New(),
+	}
+}
+
+// CryptoHash writes the blake3 cryptographic hash of
+// input into buffer and returns it.
+// Like the standard libary's hash.Hash interface's Sum() method,
+// the buffer is re-used and overwritten
+// to avoid allocation. The caller determines the byte length of
+// the outputCryptohash by the size of the supplied buffer
+// slice, and this will be exactly equal to the supplies bytes.
+// In this way, shorter or longer hashes can be provided as
+// needed.
+func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) {
+	w.hasherMu.Lock()
+	w.hasher.Reset()
+
+	// "Write implements part of the hash.Hash interface. It never returns an error."
+	//  -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write
+	_, _ = w.hasher.Write(input)
+
+	// Digest.Read reads data from the hasher into buffer.
+	// "It always fills the entire buffer and never errors."
+	//   -- https://godoc.org/github.com/zeebo/blake3#Digest
+	_, _ = w.hasher.Digest().Read(buffer)
+
+	// no chance of panic, so avoid any defer cost.
+	w.hasherMu.Unlock()
+
+	return buffer
+}
+
+// Blake3sum16 might be slower because we allocate a new hasher every time, but
+// it is more conenient for writing debug code. It returns
+// a 16 byte hash as a hexidecimal string.
+func Blake3sum16(input []byte) string {
+	hasher := blake3.New()
+
+	_, _ = hasher.Write(input)
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	return fmt.Sprintf("%x", buf)
+}
+
+// CryptoRandInt64 uses crypto/rand to get an random int64
+func CryptoRandInt64() int64 {
+	c := 8
+	b := make([]byte, c)
+	_, err := cryptorand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+	r := int64(binary.LittleEndian.Uint64(b))
+	return r
+}
+
+// HashOfDir returns the hash of the local file system directory dir
+func HashOfDir(path string) string {
+	prefix := ""
+	h, err := dirhash.HashDir(path, prefix, dirhash.Hash1)
+	if err != nil {
+		panic(err)
+	}
+
+	return h
+}

hash/blake3_test.go

@@ -0,0 +1,90 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hash
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func TestBlake3Hasher(t *testing.T) {
+
+	hasher := NewBlake3Hasher()
+	hash := make([]byte, 16)
+	input := []byte("hello world")
+	hash = hasher.CryptoHash(input, hash)
+	expected := "d74981efa70a0c880b8d8c1985d075db"
+	observed := hex.EncodeToString(hash)
+	if observed != expected {
+		panic(fmt.Sprintf("expected hash:'%v' but observed hash '%v'", expected, observed))
+	}
+
+	obs2 := Blake3sum16(input)
+	if obs2 != expected {
+		panic(fmt.Sprintf("expected hash:'%v' but observed hash from blake2sum16: '%v'", expected, obs2))
+	}
+}
+
+func TestCryptoRandInt64(t *testing.T) {
+	rnd := CryptoRandInt64()
+	if rnd == 0 {
+		panic("cryptoRandInt64() gave 0, very high odds it has broken")
+	}
+}
+
+func TestHashOfDir(t *testing.T) {
+	dir, err := testhook.TempDir(t, "TestHashOfDir-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := path.Join(dir, "A", "B")
+	if err := os.MkdirAll(b, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	c := path.Join(dir, "A", "C")
+	if err := os.MkdirAll(c, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	bmessage := []byte("hello B\n")
+	if err := ioutil.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	cmessage := []byte("hello C\n")
+	if err := ioutil.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	hsh := HashOfDir(dir)
+
+	c2message := []byte("hello C2\n")
+	if err := ioutil.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	hsh2 := HashOfDir(dir)
+	if hsh2 == hsh {
+		t.Fatal("HashOfDir did not detect 1 byte change")
+	}
+}

holder_internal_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,309 +15,235 @@
 package pilosa
 
 import (
-	"io/ioutil"
+	"context"
+	"fmt"
 	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
 	"testing"
 
-	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
-type tHolder struct {
+var _ = fmt.Printf
-	*Holder
+
+type testHolderOperator struct {
+	indexSeen, indexProcessed       int
+	fieldSeen, fieldProcessed       int
+	viewSeen, viewProcessed         int
+	fragmentSeen, fragmentProcessed int
+	waitHere                        chan struct{}
 }
 
-// Close closes the holder and removes all underlying data.
+func (t *testHolderOperator) CheckIndex(string) (bool, bool) {
-func (h *tHolder) Close() error {
+	t.indexSeen++
-	defer os.RemoveAll(h.Path)
+	return true, true
-	return h.Holder.Close()
 }
 
-// Reopen instantiates and opens a new holder.
+func (t *testHolderOperator) CheckField(string, string) (bool, bool) {
-// Note that the holder must be Closed first.
+	t.fieldSeen++
-func (h *tHolder) Reopen() error {
+	return true, true
-	path, logger := h.Path, h.Holder.Logger
-	h.Holder = NewHolder()
-	h.Holder.Path = path
-	h.Holder.Logger = logger
-	return h.Holder.Open()
 }
 
-func newHolder() *tHolder {
+func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-")
+	t.viewSeen++
+	return true, true
+}
+
+func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool {
+	t.fragmentSeen++
+	return true
+}
+
+func (t *testHolderOperator) ProcessIndex(*Index) error {
+	t.indexProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessField(*Field) error {
+	t.fieldProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessView(*view) error {
+	t.viewProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessFragment(*fragment) error {
+	if t.waitHere != nil {
+		<-t.waitHere
+	}
+	t.fragmentProcessed++
+	return nil
+}
+
+func makeHolder(tb testing.TB) (*Holder, string, error) {
+	path, err := testhook.TempDir(tb, "pilosa-")
 	if err != nil {
-		panic(err)
+		return nil, "", err
+	}
+	h := NewHolder(path, nil)
+	return h, path, h.Open()
+}
+
+func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
+
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
 	}
 
-	h := &tHolder{Holder: NewHolder()}
+	f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
-	h.Path = path
-	return h
-}
-
-// MustCreateFieldIfNotExists returns a given field. Panic on error.
-func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
-	f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
 	if err != nil {
-		panic(err)
+		t.Fatalf("setting bit: %v", err)
 	}
-	return f
+	_, err = f.SetBit(nil, rowID, columnID, nil)
-}
-
-// MustCreateIndexIfNotExists returns a given index. Panic on error.
-func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index {
-	idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
 	if err != nil {
-		panic(err)
+		t.Fatalf("setting bit: %v", err)
-	}
-	return idx
-}
-
-// SetBit clears a bit on the given field.
-func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) {
-	f := h.MustCreateFieldIfNotExists(index, field)
-	_, err := f.SetBit(rowID, columnID, nil)
-	if err != nil {
-		panic(err)
 	}
 }
 
-// Row returns a Row for a given field.
+func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
-func (h *tHolder) Row(index, field string, rowID uint64) *Row {
+
-	f := h.MustCreateFieldIfNotExists(index, field)
+	//shard := columnID / ShardWidth
-	row, err := f.Row(rowID)
+
-	if err != nil {
+	// hmm... if its a new holder, meta data isn't there, so ask for it.
-		panic(err)
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	panicOn(err)
+
+	f := idx.Field(field)
+	if f == nil {
+		t.Fatalf("no such field '%v'", field)
 	}
-	return row
+
+	row, err := f.Row(nil, rowID)
+	if err != nil {
+		t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err)
+	}
+
+	cols := row.Columns()
+	if len(cols) == 0 {
+		t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID)
+	}
+
+	for _, c := range cols {
+		if c == columnID {
+			return // ok, found it.
+		}
+	}
+	t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID)
 }
 
-func TestHolder_Optn(t *testing.T) {
+func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
-	t.Run("ErrViewPermission", func(t *testing.T) {
+	if testHasBit(t, h, index, field, rowID, columnID) {
-		if os.Geteuid() == 0 {
+		t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID)
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := newHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
-			t.Fatal(err)
-		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer func() {
-			// we don't care about a failure here
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755)
-		}()
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-	t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := newHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
-			t.Fatal(err)
-		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer func() {
-			// we don't care about a failure here
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755)
-		}()
-
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-	t.Run("ErrFragmentCachePermission", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := newHolder()
-		defer h.Close()
-
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
-			t.Fatal(err)
-		} else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
-			t.Fatal(err)
-		} else if _, err := field.SetBit(0, 0, nil); err != nil {
-			t.Fatal(err)
-		} else if err := view.Fragment(0).FlushCache(); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer func() {
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644)
-		}()
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-}
-
-// Ensure holder can clean up orphaned fragments.
-func TestHolderCleaner_CleanHolder(t *testing.T) {
-	cluster := NewTestCluster(2)
-
-	// Create a local holder.
-	hldr0 := newHolder()
-	defer hldr0.Close()
-
-	// Mock 2-node, fully replicated cluster.
-	cluster.ReplicaN = 2
-
-	cluster.nodes[0].URI = NewTestURIFromHostPort("localhost", 0)
-
-	// Create fields on nodes.
-	for _, hldr := range []*tHolder{hldr0} {
-		hldr.MustCreateFieldIfNotExists("i", "f")
-		hldr.MustCreateFieldIfNotExists("i", "f0")
-		hldr.MustCreateFieldIfNotExists("y", "z")
-	}
-
-	// Set data on the local holder.
-	hldr0.SetBit("i", "f", 0, 10)
-	hldr0.SetBit("i", "f", 0, 4000)
-	hldr0.SetBit("i", "f", 2, 20)
-	hldr0.SetBit("i", "f", 3, 10)
-	hldr0.SetBit("i", "f", 120, 10)
-	hldr0.SetBit("i", "f", 200, 4)
-
-	hldr0.SetBit("i", "f0", 9, ShardWidth+5)
-
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
-
-	// Set highest shard.
-	err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1))
-	if err != nil {
-		t.Fatalf("adding remote shards: %v", err)
-	}
-	err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2))
-	if err != nil {
-		t.Fatalf("adding remote shards: %v", err)
-	}
-
-	// Keep replication the same and ensure we get the expected results.
-	cluster.ReplicaN = 2
-
-	// Set up cleaner for replication 2.
-	cleaner2 := holderCleaner{
-		Node:    cluster.nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner2.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*tHolder{hldr0} {
-		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
-		}
-
-		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
-	}
-
-	// Change replication factor to ensure we have fragments to remove.
-	cluster.ReplicaN = 1
-
-	// Set up cleaner for replication 1.
-	cleaner1 := holderCleaner{
-		Node:    cluster.nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner1.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*tHolder{hldr0} {
-		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		f := hldr.fragment("i", "f0", viewStandard, 1)
-		if f != nil {
-			t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
-		}
-
-		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
 	}
 }
 
-// Ensure holder can reopen.
+func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool {
-func TestHolderCleaner_Reopen(t *testing.T) {
+
-	h := NewHolder()
+	idx := h.Index(index)
-	h.Path = "path"
+	if idx == nil {
-	err := h.Open()
+		return false // not even an index by this name. Obviously no hot bits either.
-	if err != nil {
-		t.Fatalf("couldn't open holder: %v", err)
 	}
-	err = h.Close()
+
-	if err != nil {
+	f := idx.Field(field)
-		t.Fatalf("couldn't close holder: %v", err)
+	if f == nil {
+		return false
 	}
-	err = h.Open()
+
+	row, err := f.Row(nil, rowID)
 	if err != nil {
-		t.Fatalf("couldn't open holder: %v", err)
+		return false
 	}
-	err = h.Close()
+
+	cols := row.Columns()
+	if len(cols) == 0 {
+		return false
+	}
+
+	for _, c := range cols {
+		if c == columnID {
+			return true // ok, found it.
+		}
+	}
+	return false
+}
+
+func TestHolderOperatorProcess(t *testing.T) {
+	h, path, err := makeHolder(t)
 	if err != nil {
-		t.Fatalf("couldn't close holder: %v", err)
+		t.Fatalf("creating holder: %v", err)
+	}
+	defer os.RemoveAll(path)
+	defer h.Close()
+
+	// Write bits to separate indexes.
+	testSetBit(t, h, "i0", "f", 100, 200)
+	testSetBit(t, h, "i1", "f", 100, 200)
+	testSetBit(t, h, "i1", "f", 100, 12345678)
+
+	testOp := testHolderOperator{}
+	ctx := context.Background()
+	err = h.Process(ctx, &testOp)
+	if err != nil {
+		t.Fatalf("processing holder: %v", err)
+	}
+	expected := testHolderOperator{
+		indexSeen: 2, indexProcessed: 2,
+		fieldSeen: 2, fieldProcessed: 2,
+		viewSeen: 2, viewProcessed: 2,
+		fragmentSeen: 3, fragmentProcessed: 3,
+	}
+	if testOp != expected {
+		t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
+	}
+}
+
+func TestHolderOperatorCancel(t *testing.T) {
+	h, path, err := makeHolder(t)
+	if err != nil {
+		t.Fatalf("creating holder: %v", err)
+	}
+	defer os.RemoveAll(path)
+	defer h.Close()
+
+	// Write bits to separate indexes.
+	testSetBit(t, h, "i0", "f", 100, 200)
+	testSetBit(t, h, "i1", "f", 100, 200)
+	testSetBit(t, h, "i1", "f", 100, 12345678)
+
+	// Here, we want to ensure that the operation gets cancelled
+	// successfully. In practice we expect it to process one fragment, then
+	// end up blocked on the waitHere, then get cancelled... But the
+	// waitHere blockage isn't really something holder.Process can do
+	// anything about, so we close the channel, so two fragments are
+	// processed. But in theory you could end up with only one fragment
+	// processed if this goroutine managed to cancel before the processor
+	// gets to the next fragment. Point is, it shouldn't hit all three,
+	// because the checks against the cancellation should fire before it
+	// gets there.
+	testOp := testHolderOperator{waitHere: make(chan struct{})}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		err = h.Process(ctx, &testOp)
+		close(done)
+	}()
+	testOp.waitHere <- struct{}{}
+	cancel()
+	close(testOp.waitHere)
+	<-done
+	if err != context.Canceled {
+		t.Fatalf("processing holder: expected context.Canceled, got %v", err)
+	}
+	testOp.waitHere = nil
+	expected := testHolderOperator{
+		indexSeen: 2, indexProcessed: 2,
+		fieldSeen: 2, fieldProcessed: 2,
+		viewSeen: 2, viewProcessed: 2,
+		fragmentSeen: 3, fragmentProcessed: 3,
+	}
+	if testOp == expected {
+		t.Fatalf("holder processor did not cancel. expected something other than %#v", expected)
 	}
 }

holder_test.go

@@ -17,6 +17,7 @@ package pilosa_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,12 +26,14 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 func TestHolder_Open(t *testing.T) {
 	t.Run("ErrIndexName", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 
 		bufLogger := test.NewBufferLogger()
 		h.Holder.Logger = bufLogger
@@ -57,7 +60,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -76,7 +79,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -96,7 +99,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@@ -105,27 +108,32 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil {
+		} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0000); err != nil {
 			t.Fatal(err)
 		}
 		defer func() {
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0755)
+			_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar"), 0755)
 		}()
 		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 	t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+		var idx *pilosa.Index
+		var err error
+
+		if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+		}
+
+		if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil {
+		} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", ".meta"), 2); err != nil {
 			t.Fatal(err)
 		}
 
@@ -134,16 +142,20 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+		var idx *pilosa.Index
+		var err error
+		if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+		}
+
+		if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil {
+		} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", ".data"), 2); err != nil {
 			t.Fatal(err)
 		}
 
@@ -153,43 +165,70 @@ func TestHolder_Open(t *testing.T) {
 	})
 
 	t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
+		roaringOnlyTest(t)
+
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+		var idx *pilosa.Index
+		var err error
+		if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+		}
+
+		var shard uint64
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		defer tx.Rollback()
+
+		if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
 			t.Fatal(err)
-		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+		} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := tx.Commit(); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
+		} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
 			t.Fatal(err)
 		}
 		defer func() {
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0644)
+			_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0644)
 		}()
 		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		roaringOnlyTest(t)
+
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+		var idx *pilosa.Index
+		var err error
+		if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
 			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+		}
+
+		var shard uint64
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		if err != nil {
 			t.Fatal(err)
-		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+		}
+		defer tx.Rollback()
+
+		if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := tx.Commit(); err != nil {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
+		} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
 			t.Fatal(err)
 		}
 
@@ -197,12 +236,125 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
+	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		roaringOnlyTest(t)
 
+		h := test.MustOpenHolder(t)
+		defer h.Close()
+
+		idx, err := h.CreateIndex("foo", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var shard uint64
+		tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
+		defer tx.Rollback()
+
+		if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := tx.Commit(); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := h.Reopen(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ForeignIndex", func(t *testing.T) {
+		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
+			h := test.MustOpenHolder(t)
+			defer h.Close()
+
+			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else {
+				_, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("nonexistent"))
+				if err == nil {
+					t.Fatalf("expected error: %s", pilosa.ErrForeignIndexNotFound)
+				} else if errors.Cause(err) != pilosa.ErrForeignIndexNotFound {
+					t.Fatalf("expected error: %s, but got: %s", pilosa.ErrForeignIndexNotFound, err)
+				}
+			}
+		})
+
+		// Foreign index zzz is opened after foo/bar.
+		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
+			h := test.MustOpenHolder(t)
+			defer h.Close()
+
+			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("zzz")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Foreign index aaa is opened before foo/bar.
+		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
+			h := test.MustOpenHolder(t)
+			defer h.Close()
+
+			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("aaa")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Try to re-create existing index
+		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
+			h := test.MustOpenHolder(t)
+			defer h.Close()
+
+			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err = h.CreateIndex("aaa", pilosa.IndexOptions{}); err == nil {
+				t.Fatalf("expected: ConflictError, got: nil")
+			} else if _, ok := err.(pilosa.ConflictError); !ok {
+				t.Fatalf("expected: ConflictError, got: %s", err)
+			}
+
+			idx2, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if idx1 != idx2 {
+				t.Fatalf("expected the same indexes, got: %s and %s", idx1.Name(), idx2.Name())
+			}
+		})
+	})
 }
 
 func TestHolder_HasData(t *testing.T) {
 	t.Run("IndexDirectory", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if ok, err := h.HasData(); ok || err != nil {
@@ -219,7 +371,7 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek", func(t *testing.T) {
-		h := test.NewHolder()
+		h := test.NewHolder(t)
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)
@@ -236,10 +388,12 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek at missing directory", func(t *testing.T) {
-		h := test.NewHolder()
-
 		// Ensure that hasData is false when dir doesn't exist.
-		h.Path = "bad-path"
+
+		// Note that we are intentionally not using test.NewHolder,
+		// because we want to create a Holder object with an invalid path,
+		// rather than creating a valid holder with a temporary path.
+		h := pilosa.NewHolder("bad-path", nil)
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)
@@ -249,7 +403,8 @@ func TestHolder_HasData(t *testing.T) {
 
 // Ensure holder can delete an index and its underlying files.
 func TestHolder_DeleteIndex(t *testing.T) {
-	hldr := test.MustOpenHolder()
+
+	hldr := test.MustOpenHolder(t)
 	defer hldr.Close()
 
 	// Write bits to separate indexes.
@@ -277,39 +432,43 @@ func TestHolder_DeleteIndex(t *testing.T) {
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index y: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f0: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field z in y: %v", err)
 	}
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
+	if err != nil {
+		t.Fatalf("creating field b in y: %v", err)
+	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder.
 	hldr0.SetBit("i", "f", 0, 10)
@@ -321,6 +480,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 
 	// Set a bit to create the fragment.
 	hldr0.SetBit("y", "z", 0, 0)
+	hldr0.SetBit("y", "b", 0, 0) // rowID = 0 means false
 
 	// Set data on the remote holder.
 	hldr1.SetBit("i", "f", 0, 4000)
@@ -331,11 +491,15 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
 
-	err = c[0].Server.SyncData()
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+4) // true
+	hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
+
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
-	err = c[1].Server.SyncData()
+	err = c.GetNode(1).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 1: %v", err)
 	}
@@ -365,16 +529,128 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
 			t.Errorf("unexpected columns(%d/y/z): %+v", i, a)
 		}
+
+		if a := hldr.Row("y", "b", 0).Columns(); !reflect.DeepEqual(a, []uint64{0, (3 * ShardWidth) + 5}) {
+			t.Errorf("unexpected false columns(%d/y/b): %+v", i, a)
+		}
+		if a := hldr.Row("y", "b", 1).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 7}) {
+			t.Errorf("unexpected true columns(%d/y/b): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder can sync with a remote holder and respects
+// the row boundaries of the block.
+func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	blockEdge := uint64(pilosa.HashBlockSize)
+
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
+
+	// Set data on the local holder.
+	hldr0.SetBit("i", "f", blockEdge-1, 10)
+	hldr0.SetBit("i", "f", blockEdge, 20)
+
+	// Set the same data on one of the replicas
+	// so that we have a quorum.
+	hldr1.SetBit("i", "f", blockEdge-1, 10)
+	hldr1.SetBit("i", "f", blockEdge, 20)
+
+	// Leave the third replica empty to force a block merge.
+	//
+	err = c.GetNode(0).Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", blockEdge-1).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Errorf("unexpected columns(%d/block 0): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", blockEdge).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d/block 1): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder correctly handles clears during block sync.
+func TestHolderSyncer_Clears(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
+
+	// Set data on the local holder that should be cleared
+	// because it's the only instance of this value.
+	hldr0.SetBit("i", "f", 0, 30)
+
+	// Set similar data on the replicas, but
+	// different from what's on local. This should end
+	// up being set on all replicas
+	hldr1.SetBit("i", "f", 0, 20)
+	hldr2.SetBit("i", "f", 0, 20)
+
+	err = c.GetNode(0).Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d): %+v", i, a)
+		}
 	}
 }
 
 // Ensure holder can sync time quantum views with a remote holder.
 func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
@@ -383,17 +659,17 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 
 	quantum := "D"
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder for node0.
 	t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC)
@@ -404,7 +680,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	// Set data on node1.
 	hldr1.SetBitTime("i", "f", 0, 22, &t2)
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -419,3 +695,129 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 		}
 	}
 }
+
+// Ensure holder can sync integer views with a remote holder.
+func TestHolderSyncer_IntField(t *testing.T) {
+	t.Run("BasicSync", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		var idx0 *pilosa.Index
+		_ = idx0
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		_ = idx0
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+
+		// Set data on the local holder for node0. columnID=1, value=1
+		hldr0.SetValue("i", "f", 1, 1)
+
+		// in c0 expect the 1 bit
+
+		// Set data on node1. columnID=2, value=2
+		idx1 := hldr1.SetValue("i", "f", 2, 2)
+		_ = idx1
+
+		err = c.GetNode(0).Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+
+		// expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1.
+
+		// Problem is: data at c1 was replaced by c0, instead of being merged with existing c1.
+		// Problem is: data at c0 did not receive and merge the c1 data.
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
+				// expects exists==true, a==1
+				t.Errorf("unexpected value(node%d/0): a:%d, exists: %v", i, a, exists)
+			}
+			if a, exists := hldr.Value("i", "f", 2); exists {
+				t.Errorf("unexpected value(node%d/1): a:%d, exists: %v", i, a, exists)
+			}
+		}
+	})
+
+	t.Run("MultiShard", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		var idx0 *pilosa.Index
+		_ = idx0
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		_ = idx0
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+
+		// Set data on the local holder for node0.
+		hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
+		hldr0.SetValue("i", "f", 3*pilosa.ShardWidth, 32)
+		hldr0.SetValue("i", "f", 4*pilosa.ShardWidth, math.MinInt32)
+		hldr0.SetValue("i", "f", 7*pilosa.ShardWidth, math.MinInt32)
+
+		// Set data on node1.
+		hldr1.SetValue("i", "f", 0*pilosa.ShardWidth, 2)
+		hldr1.SetValue("i", "f", 2*pilosa.ShardWidth, 22)
+		hldr1.SetValue("i", "f", 4*pilosa.ShardWidth, math.MaxInt32)
+		hldr1.SetValue("i", "f", 7*pilosa.ShardWidth, math.MaxInt32)
+
+		// Primary for shards (for index "i"):
+		// node0: [0,3,7]
+		// node1: [1,2,4]
+
+		err = c.GetNode(0).Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+		err = c.GetNode(1).Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 1: %v", err)
+		}
+
+		// dump the badger keys for both c0 and c1
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a := hldr.Range("i", "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+			if a := hldr.Range("i", "f", pql.LT, 0); !reflect.DeepEqual(a.Columns(), []uint64{7 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+		}
+	})
+}

http/client.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/encoding/proto"
@@ -290,6 +291,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -392,6 +394,13 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits
 
 	// Get the coordinator node; all bits are sent to the
 	// primary translate store (i.e. coordinator).
+	// TODO... is that right^^?
+	// RESPONSE: It looks like in ctl/import.go, we could change the
+	// logic in ImportCommand.importBits() to only use ImportK
+	// when useRowKeys = true. It's no longer necessary to
+	// send column key translations to the coordinator (although
+	// it should still work). As far as I know, the only thing
+	// that uses ImportK is the pilosa import sub-command.
 	nodes, err := c.Nodes(ctx)
 	if err != nil {
 		return fmt.Errorf("getting nodes: %s", err)
@@ -487,6 +496,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -553,6 +563,35 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 	return nil
 }
 
+// ImportValue2 is a simplified ImportValue method which just uses the
+// ImportValueRequest instead of splitting up ImportValue and
+// ImportValueK... it also supports importing float values. The idea
+// being that (assuming it works) this will become the default (and be
+// renamed) for 2.0, and we can deprecate the other methods.
+func (c *InternalClient) ImportValue2(ctx context.Context, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.NewImportValue")
+	defer span.Finish()
+
+	buf, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Errorf("marshal import request: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, req.Index, req.Shard)
+	if err != nil {
+		return errors.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, req.Index, req.Field, buf, options); err != nil {
+			return errors.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+	return nil
+}
+
 // ImportValueK bulk imports keyed field values to a host.
 func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
@@ -645,6 +684,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	}
 	httpReq.Header.Set("Content-Type", "application/x-protobuf")
 	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("X-Pilosa-Row", "roaring")
 	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -667,6 +707,57 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	return nil
 }
 
+// ImportColumnAttrs does bulk import of column attrs
+func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	url := fmt.Sprintf("%s/index/%s/import-column-attrs", uri, index)
+
+	// Marshal data to protobuf.
+	data, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Wrap(err, "marshal import-column-attrs request")
+	}
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("Content-Type", "application/x-protobuf")
+	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("X-Pilosa-Row", "roaring")
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	dec := json.NewDecoder(resp.Body)
+	rbody := &pilosa.ImportResponse{}
+	err = dec.Decode(rbody)
+	// Decode can return EOF when no error occurred. helpful!
+	if err != nil && err != io.EOF {
+		return errors.Wrap(err, "decoding response body")
+	}
+	if rbody.Err != "" {
+		return errors.Wrap(errors.New(rbody.Err), "importing roaring")
+	}
+
+	return nil
+}
+
 // ExportCSV bulk exports data for a single shard from a host to CSV format.
 func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
@@ -791,18 +882,28 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
 	}
 
 	// convert pilosa.FieldOptions to fieldOptions
+	//
+	// TODO this kind of sucks because it's one more place that needs
+	// changes when we change anything with field options (and there
+	// are a lot of places already). It's not clear to me that this is
+	// providing a lot of value, but I think this kind of validation
+	// should probably happen in the field anyway??
 	fieldOpt := fieldOptions{
 		Type: opt.Type,
 		Keys: &opt.Keys,
 	}
-	if fieldOpt.Type == "set" {
+	if fieldOpt.Type == pilosa.FieldTypeSet {
 		fieldOpt.CacheType = &opt.CacheType
 		fieldOpt.CacheSize = &opt.CacheSize
-	} else if fieldOpt.Type == "int" {
+	} else if fieldOpt.Type == pilosa.FieldTypeInt {
 		fieldOpt.Min = &opt.Min
 		fieldOpt.Max = &opt.Max
-	} else if fieldOpt.Type == "time" {
+	} else if fieldOpt.Type == pilosa.FieldTypeTime {
 		fieldOpt.TimeQuantum = &opt.TimeQuantum
+	} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
+		fieldOpt.Min = &opt.Min
+		fieldOpt.Max = &opt.Max
+		fieldOpt.Scale = &opt.Scale
 	}
 
 	// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
@@ -909,6 +1010,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
 	req.Header.Set("Content-Type", "application/protobuf")
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Accept", "application/protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	resp, err := c.executeRequest(req.WithContext(ctx))
@@ -999,7 +1101,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
-			return nil, pilosa.ErrFieldNotFound
+			return nil, errors.Wrap(pilosa.ErrFieldNotFound, field)
 		}
 		return nil, err
 	}
@@ -1026,21 +1128,291 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Connection", "keep-alive")
 
 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
 		return errors.Wrap(err, "executing request")
 	}
-	return errors.Wrap(resp.Body.Close(), "closing response body")
+	defer resp.Body.Close()
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	return errors.Wrap(err, "draining SendMessage response body")
+}
+
+// TranslateKeysNode function is mainly called to translate keys from coordinator node.
+// If coordinator node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound.
+func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{
+		Index:       index,
+		Field:       field,
+		Keys:        keys,
+		NotWritable: !writable,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateKeysRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/keys")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			return nil, errors.Wrap(pilosa.ErrTranslatingKeyNotFound, err.Error())
+		}
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateKeysResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.IDs, nil
+}
+
+// TranslateIDsNode sends an id translation request to a specific node.
+func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{
+		Index: index,
+		Field: field,
+		IDs:   ids,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateIDsRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/ids")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateIDsResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.Keys, nil
+}
+
+func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transactions")
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating transactions request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	trnsMap := make(map[string]*pilosa.Transaction)
+	err = json.NewDecoder(resp.Body).Decode(&trnsMap)
+	return trnsMap, errors.Wrap(err, "json decoding")
+}
+
+func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.StartTransaction")
+	defer span.Finish()
+	buf, err := json.Marshal(&pilosa.Transaction{
+		ID:        id,
+		Timeout:   timeout,
+		Exclusive: exclusive,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshalling payload")
+	}
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating post transaction request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+	if resp.StatusCode == 409 {
+		err = pilosa.ErrTransactionExclusive
+	} else if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FinishTransaction")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id+"/finish")
+	req, err := http.NewRequest("POST", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating finish transaction request")
+	}
+
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetTransaction")
+	defer span.Finish()
+
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating get transaction request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+type executeOpts struct {
+	// giveRawResponse instructs executeRequest not to process the
+	// respStatusCode and try to extract errors or whatever.
+	giveRawResponse bool
+}
+
+type executeRequestOption func(*executeOpts)
+
+func giveRawResponse(b bool) executeRequestOption {
+	return func(eo *executeOpts) {
+		eo.giveRawResponse = b
+	}
 }
 
 // executeRequest executes the given request and checks the Response. For
 // responses with non-2XX status, the body is read and closed, and an error is
 // returned. If the error is nil, the caller must ensure that the response body
 // is closed.
-func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) {
+func (c *InternalClient) executeRequest(req *http.Request, opts ...executeRequestOption) (*http.Response, error) {
+	eo := &executeOpts{}
+	for _, opt := range opts {
+		opt(eo)
+	}
+
 	tracing.GlobalTracer.InjectHTTPHeaders(req)
+	req.Close = false
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		if resp != nil {
@@ -1048,6 +1420,9 @@ func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, erro
 		}
 		return nil, errors.Wrap(err, "getting response")
 	}
+	if eo.giveRawResponse {
+		return resp, nil
+	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		defer resp.Body.Close()
 		buf, err := ioutil.ReadAll(resp.Body)
@@ -1278,3 +1653,103 @@ func nodePathToURL(node *pilosa.Node, path string) url.URL {
 		Path:   path,
 	}
 }
+
+// RetrieveTranslatePartitionFromURI returns a ReadCloser which contains the data of the
+// specified translate partition from the specified node. Caller *must* close the returned
+// ReadCloser or risk leaking goroutines/tcp connections.
+func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pilosa.URI) (io.ReadCloser, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveTranslatePartitionFromURI")
+	defer span.Finish()
+
+	node := &pilosa.Node{
+		URI: uri,
+	}
+
+	u := nodePathToURL(node, "/internal/translate/data")
+	u.RawQuery = url.Values{
+		"index":     {index},
+		"partition": {strconv.FormatInt(int64(partition), 10)},
+	}.Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			return nil, pilosa.ErrFragmentNotFound
+		}
+		return nil, err
+	}
+
+	return resp.Body, nil
+}
+func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	vals := url.Values{}
+	vals.Set("remote", strconv.FormatBool(remote))
+	url := fmt.Sprintf("%s/internal/translate/index/%s/%d", uri, index, partitionID)
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, rddbdata)
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pilosa.URI, index, field string, remote bool, rddbdata io.Reader) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	vals := url.Values{}
+	vals.Set("remote", strconv.FormatBool(remote))
+	url := fmt.Sprintf("%s/internal/translate/field/%s/%s", uri, index, field)
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, rddbdata)
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}

http/handler_internal_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 // Test custom UnmarshalJSON for postIndexRequest object
@@ -99,8 +100,8 @@ func stringPtr(s string) *string {
 	return &s
 }
 
-func int64Ptr(i int64) *int64 {
+func decimalPtr(d pql.Decimal) *pql.Decimal {
-	return &i
+	return &d
 }
 
 // Test fieldOption validation.
@@ -135,10 +136,10 @@ func TestFieldOptionValidation(t *testing.T) {
 		// FieldType: Int
 		{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
 		{json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"},
-		{json: `{"options": {"type": "int", "min": 0, "max": 1000}}`, expected: postFieldRequest{Options: fieldOptions{
+		{json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{
 			Type: pilosa.FieldTypeInt,
-			Min:  int64Ptr(0),
+			Min:  decimalPtr(pql.NewDecimal(0, 0)),
-			Max:  int64Ptr(1000),
+			Max:  decimalPtr(pql.NewDecimal(1001, 0)),
 		}}},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"},

http/handler_test.go

@@ -15,11 +15,13 @@
 package http_test
 
 import (
+	"encoding/json"
 	"net"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/test"
 )
 
 func TestHandlerOptions(t *testing.T) {
@@ -40,3 +42,39 @@ func TestHandlerOptions(t *testing.T) {
 		t.Fatalf("expected error making handler without options, got nil")
 	}
 }
+
+func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
+	tests := []struct {
+		name string
+		tr   *http.TransactionResponse
+	}{
+		{
+			name: "nil transaction",
+			tr:   &http.TransactionResponse{},
+		},
+		{
+			name: "empty transaction",
+			tr:   &http.TransactionResponse{Transaction: &pilosa.Transaction{}},
+		},
+	}
+
+	for _, tst := range tests {
+		t.Run(tst.name, func(t *testing.T) {
+			data, err := json.Marshal(tst.tr)
+			if err != nil {
+				t.Fatalf("marshaling: %v", err)
+			}
+
+			mytr := &http.TransactionResponse{}
+			err = json.Unmarshal(data, mytr)
+			if err != nil {
+				t.Fatalf("unmarshalling: %v", err)
+			}
+
+			if mytr.Error != tst.tr.Error {
+				t.Errorf("errors mismatch:exp/got \n%v\n%v", tst.tr.Error, mytr.Error)
+			}
+			test.CompareTransactions(t, tst.tr.Transaction, mytr.Transaction)
+		})
+	}
+}

http/translator.go

@@ -22,19 +22,26 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"sync"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/logger"
 )
 
 func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc {
+	return GetOpenTranslateReaderWithLockerFunc(client, nopLocker{})
+}
+
+func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) pilosa.OpenTranslateReaderFunc {
 	return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
-		return openTranslateReader(ctx, nodeURL, offsets, client)
+		return openTranslateReader(ctx, nodeURL, offsets, client, locker)
 	}
 }
 
-func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client) (pilosa.TranslateEntryReader, error) {
+func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client, locker sync.Locker) (pilosa.TranslateEntryReader, error) {
 	r := NewTranslateEntryReader(ctx, client)
+	r.locker = locker
+
 	r.URL = nodeURL + "/internal/translate/data"
 	r.Offsets = offsets
 	if err := r.Open(); err != nil {
@@ -43,9 +50,16 @@ func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.Tra
 	return r, nil
 }
 
+type nopLocker struct{}
+
+func (nopLocker) Lock()   {}
+func (nopLocker) Unlock() {}
+
 // TranslateEntryReader represents an implementation of pilosa.TranslateEntryReader.
 // It consolidates all index & field translate entries into a single reader.
 type TranslateEntryReader struct {
+	locker sync.Locker
+
 	ctx    context.Context
 	cancel func()
 
@@ -70,7 +84,7 @@ func NewTranslateEntryReader(ctx context.Context, client *http.Client) *Translat
 	if client == nil {
 		client = http.DefaultClient
 	}
-	r := &TranslateEntryReader{HTTPClient: client, Logger: logger.NopLogger}
+	r := &TranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger}
 	r.ctx, r.cancel = context.WithCancel(ctx)
 	return r
 }
@@ -116,7 +130,10 @@ func (r *TranslateEntryReader) Close() error {
 		r.cancel()
 	}
 	if r.body != nil {
-		return r.body.Close()
+		r.locker.Lock()
+		err := r.body.Close()
+		r.locker.Unlock()
+		return err
 	}
 	return nil
 }
@@ -124,5 +141,8 @@ func (r *TranslateEntryReader) Close() error {
 // ReadEntry reads the next entry from the stream into entry.
 // Returns io.EOF at the end of the stream.
 func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
+	r.locker.Lock()
+	defer r.locker.Unlock()
+
 	return r.dec.Decode(&entry)
 }

http/translator_test.go

@@ -17,6 +17,7 @@ package http_test
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -35,7 +36,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 			cluster := test.MustRunCluster(t, 1)
 			defer cluster.Close()
-			primary := cluster[0]
+			primary := cluster.GetNode(0)
 
 			hldr := test.Holder{Holder: primary.Server.Holder()}
 			index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
@@ -151,3 +152,84 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		})
 	*/
 }
+
+func benchmarkSetup(b *testing.B, ctx context.Context, key string, nkeys int) (string, pilosa.TranslateOffsetMap, func()) {
+	b.Helper()
+
+	cluster := test.MustRunCluster(b, 1)
+	primary := cluster.GetNode(0)
+
+	idx := primary.MustCreateIndex(b, "i", pilosa.IndexOptions{})
+	fld := primary.MustCreateField(b, idx.Name(), "f", pilosa.OptFieldKeys())
+	offset := make(pilosa.TranslateOffsetMap)
+	offset.SetIndexPartitionOffset(idx.Name(), 0, 1)
+	offset.SetFieldOffset(idx.Name(), fld.Name(), 1)
+
+	// Set data on the primary node.
+	for k := 0; k < nkeys; k++ {
+		if _, err := primary.API.Query(ctx, &pilosa.QueryRequest{
+			Index: idx.Name(),
+			Query: fmt.Sprintf(`Set(%d, %s="%s%[1]d")`, k, fld.Name(), key),
+		}); err != nil {
+			b.Fatalf("quering api: %+v", err)
+		}
+	}
+
+	return primary.URL(), offset, func() {
+		b.Helper()
+
+		if err := primary.API.DeleteIndex(ctx, idx.Name()); err != nil {
+			panic(err)
+		}
+		if err := cluster.Close(); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func benchmarkReadEntry(b *testing.B, r pilosa.TranslateEntryReader, key string, nkeys int) {
+	var entry pilosa.TranslateEntry
+	for k := 0; k < nkeys; k++ {
+		if err := r.ReadEntry(&entry); err != nil {
+			b.Fatalf("reading entry: %+v", err)
+		}
+		if entry.Key != fmt.Sprintf("%s%d", key, k) {
+			b.Fatalf("got: %s, expected: %s%d", entry.Key, key, k)
+		}
+	}
+}
+
+const (
+	key   = "foo"
+	nkeys = 1000
+)
+
+func BenchmarkReadEntryNoMutex(b *testing.B) {
+	ctx := context.Background()
+	url, offset, teardown := benchmarkSetup(b, ctx, key, nkeys)
+	defer teardown()
+
+	for n := 0; n < b.N; n++ {
+		r, err := http.GetOpenTranslateReaderFunc(nil)(ctx, url, offset)
+		if err != nil {
+			b.Fatalf("opening translate reader: %+v", err)
+		}
+		benchmarkReadEntry(b, r, key, nkeys)
+		r.Close()
+	}
+}
+
+func BenchmarkReadEntryWithMutex(b *testing.B) {
+	ctx := context.Background()
+	url, offset, teardown := benchmarkSetup(b, ctx, key, nkeys)
+	defer teardown()
+
+	for n := 0; n < b.N; n++ {
+		r, err := http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset)
+		if err != nil {
+			b.Fatalf("opening translate reader: %+v", err)
+		}
+		benchmarkReadEntry(b, r, key, nkeys)
+		r.Close()
+	}
+}