Add new Docker build process and continuous delivery

Add benchmarking of autogenerated ID data

.circleci/config.yml

@@ -1,195 +1,268 @@
-version: 2
-defaults: &defaults
-  working_directory: /go/src/github.com/pilosa/pilosa
-  docker:
-    - image: circleci/golang:1.13
-  environment:
-    GO111MODULE: "on"
-fast-checkout: &fast-checkout
-  attach_workspace:
-    at: .
-jobs:
-  setup:
-    <<: *defaults
+version: 2.1
+
+executors:
+  golang:
+    parameters:
+      version:
+        type: string
+        default: "1.15.8"
+      resource_class:
+        type: string
+        default: medium
+    docker:
+      - image: circleci/golang:<< parameters.version >>
+    resource_class: << parameters.resource_class >>
+    working_directory: /go/src/github.com/pilosa/pilosa
+
+commands:
+  add-github-auth:
+    steps:
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
+  restore-mod-cache:
     steps:
-      - checkout
       - restore_cache:
-          keys:
-            - mod-cache-{{ checksum "go.sum" }}
-      - run: "go mod download"
+          key: mod-cache-{{ checksum "go.sum" }}
+  save-mod-cache:
+    steps:
       - save_cache:
           key: mod-cache-{{ checksum "go.sum" }}
           paths:
             - /go/pkg/mod/
-      - persist_to_workspace:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
+  checkout-plus:
     steps:
-      - *fast-checkout
+      - add-github-auth
+      - checkout
+      - restore-mod-cache
+
+jobs:
+  setup:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod download
+      - save-mod-cache
+  check-license-headers:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
+      - checkout-plus
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
       - run: make golangci-lint
-  test-build-arm:
-    <<: *defaults
+  go-mod-tidy:
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - run: go mod tidy
+      - run: git diff --exit-code -- go.mod go.sum
+  check-changelog-label:
+    executor:
+      name: golang
+    steps:
+      - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
+      - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
+  test-build-arm:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
-    <<: *defaults
+  test:
+    parameters:
+      resource_class:
+        type: string
+        default: medium
+      golang_version:
+        type: string
+        default: "1.15.8"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
+      resource_class: << parameters.resource_class >>
     steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
+      - checkout-plus
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - setup_remote_docker
       - run: make clustertests-build
-  prerelease:
-    <<: *base-test
-    steps:
-      - *fast-checkout
-      - run: make prerelease
-      - store_artifacts:
-          path: build
-      - persist_to_workspace:
-          root: .
-          paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
-      - run: make release
+      - checkout-plus
+      - attach_workspace:
+          at: .
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker-release
       - store_artifacts:
           path: build
       - persist_to_workspace:
           root: .
           paths: build
-  prerelease-upload:
-    docker:
-      - image: circleci/python:2.7-jessie
+  publish_release:
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
-      - run: sudo pip install awscli
-      - run: make prerelease-upload
-  dockerhub-upload:
-    <<: *defaults
+      - attach_workspace:
+          at: .
+      - run: go get github.com/tcnksm/ghr
+      - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
+  docker-build:
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker GO_VERSION=1.15.8
+      - run: docker run pilosa:$(git describe --tags) help
+  dockerhub-upload-unstable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - setup_remote_docker
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
-      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
+      - run: docker run pilosa:$(git describe --tags) help
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker
+      - run: make docker
+      - run: docker run pilosa:$(git describe --tags) help
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
-  version: 2
-  test:
+  build:
     jobs:
-      - setup
+      - setup:
+          context: molecula
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
+          context: molecula
           requires:
             - setup
       - check-license-headers:
+          context: molecula
+          requires:
+            - setup
+      - go-mod-tidy:
+          context: molecula
+          requires:
+            - setup
+      - check-changelog-label:
+          context: molecula
           requires:
             - setup
       - test-build-arm:
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          resource_class: large
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-race:
+          matrix:
+            parameters:
+              golang_version: ["1.14.15", "1.15.8"]
+      - test:
+          name: << matrix.test_make_target >>
+          resource_class: xlarge
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-386:
-          requires:
-            - setup
-      - test-golang-1.13:
-          requires:
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
+          matrix:
+            parameters:
+              test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_bolt"]
+      - test:
+          name: test-shardwidth-22
+          context: molecula
+          shard_width: "22"
+          resource_class: large
           requires:
             - setup
       - cluster-tests:
+          context: molecula
           requires:
             - setup
-      - prerelease:
+      - docker-build:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
       - release:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
+      - publish_release:
+          context: molecula
           requires:
-            - prerelease
-      - dockerhub-upload:
+            - release
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
+          filters:
+            branches:
+              only: master
+      - dockerhub-upload-stable:
+          context: molecula
+          requires:
+            - setup
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

.dockerignore

@@ -0,0 +1,5 @@
+lattice/.git
+lattice/node_modules
+lattice/build
+statik/statik.go
+build

.gitignore

@@ -4,3 +4,9 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*
+pilosa
+*.dot

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lattice"]
+	path = lattice
+	url = git@github.com:molecula/lattice.git

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,16 +1,46 @@
-FROM golang:1.13.0 as builder
+ARG GO_VERSION=latest
 
-COPY . pilosa
+#######################
+### Lattice builder ###
+#######################
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+FROM moleculacorp/nodejs:latest as lattice-builder
+WORKDIR /lattice
 
-FROM alpine:3.9.4
+COPY lattice/package.json ./
+COPY lattice/yarn.lock ./
+RUN yarn install
 
-LABEL maintainer "dev@pilosa.com"
+COPY lattice ./
+RUN yarn build
+
+######################
+### Pilosa builder ###
+######################
+
+FROM golang:${GO_VERSION} as pilosa-builder
+ARG MAKE_FLAGS
+WORKDIR /pilosa
+
+RUN go get github.com/rakyll/statik
+
+COPY . ./
+COPY --from=lattice-builder /lattice/build /lattice
+RUN /go/bin/statik -src=/lattice -dest=/pilosa
+
+RUN make build ${MAKE_FLAGS}
+
+#####################
+### Pilosa runner ###
+#####################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=builder /go/bin/pilosa /pilosa
+COPY --from=pilosa-builder /pilosa/build/pilosa /
 
 COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
@@ -18,5 +48,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
+    && make install FLAGS="-a -mod=vendor"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,23 +1,40 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race  test-txstore-rbf lattice
 
 CLONE_URL=github.com/pilosa/pilosa
+MOD_VERSION=v2
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
-BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
+VARIANT = Molecula
+GO=go
+GOOS=$(shell $(GO) env GOOS)
+GOARCH=$(shell $(GO) env GOARCH)
+BINOUT=build
+FLAGS=-o $(BINOUT)/pilosa
+VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
-GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
-RELEASE ?= 0
-RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
-BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)"
+GO_VERSION=1.15.8
+DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+TEST_TAGS = roaringparanoia
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+UNAME := $(shell uname -s)
+ifeq ($(UNAME), Darwin)
+    IS_MACOS:=1
+else
+    IS_MACOS:=0
+endif
+
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export CGO_ENABLED=0
 
 # Run tests and compile Pilosa
 default: test build
@@ -28,14 +45,48 @@ clean:
 
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
-	go mod vendor
+	$(GO) mod vendor
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v
+
+# Run test suite with race flag
+test-race:
+	CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v
+
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
+tour:
+	./tournament.sh
 
 bench:
-	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+	$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 
 # Run test suite with coverage enabled
 cover:
@@ -44,17 +95,16 @@ cover:
 
 # Run test suite with coverage enabled and view coverage results in browser
 cover-viz: cover
-	go tool cover -html=build/coverage.out
+	$(GO) tool cover -html=build/coverage.out
 
 # Compile Pilosa
 build:
-	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
 
 # Create a single release build under the build directory
 release-build:
-	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
+	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
+	cp NOTICE README.md LICENSE build/pilosa$(VERSION_ID)
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -64,15 +114,16 @@ ifndef SKIP_CHECK_CLEAN
 	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
 endif
 
-# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
-release: check-clean
+# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
+release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
-	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
+# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
+release-sans-ui: check-clean
+	rm -f statik/statik.go
+	$(MAKE) release-build GOOS=darwin GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=amd64
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
 DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
@@ -81,59 +132,143 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
-	docker-compose -f $(DOCKER_COMPOSE) down
+clustertests-build: vendor
+	docker-compose -f $(DOCKER_COMPOSE) down -v
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
-# Create prerelease builds
-prerelease:
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
-	$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
-
-prerelease-upload:
-	aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
-
 # Install Pilosa
 install:
-	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+
+install-bench:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
+
+# Ensure lattice is cloned and the pinned version is checked out
+lattice:
+	git submodule update --init
+
+# Build the lattice assets
+build-lattice: lattice
+	docker build -t lattice:build ./lattice
+	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
+
+# Upgrade lattice to the latest version
+upgrade-lattice: lattice
+	git submodule update --remote
 
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
-	go generate github.com/pilosa/pilosa/v2/internal
+	$(GO) generate github.com/pilosa/pilosa/v2/internal
+
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	$(GO) generate github.com/pilosa/pilosa/v2/statik
+
+# `go generate` statik assets (lattice UI) in Docker
+generate-statik-docker: build-lattice
+	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
 
 # `go generate` stringers
 generate-stringer:
-	go generate github.com/pilosa/pilosa/v2
+	$(GO) generate github.com/pilosa/pilosa/v2
 
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+generate-proto-grpc: require-protoc require-protoc-gen-go
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
+	# TODO: Modify above commands and remove the below mv if possible.
+	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
+	# I couldn't get it to work during development - Cody
+	cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/
+	rm -rf proto/github.com
+
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
+
+# Create release using Docker
+docker-release:
+	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
+
+# Build a release in Docker
+docker-build: vendor lattice
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --target pilosa-builder \
+	    --tag pilosa:build .
+	docker create --name pilosa-build pilosa:build
+	mkdir -p build/pilosa-$(VERSION_ID)
+	docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID)
+	cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID)
+	docker rm pilosa-build
+	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 
 # Create Docker image from Dockerfile
-docker:
-	docker build -t "pilosa:$(VERSION)" .
+docker-image: vendor lattice
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
+	    --tag pilosa:$(VERSION) .
 	@echo Created docker image: pilosa:$(VERSION)
 
-# Compile Pilosa inside Docker container
-docker-build:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Create docker image (alias)
+docker: docker-image # alias
+
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
+pilosa-keydump:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
+
+# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
+pilosa-chk:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
+
+pilosa-fsck:
+	cd ./cmd/pilosa-fsck && make install && make release
 
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
+
+# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
+# run top tests, not subdirs. print summary red/green after.
+# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
+topt:
+	mv log.topt.roar log.topt.roar.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
+	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
+	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
+
+topt-race:
+	mv log.topt.race log.topt.race.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
+	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
+	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +296,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
 
 ######################
 # Build dependencies #
@@ -175,24 +309,37 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go get -u github.com/rakyll/statik
 
 install-stringer:
-	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
+	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
 
 install-protoc-gen-gofast:
-	GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
+	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc-gen-go:
+	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
 
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 
 install-peg:
-	GO111MODULE=off go get github.com/pointlander/peg
+	GO111MODULE=off $(GO) get github.com/pointlander/peg
 
 install-golangci-lint:
-	GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
+	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 
 install-gometalinter:
-	GO111MODULE=off go get -u github.com/alecthomas/gometalinter
+	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
-	GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
+	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
+
+test-txstore-rbf:
+	PILOSA_TXSRC=rbf $(MAKE) testv-race
+
+test-txstore-rbf_bolt:
+	PILOSA_TXSRC=rbf_bolt $(MAKE) testv-race
+

NOTICE

@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
 Third-party software licenses
 =============================
 
@@ -115,3 +94,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
 
 1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
 
+Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101).
+
 2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
 
     ```shell
@@ -40,7 +42,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
     and verify that it's running:
     
     ```shell
-    curl localhost:10101/nodes
+    curl localhost:10101/status
     ```
 
 3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
@@ -56,6 +58,15 @@ Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) w
 You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
 
 
+## Upgrading UI
+
+Lattice is now a submodule of Pilosa, for the purpose of keeping the frontend and backend components of the UI system synchronized. When making a change to the UI that requires changing both the backend and frontend, follow steps in this order:
+
+1. merge frontend (Lattice) PR
+2. run `make upgrade-lattice`
+3. run `git add lattice`, `git commit -m"Upgrade Lattice"`, `git push`
+4. merge backend (Pilosa) PR
+
 ## Client Libraries
 
 There are supported libraries for the following languages:
@@ -63,13 +74,12 @@ There are supported libraries for the following languages:
 - [Java](https://www.pilosa.com/docs/client-libraries/#java)
 - [Python](https://www.pilosa.com/docs/client-libraries/#python)
 
-## Licenses
+## License
 
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
+Pilosa is licensed under the Apache License, Version 2.0.
+
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
 
 ## Get Support
 

api/client/grpc.go

@@ -0,0 +1,214 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +31,140 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "i"
+		fieldName := "f"
+		attrKey := "k"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, indexName, fieldName)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs0,
+			AttrVals:       attrVals0,
+			Shard:          0,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs1,
+			AttrVals:       attrVals1,
+			Shard:          1,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		m := len(res.ColumnAttrSets)
+		if m != 100 {
+			t.Fatalf("incorrect number of column attrs set; m = %v", m)
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -49,62 +184,79 @@ func TestAPI_Import(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "rick"
-		field := "f"
+		indexName := "rick"
+		fieldName := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
 
 		rowID := uint64(1)
 		timestamp := int64(0)
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
-		for i := 1; i <= 10; i++ {
+		N := 10
+		for i := 1; i <= N; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		colKeys = colKeys[:N]
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowIDs:     rowIDs,
-			ColumnKeys: colKeys,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making lots of other shards? b/c this is not a restriction.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
 		}
 
-		pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
+		qcx := m0.API.Txf().NewQcx()
+
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 			t.Fatal(err)
 		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
-			t.Fatalf("unexpected column keys: %#v", keys)
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
 		}
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 				return err
 			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 				return fmt.Errorf("unexpected column keys: %#v", keys)
@@ -129,58 +281,6 @@ func TestAPI_Import(t *testing.T) {
 		}
 
 	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
@@ -200,8 +300,8 @@ func TestAPI_ImportValue(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -219,12 +319,13 @@ func TestAPI_ImportValue(t *testing.T) {
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
@@ -233,9 +334,12 @@ func TestAPI_ImportValue(t *testing.T) {
 			ColumnKeys: colKeys,
 			Values:     values,
 		}
-		if err := m0.API.ImportValue(ctx, req); err != nil {
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
 			t.Fatal(err)
 		}
+		panicOn(qcx.Finish())
 
 		pql := fmt.Sprintf("Row(%s>0)", field)
 
@@ -258,6 +362,125 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+
+		qcx := m1.API.Txf().NewQcx()
+		if err := m1.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		query := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: observerd %+v;  expected '%+v'", ids, colIDs[6:])
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: observerd %+v;  expected '%+v'", ids, []uint64{1})
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.
@@ -266,3 +489,136 @@ type offsetModHasher struct{}
 func (*offsetModHasher) Hash(key uint64, n int) int {
 	return int(key+1) % n
 }
+
+func (*offsetModHasher) Name() string { return "mod" }
+
+func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	// plan:
+	//  1. set a bit
+	//  2. clear with Import() using the ImportRequest.Clear flag
+	//  3. verifiy the clear is done.
+	//  repeat for ImportValueRequest and ImportValues()
+
+	m0 := c.GetNode(0)
+	m0api := m0.API
+
+	ctx := context.Background()
+	index := "i"
+	fieldAcct0 := "acct0"
+
+	opts := pilosa.OptFieldTypeInt(-1000, 1000)
+
+	_, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = m0api.CreateField(ctx, index, fieldAcct0, opts)
+	if err != nil {
+		t.Fatalf("creating fieldAcct0: %v", err)
+	}
+
+	iraField := "ira" // set field.
+	iraRowID := uint64(3)
+	_, err = m0api.CreateField(ctx, index, iraField)
+	if err != nil {
+		t.Fatalf("creating fieldIRA: %v", err)
+	}
+
+	acctOwnerID := uint64(78) // ColumnID
+	shard := acctOwnerID / ShardWidth
+	acct0bal := int64(500)
+
+	ivr0 := &pilosa.ImportValueRequest{
+		Index:     index,
+		Field:     fieldAcct0,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		Values:    []int64{acct0bal},
+	}
+	ir0 := &pilosa.ImportRequest{
+		Index:     index,
+		Field:     iraField,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		RowIDs:    []uint64{iraRowID},
+	}
+
+	qcx := m0api.Txf().NewQcx()
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bitIsSet := func() bool {
+		query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+		cols := res.Results[0].(*pilosa.Row).Columns()
+		for i := range cols {
+			if cols[i] == acctOwnerID {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !bitIsSet() {
+		panic("IRA bit should have been set")
+	}
+
+	queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
+		query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+
+		if len(res.Results) == 0 {
+			return 0
+		}
+		valCount := res.Results[0].(pilosa.ValCount)
+		return valCount.Val
+	}
+
+	bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+
+	if bal != acct0bal {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
+	}
+
+	// clear the bit
+	qcx = m0api.Txf().NewQcx()
+	ir0.Clear = true
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	if bitIsSet() {
+		panic("IRA bit should have been cleared")
+	}
+
+	// clear the BSI
+	qcx = m0api.Txf().NewQcx()
+	ivr0.Clear = true
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+	if bal != 0 {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
+	}
+}

apimethod_string.go

@@ -19,25 +19,30 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
-	_ = x[apiFieldAttrDiff-12]
-	_ = x[apiImport-13]
-	_ = x[apiImportValue-14]
-	_ = x[apiIndex-15]
-	_ = x[apiIndexAttrDiff-16]
-	_ = x[apiQuery-17]
-	_ = x[apiRecalculateCaches-18]
-	_ = x[apiRemoveNode-19]
-	_ = x[apiResizeAbort-20]
-	_ = x[apiSetCoordinator-21]
-	_ = x[apiShardNodes-22]
-	_ = x[apiViews-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiTranslateData-11]
+	_ = x[apiField-12]
+	_ = x[apiFieldAttrDiff-13]
+	_ = x[apiImport-14]
+	_ = x[apiImportValue-15]
+	_ = x[apiIndex-16]
+	_ = x[apiIndexAttrDiff-17]
+	_ = x[apiQuery-18]
+	_ = x[apiRecalculateCaches-19]
+	_ = x[apiRemoveNode-20]
+	_ = x[apiResizeAbort-21]
+	_ = x[apiSetCoordinator-22]
+	_ = x[apiShardNodes-23]
+	_ = x[apiViews-24]
+	_ = x[apiApplySchema-25]
+	_ = x[apiStartTransaction-26]
+	_ = x[apiFinishTransaction-27]
+	_ = x[apiTransactions-28]
+	_ = x[apiGetTransaction-29]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransaction"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 332, 345, 353, 367, 386, 406, 421, 438}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

barrier.go

@@ -0,0 +1,219 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"github.com/glycerine/idem"
+)
+
+// Barrier allows us to temporarily halt all readers, so that
+// a writer can commit alone and thus compact the db.
+// The Barrier starts unblocked, alllowing passage to any
+// caller of WaitAtGate().
+type Barrier struct {
+	wait       chan *appointment // send upon entering the waiting room.
+	halt       *idem.Halter
+	blockReqCh chan *blockReq
+	unblockCh  chan *unblock
+}
+
+type blockReq struct {
+	count int
+	done  chan struct{}
+}
+
+func newBlockReq(count int) *blockReq {
+	return &blockReq{
+		count: count,
+		done:  make(chan struct{}),
+	}
+}
+
+type appointment struct {
+	id   int
+	done chan struct{}
+}
+
+func newAppointment(id int) *appointment {
+	return &appointment{
+		id:   id,
+		done: make(chan struct{}),
+	}
+}
+
+// NewBarrier is either open, allowing immediate passage,
+// or blocked, halting all callers at WaitAtGate()
+// until the barrier is opened. By default it is open.
+//
+// Barrier.Close() must be called when the barrier
+// is no longer needed to avoid a goroutine leak.
+func NewBarrier() (b *Barrier) {
+	b = &Barrier{
+		wait:       make(chan *appointment), // waiters indicate they are waiting for the gate by sending here.
+		halt:       idem.NewHalter(),
+		blockReqCh: make(chan *blockReq),
+		unblockCh:  make(chan *unblock),
+	}
+	go func() {
+		defer b.halt.Done.Close()
+
+		var waitlist []*appointment
+		var curBlockReq *blockReq
+
+		for {
+			select {
+			case br := <-b.blockReqCh:
+				if br.count == 0 {
+					close(br.done)
+					continue
+				}
+				if curBlockReq == nil {
+					// good, changing state from open to closed barrier.
+				} else {
+					panic("got 2nd block request atop of first")
+				}
+				curBlockReq = br
+				//vv("barrier: request to block for %v waiters", br.count)
+				if len(waitlist) != 0 {
+					panic("had waiters when we were open, internal/client bug")
+				}
+			case appt := <-b.wait:
+				//vv("barrier.wait sees appt = '%#v' and  curBlockReq = '%#v'", appt, curBlockReq)
+				if curBlockReq == nil {
+					close(appt.done)
+					continue
+				}
+				waitlist = append(waitlist, appt)
+				n := len(waitlist)
+				th := curBlockReq.count
+				if th < 0 {
+					// infinite waiters. we block everybody until we
+					// see an unblock request.
+					continue
+				}
+				if n >= th {
+					close(curBlockReq.done)
+					curBlockReq = nil
+				}
+			case ub := <-b.unblockCh:
+				for _, appt := range waitlist {
+					close(appt.done)
+				}
+				waitlist = nil
+				curBlockReq = nil
+				close(ub.done)
+			case <-b.halt.ReqStop.Chan:
+				return
+			}
+		}
+	}()
+	return
+}
+
+// WaitAtGate will return immediately
+// if the barrier is unblocked. Otherwise
+// it will not return until another
+// goroutine unblocks the barrier.
+func (b *Barrier) WaitAtGate(id int) {
+	appt := newAppointment(id)
+	select {
+	case b.wait <- appt:
+		select {
+		case <-appt.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// Close should be called to stop the
+// barrier's background goroutine when
+// you are done using the barrier.
+func (b *Barrier) Close() {
+	b.halt.ReqStop.Close()
+	<-b.halt.Done.Chan
+}
+
+type unblock struct {
+	done chan struct{}
+}
+
+func newUnblock() *unblock {
+	return &unblock{
+		done: make(chan struct{}),
+	}
+}
+
+// Unblock lets all waiting goroutines resume execution.
+func (b *Barrier) UnblockReaders() {
+	ub := newUnblock()
+	select {
+	case b.unblockCh <- ub:
+		select {
+		case <-ub.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// BlockUntil is called with a count, the
+// number of waiters required to be present and waiting
+// at the gate before call returns.
+// A count of < 0 will return immediately and raise
+// the barrier to any number of arriving readers.
+// A count of 0 is a no-op.
+//
+// Otherwise we raise the barrier
+// and wait until we have seen count other goroutines waiting
+// on it.
+//
+// We return without releasing the waiters. Call
+// Open when you want them to resume.
+func (b *Barrier) BlockUntil(count int) {
+	if count == 0 {
+		return
+	}
+	req := newBlockReq(count)
+	b.blockReqCh <- req
+	if count > 0 {
+		<-req.done
+	}
+}
+
+// BlockAllReadersNoWait raises the barrier to
+// an infinite number of waiters and returns immediately
+// to the caller.
+func (b *Barrier) BlockAllReadersNoWait() {
+	req := newBlockReq(-1) // -1 means block any number of readers.
+	b.blockReqCh <- req
+	// don't wait. <-req.done
+}

barrier_test.go

@@ -0,0 +1,90 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestBarrierHolds(t *testing.T) {
+	b := NewBarrier()
+	defer b.Close()
+
+	released := int64(0)
+
+	waiter := func(i int) {
+		b.WaitAtGate(i)
+		//vv("goro %v is released", i)
+		atomic.AddInt64(&released, 1)
+	}
+
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+	time.Sleep(time.Second)
+	r := atomic.SwapInt64(&released, 0)
+	if r != 3 {
+		panic("open barrier held back goro")
+	}
+	//vv("good: barrier started open")
+
+	//seenAll := make(chan bool)
+	b.BlockAllReadersNoWait()
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic("bad: barrier did not hold back goro")
+	}
+	//vv("good: barrier of 4 did not release on 3")
+	go waiter(4)
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r))
+	}
+
+	b.UnblockReaders()
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 4 {
+		panic("bad: unblock should have released 4 goro")
+	}
+
+}

bluegreentx_test.go

@@ -0,0 +1,98 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	cryrand "crypto/rand"
+)
+
+var _ = context.Background
+var _ = os.Open
+var _ = strings.Split
+
+func TestMultiReaderB(t *testing.T) {
+	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
+	// member io.Readers, else it should panic. This should hold for
+	// varying sizes of inputs.
+
+	for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
+		src := io.LimitReader(cryrand.Reader, int64(n))
+
+		a := make([]byte, n)
+		nr := 0
+		for nr < n {
+			na, err := src.Read(a)
+			panicOn(err)
+			nr += na
+		}
+		if nr != n {
+			panic("short read")
+		}
+
+		b := make([]byte, n)
+		copy(b, a)
+		if !bytes.Equal(a, b) {
+			panic("test prep failed")
+		}
+
+		m := &MultiReaderB{
+			a: ioutil.NopCloser(bytes.NewBuffer(a)),
+			b: ioutil.NopCloser(bytes.NewBuffer(b)),
+		}
+
+		// should not trigger the internal panic of MultiReadB
+		ncp, err := io.Copy(ioutil.Discard, m)
+		panicOn(err)
+		if ncp != int64(n) {
+			panic("short copy")
+		}
+
+		for victim := 0; victim < n; victim += 7 {
+
+			copy(b, a)
+			if victim%2 == 0 {
+				// corrupt b
+				b[victim] = (b[victim] + 1) % 255
+			} else {
+				// corrupt a
+				a[victim] = (a[victim] + 1) % 255
+			}
+			m = &MultiReaderB{
+				a: ioutil.NopCloser(bytes.NewBuffer(a)),
+				b: ioutil.NopCloser(bytes.NewBuffer(b)),
+			}
+			helperShouldPanicOnCopy(m)
+		}
+	}
+}
+
+func helperShouldPanicOnCopy(m *MultiReaderB) {
+	// differences in bytes read should be noticed
+	defer func() {
+		r := recover()
+		if r == nil {
+			panic("expected panic on byte difference but didn't see it")
+		}
+	}()
+	_, _ = io.Copy(ioutil.Discard, m)
+}

boltdb/attrstore.go

@@ -25,9 +25,9 @@ import (
 
 	"github.com/cespare/xxhash"
 
-	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 // attrBlockSize is the size of attribute blocks for anti-entropy.

boltdb/translate_test.go

@@ -14,9 +14,13 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
@@ -24,28 +28,29 @@ import (
 	"github.com/pilosa/pilosa/v2/boltdb"
 )
 
+//var vv = pilosa.VV
+
 func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	// Ensure initial key translates to first ID for shard
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
-	if id, err := s.TranslateKey("bar"); err != nil {
+	if id, err := s.TranslateKey("bar", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	if id, err := s.TranslateKey("foo", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
@@ -54,59 +59,182 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial keys translate to incrementing IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"abc", "abc"}, true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids[0]; got != want {
+		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
+	}
+
+	// Ensure initial keys translate to incrementing IDs.
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
-	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
 
+func TestTranslateStore_CreateKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.CreateKeys("abc", "abc")
+	if err != nil {
+		t.Fatal(err)
+	} else if _, ok := ids["abc"]; !ok {
+		t.Fatalf(`missing "abc"; got %v`, ids)
+	} else if len(ids) > 1 {
+		t.Fatalf("expected one key, got %d in %v", len(ids), ids)
+	}
+
+	// Ensure different keys translate to different IDs.
+	ids1, err := s.CreateKeys("foo", "bar")
+	if err != nil {
+		t.Fatal(err)
+	} else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar {
+		t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo)
+	}
+
+	// Ensure retranslation returns original IDs.
+	if ids, err := s.CreateKeys("bar", "foo"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, ids1) {
+		t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1)
+	}
+
+	// Ensure retranslating with existing and non-existing keys returns correctly.
+	if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil {
+		t.Fatal(err)
+	} else if got, want := ids["foo"], ids1["foo"]; got != want {
+		t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want)
+	} else if _, ok := ids["baz"]; !ok {
+		t.Fatalf(`missing translation for "baz"; got %v`, ids)
+	} else if got, want := ids["bar"], ids1["bar"]; got != want {
+		t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want)
+	}
+}
+
+func TestTranslateStore_ReadKey(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	id, err := s.TranslateKey("foo", false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+
+	s.SetReadOnly(true)
+	id, err = s.TranslateKey("foo", true)
+	if err == nil {
+		t.Fatalf("got error: %+v, want: 'translate store read only'", err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+	s.SetReadOnly(false)
+
+	// Ensure next key autoincrements.
+	if id, err = s.TranslateKey("foo", true); err != nil {
+		t.Fatal(err)
+	}
+	id1, err := s.TranslateKey("foo", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id1 != id {
+		t.Fatalf("TranslateKey()=%d, want %d", id1, id)
+	}
+}
+
+func TestTranslateStore_ReadKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	for _, id := range ids {
+		if id != 0 {
+			t.Fatalf("TranslateKeys()=%d, want %d", id, 0)
+		}
+	}
+
+	// Ensure next key autoincrements.
+	if ids, err = s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, true); err != nil {
+		t.Fatal(err)
+	}
+	ids1, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range ids1 {
+		if ids1[i] != ids[i] {
+			t.Fatalf("TranslateKeys()=%d, want %d", ids1[i], ids[i])
+		}
+	}
+}
 func TestTranslateStore_TranslateID(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("", true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
@@ -114,12 +242,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -130,13 +259,116 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	}
 }
 
+func TestTranslateStore_FindKeys(t *testing.T) {
+	cases := []struct {
+		name   string
+		data   []string
+		lookup []string
+	}{
+		{
+			name:   "All",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h"},
+		},
+		{
+			name:   "Extra",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h", "65"},
+		},
+		{
+			name:   "None",
+			data:   []string{"a", "b", "c"},
+			lookup: []string{"d", "e"},
+		},
+		{
+			name:   "Empty",
+			lookup: []string{"h"},
+		},
+		{
+			name: "LookupNothing",
+		},
+	}
+
+	for _, c := range cases {
+		c := c
+		t.Run(c.name, func(t *testing.T) {
+			s := MustOpenNewTranslateStore()
+			defer MustCloseTranslateStore(s)
+
+			var naiveMap map[string]uint64
+			if c.data != nil {
+				// Load in key data.
+				keys := c.data
+				ids, err := s.TranslateKeys(keys, true)
+				if err != nil {
+					t.Errorf("failed to import keys: %v", err)
+					return
+				}
+				if len(ids) != len(keys) {
+					t.Errorf("mapped %d keys to %d ids", len(keys), len(ids))
+					return
+				}
+				naiveMap = make(map[string]uint64, len(keys))
+				for i, key := range keys {
+					naiveMap[key] = ids[i]
+				}
+			}
+
+			// Compute expected lookup result.
+			result := map[string]uint64{}
+			for _, key := range c.lookup {
+				id, ok := naiveMap[key]
+				if !ok {
+					// The key is expected to be missing.
+					continue
+				}
+
+				result[key] = id
+			}
+
+			// Find the keys.
+			found, err := s.FindKeys(c.lookup...)
+			if err != nil {
+				t.Errorf("failed to find keys: %v", err)
+			} else if !reflect.DeepEqual(result, found) {
+				t.Errorf("expected %v but found %v", result, found)
+			}
+		})
+	}
+}
+
+func TestTranslateStore_MaxID(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	// Generate a bunch of keys.
+	var lastk uint64
+	for i := 0; i < 1026; i++ {
+		k, err := s.TranslateKey(strconv.Itoa(i), true)
+		if err != nil {
+			t.Fatalf("translating %d: %v", i, err)
+		}
+		lastk = k
+	}
+
+	// Verify the max ID.
+	max, err := s.MaxID()
+	if err != nil {
+		t.Fatalf("checking max ID: %v", err)
+	}
+	if max != lastk {
+		t.Fatalf("last key is %d but max is %d", lastk, max)
+	}
+}
+
 func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		s := MustOpenNewTranslateStore()
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +383,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +392,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz", true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -198,20 +431,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo", true)
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -302,11 +540,76 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, true); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
 func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 	s := MustNewTranslateStore()
@@ -324,3 +627,173 @@ func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 		panic(err)
 	}
 }
+
+func TestCryptoHashPerKey(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	// hash one translation
+
+	expect := map[int]string{
+		1: string([]byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12}),
+		2: string([]byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0}),
+		3: string([]byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9}),
+	}
+	for n := 1; n < 4; n++ {
+		var batch0 []string
+		for i := 0; i < n; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		_ = batch0IDs
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// done with setup
+		sum, err := s.ComputeTranslatorSummaryCols(0, pilosa.NewTopology(&pilosa.Jmphasher{}, pilosa.DefaultPartitionN, 1, nil))
+		if err != nil {
+			panic(err)
+		}
+		nkey := sum.KeyCount
+		nid := sum.IDCount
+		observedChecksum := sum.Checksum
+		if nkey != n {
+			panic("wrong key count")
+		}
+		if nkey != nid {
+			panic("key count should match id count")
+		}
+
+		// shardwidth 22 has different hashes, of course.
+		if pilosa.ShardWidth == 20 {
+			expectedChecksum := expect[n]
+			if observedChecksum != expectedChecksum {
+				panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum))
+			}
+		}
+	}
+
+}
+
+func TestTranslateStore_RepairNonInvertibleStringKeyTranslation(t *testing.T) {
+
+	const N = 6
+	// before repair
+	var fwd [N]map[string]uint64
+	var rev [N]map[uint64]string
+
+	// after repair
+	var fwd2 [N]map[string]uint64
+	var rev2 [N]map[uint64]string
+
+	// case 0: forward is messed up (unlikely but check for it anyway, be sure we can repair)
+	// "key0" -> id 0  // correct.
+	// "key1" -> id 0  // wrong. after Repair, should see key1 -> 1 (0xec0002)
+	//
+	// id 0 -> "key0" // correct
+	// id 1 -> "key1" // correct
+	//
+	fwd[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00001}
+	rev[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+	fwd2[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 1: reverse is messed up (we have seen this in the past)
+	// "key0" -> id 0  // correct
+	// "key1" -> id 1  // correct
+	//
+	// id 0 -> "key0" // correct.
+	// id 1 -> "key0" // wrong. after Repair, should see id 1 -> "key1"
+	//
+	fwd[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key0"}
+	fwd2[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 2: only present in reverse.
+	fwd[2] = map[string]uint64{}
+	rev[2] = map[uint64]string{0xec00001: "key0"}
+	fwd2[2] = map[string]uint64{"key0": 0xec00001}
+	rev2[2] = map[uint64]string{0xec00001: "key0"}
+
+	// case 3: same thing. with camoflage.
+	fwd[3] = map[string]uint64{"key1": 0xec00002}
+	rev[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+	fwd2[3] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 4: only present in forward.
+
+	fwd[4] = map[string]uint64{"key0": 0xec00001}
+	rev[4] = map[uint64]string{}
+	fwd2[4] = map[string]uint64{"key0": 0xec00001}
+	rev2[4] = map[uint64]string{0xec00001: "key0"}
+
+	// case 5: same thing. with camoflage.
+	fwd[5] = map[string]uint64{"key0": 0xec00001}
+	rev[5] = map[uint64]string{0xec00002: "key1"}
+	fwd2[5] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
+	rev2[5] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
+
+	// case 6: we had an id, but b/c of the fix, that id is no longer used.
+	//         now that id might still be used in the fragment for a column,
+	//         and so we will need to remove that id/column from the fragment.
+	// encapsulated: "did it affect the state of the fields?"
+
+	for i := 0; i < 5; i++ {
+		//println("i = ", i)
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		if err := s.SetFwdRevMaps(nil, fwd[i], rev[i]); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := verifyState("setup", i, s, fwd[i], rev[i]); err != nil {
+			t.Fatal(err)
+		}
+
+		var topo *pilosa.Topology
+		verbose := false
+		applyKeyRepairs := true
+		changed, err := s.RepairKeys(topo, verbose, applyKeyRepairs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !changed {
+			t.Fatalf("expected changes!")
+		}
+
+		if err := verifyState("afterRepair", i, s, fwd2[i], rev2[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func verifyState(label string, i int, s *boltdb.TranslateStore, fwd map[string]uint64, rev map[uint64]string) error {
+
+	// verify the setup
+	const writable = true
+	for key, expectID := range fwd {
+		id, err := s.TranslateKey(key, !writable)
+		if err != nil {
+			return err
+		}
+		if id != expectID {
+			return fmt.Errorf("fwd %v problem. i=%v, for key '%v', expected %x, observed %x", label, i, key, expectID, id)
+		}
+	}
+	for id, expectKey := range rev {
+		key, err := s.TranslateID(id)
+		if err != nil {
+			return err
+		}
+		if key != expectKey {
+			return fmt.Errorf("rev %v problem. i=%v, for id '%x', expected %v, observed %v", label, i, id, expectKey, key)
+		}
+	}
+	return nil
+}

broadcast.go

@@ -69,6 +69,7 @@ const (
 	messageTypeRecalculateCaches
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -116,6 +117,8 @@ func getMessage(typ byte) Message {
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -155,6 +158,8 @@ func getMessageType(m Message) byte {
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

bsi.go

@@ -0,0 +1,296 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"math/bits"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// bsiData contains BSI-structured data.
+type bsiData []*Row
+
+// pivotDescending loops over nonzero BSI values in descending order.
+// For each value, the provided function is called with the value and a slice of the associated columns.
+// If limit or offset are not-nil, they will be applied.
+// Applying a limit or offset may modify the pointed-to value.
+func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+	// This "pivot" algorithm works by treating the BSI data as a tree.
+	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
+	// Each range is subdivided into 2 ranges of half size, which form lower branches.
+	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
+	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
+	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
+	// This function uses a depth-first search over this virtual tree.
+
+	switch {
+	case !filter.Any():
+		// There are no remaining data.
+
+	case offset != nil && *offset >= filter.Count():
+		// Skip this entire branch.
+		*offset -= filter.Count()
+
+	case limit != nil && *limit == 0:
+		// The limit has been reached.
+		// No more data is necessary.
+
+	case len(bsi) == 0:
+		// This is a leaf node.
+		cols := filter.Columns()
+		if offset != nil {
+			cols = cols[*offset:]
+			*offset = 0
+		}
+		if limit != nil {
+			if *limit < uint64(len(cols)) {
+				cols = cols[:*limit]
+			}
+			*limit -= uint64(len(cols))
+		}
+		fn(branch, cols...)
+
+	default:
+		// Pivot over the highest bit.
+		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
+		splitBit := bsi[len(bsi)-1]
+		lowerBits := bsi[:len(bsi)-1]
+		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+	}
+}
+
+/*
+// distribution generates a BSI histogram for the input.
+// TODO: I forgot what I was going to use this for.
+// Could probbably use this for:
+// - quartile queries
+// - TopN on int
+func (bsi bsiData) distribution(filter *Row) bsiData {
+	var dist bsiData
+	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+		dist.insert(count, uint64(len(values)))
+	})
+	return dist
+}
+*/
+
+var placeholderBitmap = roaring.NewBitmap()
+
+// addBSI adds two BSI bitmaps together.
+// It does not handle sign and has no concept of overflow.
+func addBSI(x, y bsiData) bsiData {
+	// Accumulate row segments.
+	segments := make([][]rowSegment, len(x)+len(y))
+	xsegs, ysegs := segments[:len(x)], segments[len(x):]
+	for i, r := range x {
+		xsegs[i] = r.segments
+	}
+	for i, r := range y {
+		ysegs[i] = r.segments
+	}
+
+	var dst bsiData
+	var xbitmaps, ybitmaps []*roaring.Bitmap
+	for {
+		// Find the next shard.
+		next := ^uint64(0)
+		for _, s := range segments {
+			if len(s) == 0 {
+				continue
+			}
+			shard := s[0].shard
+			if shard < next {
+				next = shard
+			}
+		}
+		if next == ^uint64(0) {
+			// There are no remaining shards.
+			break
+		}
+
+		// Accumulate bitmaps for this shard.
+		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
+		for i, segs := range xsegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			xsegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(xbitmaps) < i {
+				xbitmaps = append(xbitmaps, placeholderBitmap)
+			}
+			xbitmaps = append(xbitmaps, bm)
+		}
+		for i, segs := range ysegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			ysegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(ybitmaps) < i {
+				ybitmaps = append(ybitmaps, placeholderBitmap)
+			}
+			ybitmaps = append(ybitmaps, bm)
+		}
+
+		// Add the shard values together.
+		var out []*roaring.Bitmap
+		switch {
+		case len(xbitmaps) == 0:
+			// There are no values in x.
+			out = ybitmaps
+		case len(ybitmaps) == 0:
+			// There are no values in y.
+			out = xbitmaps
+		default:
+			out = roaring.Add(xbitmaps, ybitmaps)
+		}
+
+		// Convert the bitmaps to output segments.
+		for i, b := range out {
+			if !b.Any() {
+				continue
+			}
+			for len(dst) <= i {
+				dst = append(dst, NewRow())
+			}
+			dst[i].segments = append(dst[i].segments, rowSegment{
+				shard:    next,
+				writable: true,
+				data:     b,
+				n:        b.Count(),
+			})
+		}
+	}
+
+	return dst
+}
+
+// rowBuilder builds a row quickly from individual values.
+// It is optimized for the case in which values are generated sequentially.
+type rowBuilder struct {
+	bm    *roaring.Bitmap
+	mask  *[1024]uint64
+	array []uint16
+	key   uint64
+	n     int32
+}
+
+// flushKey flushes the data at the current key to the bitmap.
+func (b *rowBuilder) flushKey() {
+	var c *roaring.Container
+	switch {
+	case b.mask != nil:
+		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
+		b.mask = nil
+	case len(b.array) > 0:
+		c = roaring.NewContainerArrayCopy(b.array)
+		b.array = b.array[:0]
+	default:
+		return
+	}
+
+	if b.bm == nil {
+		b.bm = roaring.NewBitmap()
+	}
+	if old := b.bm.Containers.Get(b.key); old != nil {
+		c = roaring.Union(c, old)
+	}
+	b.bm.Containers.Put(b.key, c)
+}
+
+// Add a value to the bitmap.
+// Values must be added sequentially.
+func (b *rowBuilder) Add(v uint64) {
+	vkey := v / (1 << 16)
+	if b.key != vkey {
+		// This is a new key, so flush the old one.
+		b.flushKey()
+		b.key = vkey
+	}
+
+	if b.mask != nil {
+		// Add to the mask.
+		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
+		b.mask[uint16(v)/64] |= 1 << (v % 64)
+		return
+	}
+
+	// Add to an array.
+	b.array = append(b.array, uint16(v))
+	if len(b.array) >= roaring.ArrayMaxSize {
+		// The array is too big.
+		// Convert it to a bitmask.
+		m := [1024]uint64{}
+		for _, v := range b.array {
+			m[v/64] |= 1 << (v % 64)
+		}
+		b.n = int32(len(b.array))
+		b.array = b.array[:0]
+		b.mask = &m
+	}
+}
+
+// Build a Row from stored data.
+// This resets the builder.
+func (b *rowBuilder) Build() *Row {
+	// Flush the active key to the bitmap.
+	b.flushKey()
+
+	// Remove the bitmap and convert it to a Row.
+	bm := b.bm
+	b.bm = nil
+	if bm == nil {
+		return NewRow()
+	}
+	return NewRowFromBitmap(bm)
+}
+
+// bsiBuilder assembles BSI data.
+// It is optimized for the case in which values are generated sequentially.
+type bsiBuilder []rowBuilder
+
+// Insert a value into the BSI data.
+// Columns must be inserted sequentially, and duplicates are not allowed.
+func (b *bsiBuilder) Insert(col, val uint64) {
+	for val != 0 {
+		i := bits.TrailingZeros64(val)
+		val &^= 1 << i
+		for len(*b) <= i {
+			*b = append(*b, rowBuilder{})
+		}
+		(*b)[i].Add(col)
+	}
+}
+
+// Build BSI data.
+// This resets the builder.
+func (b *bsiBuilder) Build() bsiData {
+	builders := *b
+	*b = builders[:0]
+	rows := make(bsiData, len(builders))
+	for i := range builders {
+		rows[i] = builders[i].Build()
+	}
+	return rows
+}

bsi_test.go

@@ -0,0 +1,175 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+)
+
+// TestBSIAdd does a number of iterations. For each iteration, it
+// generates a random number of ids, and two random values for each id
+// to add together.
+func TestBSIAdd(t *testing.T) {
+	// TODO wouldn't it be cool if our test suite had a randomized
+	// burn-in mode where you could run any test which supported it
+	// with a random seed and way more iterations?
+	rnd := rand.New(rand.NewSource(99))
+	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
+	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
+	var builderA, builderB bsiBuilder
+	// a and b are generated slices of numbers to add together
+	var a, b []uint64
+	// idToIndex maps record ids to indexes in a and b
+	idToIndex := make(map[int]int)
+	// indexToID has the record id for each value in a and b
+	indexToID := []uint64{}
+
+	min := 999999999
+	max := 0
+
+	for iteration := 0; iteration < 1; iteration++ {
+		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
+			// reset generated data
+			a, b = a[:0], b[:0]
+			indexToID = indexToID[:0]
+			for k := range idToIndex {
+				delete(idToIndex, k)
+			}
+
+			// z generates the values, they can be fairly large, but are usually small
+			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
+			id := -1
+			for i := 0; true; i++ {
+				// get the next id, skipping a random amount
+				id = id + int(idZipf.Uint64()+1)
+				if id >= ShardWidth {
+					if i < min {
+						min = i
+					}
+					if max < i {
+						max = i
+					}
+					t.Log("num: ", i)
+					break
+				}
+				idToIndex[id] = int(i)
+				indexToID = append(indexToID, uint64(id))
+
+				// append a random value to each data slice
+				a = append(a, z.Uint64())
+				b = append(b, z.Uint64())
+			}
+
+			// build the BSIs based on the data slices and generated IDs
+			for index, id := range indexToID {
+				va, vb := a[index], b[index]
+				builderA.Insert(uint64(id), va)
+				builderB.Insert(uint64(id), vb)
+			}
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+
+			// build results from added bsiData; results[i] should hold a[i]+b[i]
+			results := make([]uint64, len(a))
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[idToIndex[int(id)]] = count
+				}
+			})
+
+			for i, res := range results {
+				if res != a[i]+b[i] {
+					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
+				}
+			}
+		})
+	}
+	t.Log("min", min)
+	t.Log("max", max)
+}
+
+type bsiAddCase struct {
+	positions []uint64
+	a         []uint64
+	b         []uint64
+}
+
+func (b bsiAddCase) Len() int {
+	return len(b.positions)
+}
+
+// Less reports whether the element with
+// index i should sort before the element with index j.
+func (b bsiAddCase) Less(i, j int) bool {
+	return b.positions[i] < b.positions[j]
+}
+
+// Swap swaps the elements with indexes i and j.
+func (b bsiAddCase) Swap(i, j int) {
+	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
+	b.a[i], b.a[j] = b.a[j], b.a[i]
+	b.b[i], b.b[j] = b.b[j], b.b[i]
+}
+
+// TestBSIAddCases tests specific cases of bsiAdd (would generally be
+// pulled from randomly generated ones from TestBSIAdd upon failure).
+func TestBSIAddCases(t *testing.T) {
+	tests := []bsiAddCase{
+		{
+			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
+			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
+			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
+		},
+		{
+			positions: []uint64{17570, 54607},
+			a:         []uint64{1010, 71},
+			b:         []uint64{25, 150},
+		},
+	}
+
+	var builderA, builderB bsiBuilder
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
+				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
+			}
+			sort.Sort(tst)
+
+			for i := 0; i < len(tst.a); i++ {
+				builderA.Insert(tst.positions[i], tst.a[i])
+				builderB.Insert(tst.positions[i], tst.b[i])
+			}
+
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+			// maps id to count
+			results := make(map[uint64]uint64)
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[id] = count
+				}
+			})
+
+			for i, id := range tst.positions {
+				if results[id] != tst.a[i]+tst.b[i] {
+					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
+				}
+			}
+		})
+	}
+}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +121,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -134,9 +138,11 @@ var _ cache = &lruCache{}
 
 // rankCache represents a cache with sorted entries.
 type rankCache struct {
-	mu       sync.Mutex
-	entries  map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	mu           sync.Mutex
+	entries      map[uint64]uint64
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -168,10 +174,16 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	// Ignore if the column count is below the threshold,
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -184,7 +196,13 @@ func (c *rankCache) Add(id uint64, n uint64) {
 func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -209,12 +227,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
-	for id := range c.entries {
-		a = append(a, id)
+	if len(c.entries) == 0 {
+		return nil
 	}
-	sort.Sort(uint64Slice(a))
-	return a
+	ids := make([]uint64, 0, len(c.entries))
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +249,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -236,27 +257,40 @@ func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,11 +306,14 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
+
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // SetStats defines the stats client used in the cache.
@@ -285,7 +322,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +358,68 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
+func (p *bitmapPairs) Len() int           { return len(*p) }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
+	}
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +495,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
+	}
+	copy(r.Pairs, p.Pairs)
+	return
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -431,13 +607,7 @@ func (p uint64Slice) merge(other []uint64) []uint64 {
 	return ret
 }
 
-// bitmapCache provides an interface for caching full bitmaps.
-type bitmapCache interface {
-	Fetch(id uint64) (*Row, bool)
-	Add(id uint64, b *Row)
-}
-
-// simpleCache implements BitmapCache
+// simpleCache implements a bitmap Rowcache.
 // it is meant to be a short-lived cache for cases where writes are continuing to access
 // the same row within a short time frame (i.e. good for write-heavy loads)
 // A read-heavy use case would cause the cache to get bigger, potentially causing the
@@ -452,6 +622,12 @@ func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
 	return m, ok
 }
 
+func newSimpleCache() *simpleCache {
+	return &simpleCache{
+		cache: make(map[uint64]*Row),
+	}
+}
+
 // Add adds the bitmap to the cache, keyed on the id. A nil row means
 // deleting the row from the cache.
 func (s *simpleCache) Add(id uint64, b *Row) {

cache_test.go

@@ -15,13 +15,14 @@
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +32,54 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
 }

catcher.go

@@ -0,0 +1,323 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	//txkey "github.com/pilosa/pilosa/v2/txkey"
+)
+
+// catcher is useful to report error locations with a
+// stack dump before the complexity
+// of the executor_test swallows up
+// the location of a panic.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
+	c.b.IncrementOpN(index, field, view, shard, changedN)
+}
+
+func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	return c.b.NewTxIterator(index, field, view, shard)
+}
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
+}
+
+func (c *catcherTx) Dump(short bool, shard uint64) {
+	c.b.Dump(short, shard)
+}
+
+func (c *catcherTx) Readonly() bool {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Readonly()
+}
+
+func (tx *catcherTx) Pointer() string {
+	return fmt.Sprintf("%p", tx)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) UseRowCache() bool {
+	return c.b.UseRowCache()
+}
+
+func (c *catcherTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, batched, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEach(index, field, view, shard, fn)
+}
+
+func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.UnionInPlace(index, field, view, shard, others...)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+
+func (c *catcherTx) Group() *TxGroup {
+	return c.b.Group()
+}
+
+func (c *catcherTx) Options() Txo {
+	return c.b.Options()
+}
+
+// Sn retreives the serial number of the Tx.
+func (c *catcherTx) Sn() int64 {
+	return c.b.Sn()
+}
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
+	return c.b.GetSortedFieldViewList(idx, shard)
+}
+
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

client.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -43,7 +44,11 @@ type FieldValue struct {
 // something hasn't been architected correctly.
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
+// Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client.
+// Doing that may obviate the need to refactor this.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
@@ -51,7 +56,6 @@ type InternalClient interface {
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
 	Nodes(ctx context.Context) ([]*Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,6 +63,7 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@@ -68,7 +73,17 @@ type InternalClient interface {
 	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
 	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error)
 	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
+
+	GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error)
+	GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error)
 }
 
 //===============
@@ -76,23 +91,57 @@ type InternalClient interface {
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
 	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+
+	// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
+	TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
+
+	FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error)
+	FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error)
+
+	CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error)
+	CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error)
 }
 
 type nopInternalQueryClient struct{}
 
-func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
+func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
 
-func newNopInternalQueryClient() *nopInternalQueryClient {
-	return &nopInternalQueryClient{}
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func newNopInternalQueryClient() nopInternalQueryClient {
+	return nopInternalQueryClient{}
 }
 
 var _ InternalQueryClient = newNopInternalQueryClient()
 
 //===============
 
-type nopInternalClient struct{}
+type nopInternalClient struct{ nopInternalQueryClient }
 
 func newNopInternalClient() nopInternalClient {
 	return nopInternalClient{}
@@ -120,18 +169,24 @@ func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
 func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
-func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
+	return nil
+}
+
 func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -172,3 +227,27 @@ func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte
 func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
 	return nil, nil
 }
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) {
+	return nil, nil
+}

cluster_internal_test.go

@@ -17,7 +17,6 @@ package pilosa
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -35,6 +34,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -91,12 +91,17 @@ func TestFragCombos(t *testing.T) {
 }
 
 // newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	h := NewHolder(path, nil)
+	panicOn(h.Open())
+	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -105,7 +110,6 @@ func newIndexWithTempPath(name string) *Index {
 
 // Ensure that fragSources creates the correct fragment mapping.
 func TestFragSources(t *testing.T) {
-
 	uri0, err := NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
@@ -157,27 +161,53 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 101, nil)
+
+	// Obtain transaction.
+	var shard uint64
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 1
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*2+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 2
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*3+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 3
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	panicOn(tx.Commit())
 
 	tests := []struct {
 		from     *cluster
@@ -351,7 +381,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := c.shardToShardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -381,7 +411,7 @@ func TestHasher(t *testing.T) {
 		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
 	} {
 		for i, v := range tt.bucket {
-			hasher := &jmphasher{}
+			hasher := &Jmphasher{}
 			if got := hasher.Hash(tt.key, i+1); got != v {
 				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
 			}
@@ -391,7 +421,7 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
 	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
 
@@ -537,7 +567,7 @@ func TestCluster_Coordinator(t *testing.T) {
 }
 
 func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
+	c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"}
 
 	uri0 := NewTestURIFromHostPort("host0", 0)
 	uri1 := NewTestURIFromHostPort("host1", 0)
@@ -585,7 +615,7 @@ func TestCluster_Topology(t *testing.T) {
 func TestCluster_ResizeStates(t *testing.T) {
 
 	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
+		tc := NewClusterCluster(t, 1)
 
 		// Open TestCluster.
 		if err := tc.Open(); err != nil {
@@ -615,7 +645,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -647,7 +677,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -676,7 +706,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -718,7 +748,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -767,7 +797,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -782,6 +812,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		// Each tc.SetBit starts and commits its own Tx.
 		if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
 			t.Fatalf("setting bit: %v", err)
 		}
@@ -794,12 +825,21 @@ func TestCluster_ResizeStates(t *testing.T) {
 		node0Field := node0.holder.Field("i", "f")
 		node0View := node0Field.view("standard")
 		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
+		node0Checksum, err := node0Fragment.Checksum()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		idx0 := node0.holder.Index("i")
+		if idx0 == nil {
+			t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
+		}
 
 		// addNode needs to block until the resize process has completed.
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
+
 		node1 := tc.Clusters[1]
 
 		// Ensure that nodes come up in state NORMAL.
@@ -808,6 +848,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 		} else if node1.State() != ClusterStateNormal {
 			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
 		}
+		// INVAR: after node1.State() is normal, the rebalancing should have been done.
 
 		expectedTop := &Topology{
 			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
@@ -826,8 +867,15 @@ func TestCluster_ResizeStates(t *testing.T) {
 		node1View := node1Field.view("standard")
 		node1Fragment := node1View.Fragment(1)
 
+		idx1 := node1.holder.Index("i")
+		if idx1 == nil {
+			t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
+		}
+
 		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
+		if chksum, err := node1Fragment.Checksum(); err != nil {
+			t.Fatal(err)
+		} else if !bytes.Equal(chksum, node0Checksum) {
 			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
 		}
 
@@ -846,6 +894,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -857,11 +906,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@@ -899,7 +950,7 @@ func TestAE(t *testing.T) {
 // Ensures that coordinator can be changed.
 func TestCluster_UpdateCoordinator(t *testing.T) {
 	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
+		c := NewTestCluster(t, 2)
 
 		oldNode := c.nodes[0]
 		newNode := c.nodes[1]
@@ -942,15 +993,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c := newCluster()
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be up")
 	}
 
 }
 func TestCluster_confirmNodeDownTimeout(t *testing.T) {
+	sleep := 50 * time.Millisecond
+	retries := 5
+	if testing.Short() {
+		t.Skip()
+	}
 	r := mux.NewRouter()
 	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
+		time.Sleep(sleep * time.Duration(retries))
 		fmt.Fprintln(w, "ignored")
 	}))
 	server := httptest.NewServer(r)
@@ -969,21 +1027,53 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c := newCluster()
+	c.confirmDownSleep = sleep
+	c.confirmDownRetries = retries
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
 }
 
 func TestCluster_confirmNodeDownDown(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	uri := URI{}
 	uri.Scheme = "http"
 	uri.Host = "DoesntMatter"
 	uri.Port = 6666
+	c := newCluster()
+	c.confirmDownSleep = 50 * time.Millisecond
+	c.confirmDownRetries = 5
+	c.logger = logger.NewVerboseLogger(os.Stdout)
 
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
+}
+
+func TestCluster_GetNonPrimaryReplicas(t *testing.T) {
+
+	c := newCluster()
+	c.ReplicaN = 3
+	topo := NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
+	c.Topology = topo
+	nNodes := 4
+	for i := 0; i < nNodes; i++ {
+		nodeID := fmt.Sprintf("node%d", i)
+		c.nodes = append(c.nodes, &Node{
+			ID:  nodeID,
+			URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
+		})
+		c.Topology.addID(nodeID)
+	}
+
+	partitionID := 256
+	nonPrimes := topo.GetNonPrimaryReplicas(partitionID)
+	m := len(nonPrimes)
+	if m != c.ReplicaN-1 {
+		t.Fatalf("expected 2 non primes, got %v", m)
+	}
 }

cmd/badloader/badloader.go

@@ -0,0 +1,154 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+	//"fmt"
+	"fmt"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	//"log"
+	"os"
+	//"path/filepath"
+	//"sort"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *http.InternalClient) error {
+	t0 := time.Now()
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	//vv("top of tar loop")
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				panic("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				panicOn(err)
+				//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+			}
+			return nil
+		}
+		//vv("got header '%v'", header.Name)
+		n++
+		if n%500 == 0 {
+			vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		//vv("parts = '%#v'", parts)
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+// badloader reproduce a union in place issue for us. slurp is
+// the new "good" loader, and should always be preferred now
+// when not trying to repro that bug. pulled from 85fa67e8
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	panicOn(UploadTar(tarSrcPath, c))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pilosa.URI
+
+func init() {
+	var err error
+	globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
+	panicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
+	return globURI
+}

cmd/badloader/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/convert.go

@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
 			return inspector.Run(context.Background())
 		},
 	}
+	flags := inspectCmd.Flags()
+	flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+	flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+	flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+	flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+	flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+	flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
 	return inspectCmd
 }

cmd/export.go

@@ -50,7 +50,7 @@ The file does not contain any headers.
 	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
 	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
-	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
 
 	return exportCmd
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
@@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
-	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
 
 	return importCmd
 }

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
-					Min:       -10,
+					Max:       pql.NewDecimal(100, 0),
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/pilosa-bench/main.go

@@ -0,0 +1,358 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"expvar"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	phttp "github.com/pilosa/pilosa/v2/http"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	requestCountVar          = expvar.NewInt("request_count")
+	requestCurrentLatencyVar = expvar.NewFloat("request_current_latency") // seconds
+	requestAvgLatencyVar     = expvar.NewFloat("request_avg_latency")     // seconds
+	requestTotalLatencyVar   = expvar.NewFloat("request_total_latency")   // seconds
+	requestPerSecVar         = expvar.NewFloat("request_per_sec")
+)
+
+func main() {
+	if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) (err error) {
+	fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
+	hostport := fs.String("hostport", "localhost:10101", "")
+	typ := fs.String("type", "row", "query type (row)")
+	n := fs.Int("n", 1000, "number of queries")
+	rate := fs.Int("rate", 1, "number of queries per second")
+	verbose := fs.Bool("v", false, "verbose logging")
+	from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
+	to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse from/to time.
+	var opt queryOptions
+	if *from != "" {
+		if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
+			return fmt.Errorf("cannot parse -from time")
+		}
+	}
+	if *to != "" {
+		if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
+			return fmt.Errorf("cannot parse -to time")
+		}
+	}
+
+	if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
+		return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
+	}
+
+	// Clear time prefix on log.
+	log.SetFlags(0)
+	if !*verbose {
+		log.SetOutput(ioutil.Discard)
+	}
+
+	// Setup PRNG to have consistent values for the same set of data.
+	rand.Seed(0)
+
+	// Setup connection to pilosa.
+	client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
+	if err != nil {
+		return err
+	}
+
+	// Set up HTTP endpoint to provide /debug endpoints.
+	fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
+	go func() { _ = http.ListenAndServe(":7070", nil) }()
+
+	// Run separate goroutine to calculate the current req/sec & latency.
+	go monitor()
+
+	// Load all id/keys for each field.
+	log.Printf("loading field identifiers")
+	fieldIDMap, err := loadFields(ctx, client)
+	if err != nil {
+		return fmt.Errorf("cannot load field identifiers: %w", err)
+	} else if len(fieldIDMap) == 0 {
+		return fmt.Errorf("no field identifiers available, please verify data exists")
+	}
+
+	// Generate list of sorted keys.
+	fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
+	for k, f := range fieldIDMap {
+		switch *typ {
+		case "row-bsi":
+			if f.info.Options.Type != "int" {
+				continue
+			}
+		case "row-range", "topk":
+			if f.info.Options.Type != "time" {
+				continue
+			}
+		default:
+			if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
+				continue
+			}
+		}
+
+		fieldKeys = append(fieldKeys, k)
+	}
+	sort.Slice(fieldKeys, func(i, j int) bool {
+		return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
+	})
+
+	// Ensure we have appropriate fields for our query type.
+	if len(fieldKeys) == 0 {
+		return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
+	}
+
+	log.Printf("issuing %d queries at %d query/sec", *n, *rate)
+
+	// Repeatedly issue queries based on available row data.
+	ticker := time.NewTicker(time.Second / time.Duration(*rate))
+	var g errgroup.Group
+	for i := 0; i < *n; i++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		key := fieldKeys[rand.Intn(len(fieldKeys))]
+		q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
+		if err != nil {
+			return fmt.Errorf("cannot generate query: %w", err)
+		}
+
+		log.Printf("[query] %s", q)
+
+		g.Go(func() error {
+			t := time.Now()
+			_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
+			if err != nil {
+				return err
+			}
+			elapsed := time.Since(t).Seconds()
+			requestCountVar.Add(1)
+			requestTotalLatencyVar.Add(elapsed)
+			requestAvgLatencyVar.Set(requestTotalLatencyVar.Value() / float64(requestCountVar.Value()))
+			return nil
+		})
+	}
+
+	return g.Wait()
+}
+
+// monitor runs in a separate goroutine and updates metrics.
+func monitor() {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	var lastTime time.Time
+	var lastN int64
+	var lastLatency float64
+	for range ticker.C {
+		now, n := time.Now(), requestCountVar.Value()
+		latency := requestTotalLatencyVar.Value()
+
+		if !lastTime.IsZero() {
+			elapsed := lastTime.Sub(now).Seconds()
+			if n > 0 {
+				requestCurrentLatencyVar.Set((lastLatency - latency) / float64(n))
+			}
+			requestPerSecVar.Set(float64(lastN-n) / elapsed)
+		}
+		lastTime, lastN, lastLatency = now, n, latency
+	}
+}
+
+func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
+	switch typ {
+	case "row":
+		return generateRowQuery(index, field, identifiers), nil
+	case "row-bsi":
+		return generateRowBSIQuery(index, field), nil
+	case "row-range":
+		return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
+	case "count":
+		return generateCountQuery(index, field, identifiers), nil
+	case "intersect":
+		return generateIntersectQuery(index, field, identifiers), nil
+	case "union":
+		return generateUnionQuery(index, field, identifiers), nil
+	case "difference":
+		return generateDifferenceQuery(index, field, identifiers), nil
+	case "xor":
+		return generateXorQuery(index, field, identifiers), nil
+	case "groupby":
+		return generateGroupByQuery(index, field), nil
+	case "topk":
+		return generateTopKQuery(index, field, opt.from, opt.to), nil
+	default:
+		return "", fmt.Errorf("invalid query type: %q", typ)
+	}
+}
+
+func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
+	}
+	return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
+}
+
+func generateRowBSIQuery(index, field string) string {
+	return fmt.Sprintf("Row(%s > 0)", field)
+}
+
+func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+	}
+	return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	a := make([]string, rand.Intn(9)+1)
+	for i := range a {
+		a[i] = generateRowQuery(index, field, identifiers)
+	}
+	return strings.Join(a, ", ")
+}
+
+func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
+}
+
+func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateGroupByQuery(index, field string) string {
+	return fmt.Sprintf("GroupBy(Rows(%s))", field)
+}
+
+func generateTopKQuery(index, field string, from, to time.Time) string {
+	return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+// loadFields returns a mapping of index/field names to field info & identifiers.
+func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
+	indexes, err := client.Schema(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[fieldKey]*fieldInfo)
+	for _, ii := range indexes {
+		for _, f := range ii.Fields {
+			log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
+
+			switch f.Options.Type {
+			case "set", "mutex", "time":
+				identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
+				if err != nil {
+					return nil, fmt.Errorf("fetch fields: %w", err)
+				} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
+					m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
+				}
+
+			case "int":
+				m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
+			}
+		}
+	}
+	return m, nil
+}
+
+// fetchFieldIDs returns a list of field IDs or keys.
+func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
+	resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
+	if err != nil {
+		return nil, err
+	}
+
+	switch result := resp.Results[0].(type) {
+	case *pilosa.RowIdentifiers:
+		return result, nil
+	case pilosa.RowIdentifiers:
+		return &result, nil
+	default:
+		return nil, fmt.Errorf("unexpected result type: %T", result)
+	}
+}
+
+func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
+	return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
+}
+
+func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
+	return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
+}
+
+type fieldKey struct {
+	index string
+	field string
+}
+
+type fieldInfo struct {
+	info        *pilosa.FieldInfo
+	identifiers *pilosa.RowIdentifiers
+}
+
+func compareFieldKeys(x, y fieldKey) int {
+	if cmp := strings.Compare(x.index, y.index); cmp != 0 {
+		return cmp
+	}
+	return strings.Compare(x.field, y.field)
+}
+
+type queryOptions struct {
+	from, to time.Time
+}

cmd/pilosa-chk/chk.go

@@ -0,0 +1,111 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/hash"
+	"github.com/zeebo/blake3"
+)
+
+// pilosa-chk : read boltdb files and print checksums and counts on the keys. With
+// -v and -ops and -bits you can display every last bit if you want.
+//
+// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
+// forcing a customer to upgrade or downgrade their installed version.
+
+func main() {
+
+	var dir string
+	var showOpsLog bool
+	var showBits bool
+	var showFrags bool
+	var dirChecksum bool
+	home := os.Getenv("HOME")
+	flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
+	flag.BoolVar(&showFrags, "v", false, "show the checksum hash for each fragment in each index. Warning: long output")
+	flag.BoolVar(&showOpsLog, "ops", false, "show the ops log for each fragment. Warning: very long output. Implies -v")
+	flag.BoolVar(&showBits, "bits", false, "show the hot bits for each fragment. Warning: very, very long output. Implies -v")
+	flag.BoolVar(&dirChecksum, "dirsum", false, "compute a directory hash")
+	flag.Parse()
+
+	if showBits {
+		showFrags = true
+	}
+	if showOpsLog {
+		showFrags = true
+	}
+	fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
+
+	if dirChecksum {
+		fmt.Printf("path '%v' has dirhash %v\n", dir, hash.HashOfDir(dir))
+		return
+	}
+
+	fmt.Printf("    the blake-3 hash includes the value of each mapping and the field or partitionID.\n")
+
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+
+	err := holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("\ncalculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
+	var indexes []*pilosa.Index
+
+	final := pilosa.NewAllTranslatorSummary()
+	const verbose = true
+	const checkKeys = false
+	const applyKeyRepairs = false
+	for _, idx := range holder.Indexes() {
+		asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID", 10)
+		if err != nil {
+			log.Fatal(err)
+		}
+		final.Append(asum)
+		indexes = append(indexes, idx)
+	}
+	final.Sort()
+
+	hasher := blake3.New()
+	fmt.Printf("\nsummary of col/row translations%v:\n", dir)
+	for _, sum := range final.Sums {
+		//fmt.Printf("index: %v  partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		_, _ = hasher.Write([]byte(sum.Checksum))
+	}
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	fmt.Printf("all-checksum = blake3-%x\n", buf)
+
+	if showFrags {
+		for _, idx := range indexes {
+			fmt.Printf("==============================\n")
+			fmt.Printf("index: %v\n", idx.Name())
+			fmt.Printf("==============================\n")
+			idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, nil, verbose)
+		}
+	}
+}

cmd/pilosa-fsck/Makefile

@@ -0,0 +1,36 @@
+.PHONY: install build release
+
+CLONE_URL=github.com/pilosa/pilosa
+VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
+VARIANT = Molecula
+VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
+BUILD_TIME := $(shell date -u +%FT%T%z)
+SHARD_WIDTH = 20
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)"
+GOOS = $(shell go env GOOS)
+
+# Install pilosa-fsck
+install:
+	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) 
+
+# Compile pilosa-fsck
+build:
+	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS)
+
+REL = release-pilosa-fsck.$(COMMIT).$(GOOS)
+
+release:
+	mkdir $(REL)
+	cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - )
+	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck
+	tar cf - $(REL) | gzip > $(REL).tar.gz
+	rm -rf $(REL)
+	mv $(REL).tar.gz ../..
+
+clean:
+	find . -name pilosa-fsck | xargs rm -f
+	rm -f release-pilosa-fsck*.tar.gz

cmd/pilosa-fsck/fsck.go

@@ -0,0 +1,985 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/internal"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pkg/errors"
+	"github.com/zeebo/blake3"
+)
+
+// pilosa-fsck :
+// an external customer tool (originally for Q2) to do 2 jobs:
+// Given a set of cluster backups (and their .id and .topology files)
+// mounted on the same file system, we can:
+// 1) scan for fragment differences between the primary and its replicas (default); or
+// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given).
+//
+// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
+// forcing a customer to upgrade or downgrade their installed version.
+
+// FsckConfig configures the dumpcols() and/or read() runs.
+type FsckConfig struct {
+	Fix    bool // -fix
+	FixCol bool // -fixcol
+
+	Colkeydump    bool   // -col
+	JustThisIndex string // -index
+
+	// -col column key dump only options:
+	// Dir         string
+	// PartitionID int
+	// ShowHeader  bool
+	// ShowKey     bool
+	// ShowID      bool
+
+	// not flags, just the Args() left after all other flags. Should be the list
+	// of pilosa (holder) directories for the cluster.
+	Dirs []string
+
+	Verbose bool // -v
+	Quiet   bool // -q
+
+	// manual workaround for not having PilosaConfigPath, if really need be.
+	ReplicaN         int    // -replicas
+	PilosaConfigPath string // -config
+
+	ParallelReaders int //  -readers
+
+	topo *pilosa.Topology
+}
+
+// call DefineFlags before myflags.Parse()
+func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol")
+	fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.")
+	//fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis")
+	fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet")
+
+	fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.")
+
+	fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.")
+
+	fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)")
+
+	fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.")
+
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo())
+		fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
+
+  -fix
+    	(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
+
+  -replicas R 
+        (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
+        the number of replicas maintained in the cluster. Must be the same as the 
+        [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
+
+  -index index_name
+        (optional) restrict to just this index. Otherwise we default to all indexes.
+
+  -readers PR
+        how many parallel readers to use to scan at once. PR==0 means do everything 
+        possible in parallel. PR==1 means serialize everything through a single reader. 
+        Adjust PR to control memory consumption if needed. As a practical limit, setting 
+        PR > 10000 will have no effect. (default is 10).
+
+  -q
+        be very quiet during analysis and repair
+
+`)
+		fmt.Fprintf(os.Stderr, `
+Welcome to pilosa-fsck. This is a scan and repair 
+tool that is modeled after the classic unix file 
+system utility fsck. 
+
+WARNING: DO NOT RUN ON A LIVE SYSTEM.
+
+The most important point to remember is that analysis
+and repair must be done *offline*.
+
+Just as fsck must be run on an unmounted disk, 
+pilosa-fsck must be run on a backup. It must 
+not be run on the directories where a live Pilosa system
+is serving queries. Instead, take a backup first.
+A backup is a set of N Pilosa data directories that have been
+copied from your live system. They must all 
+be visible and mounted on one filesystem together.
+
+pilosa-fsck can be run in scan-mode (without -fix),
+or in repair-mode with -fix. The console output
+supplies a log documenting the analysis 
+and showing what data changes would have been made.
+
+REQUIRED COMMAND LINE ARGUMENTS
+
+The paths to all the top-level Pilosa 
+data directories in a cluster must be given on the command
+line. The -replicas R flag is also always required. It
+must be correct for your cluser. Here R is the same as
+the [cluster] stanza "replicas = R" line from your
+pilosa.conf.
+
+Example:
+
+Suppose you are ready to run pilosa-fsck: 
+you have taken a backup of your four node Pilosa 
+cluster and stored it all on one filesystem with 
+all nodes visible and uncompressed. This
+is a pre-requisite to running pilosa-fsck. 
+Let's suppose we have replication R = 3 set.
+In this example, have stored our backed-up directories in 
+
+/backup/molecula
+
+and the four node backups are in 
+subdirectories node1/ node2/ node3/ node4/ under this:
+
+/backup/molecula/node1/
+/backup/molecula/node1/.pilosa/.id
+/backup/molecula/node1/.pilosa/.topology
+/backup/molecula/node1/.pilosa/myindex
+
+/backup/molecula/node2/
+/backup/molecula/node2/.pilosa/.id
+/backup/molecula/node2/.pilosa/.topology
+/backup/molecula/node2/.pilosa/myindex
+
+/backup/molecula/node3/
+/backup/molecula/node3/.pilosa/.id
+/backup/molecula/node3/.pilosa/.topology
+/backup/molecula/node3/.pilosa/myindex
+
+/backup/molecula/node4/
+/backup/molecula/node4/.pilosa/.id
+/backup/molecula/node4/.pilosa/.topology
+/backup/molecula/node4/.pilosa/myindex
+
+NOTE: your .pilosa directories need not be named .pilosa. They can
+be something else, such as when the -d flag to pilosa server was used.
+The .id file, the .topology file, and the index directories must be 
+found directly underneath.
+
+Then a typical invocation to scan a cluster backup for issues:
+
+$ cd /backup/molecula/
+$ pilosa-fsck -replicas 3  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+A typical invocation to repair the replication in the same backup:
+
+$ pilosa-fsck -replicas 3 -fix  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+In both cases, the .id and .topology files must 
+be present in the backups.
+
+Without -fix, no modifications will be made to the backups. Only
+by running with -fix will repairs be made. The user can safely
+always run with -fix to repair only if needed.
+
+A zero error code will be returned to the shell if no repairs were needed.
+
+A zero error code will be also be returned to the shell if 
+repairs were needed and they were accomplished under -fix.
+
+A non-zero error code indicates that repairs were needed but
+were not made.
+`)
+	}
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *FsckConfig) ValidateConfig() error {
+	if c.Fix {
+		c.FixCol = true
+	}
+	if c.ReplicaN == 0 && c.PilosaConfigPath == "" {
+		return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)")
+	}
+
+	if c.ReplicaN == 0 && c.PilosaConfigPath != "" {
+
+		if !FileExists(c.PilosaConfigPath) {
+			return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath)
+		}
+		by, err := ioutil.ReadFile(c.PilosaConfigPath)
+		if err != nil {
+			return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err)
+		}
+		srvcfg, err := server.ParseConfig(string(by))
+		if err != nil {
+			//vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err)
+
+			// fall back to manual parsing of config
+			lines := strings.Split(string(by), "\n")
+			clusterStart := -1
+			for i, line := range lines {
+				if strings.Contains(line, `[cluster]`) {
+					clusterStart = i
+				}
+				if i > clusterStart {
+					if strings.Contains(line, "replicas") {
+						split := strings.Split(line, "=")
+						ns := strings.TrimSpace(split[1])
+						n, err := strconv.Atoi(ns)
+						if err != nil {
+							return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err)
+						}
+						c.ReplicaN = n
+					}
+				}
+			}
+		} else {
+			c.ReplicaN = srvcfg.Cluster.ReplicaN
+		}
+		if c.ReplicaN == 0 {
+			return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath)
+		}
+		//vv("c.ReplicaN = %v", c.ReplicaN)
+	}
+	return nil
+}
+
+var ProgramName = "pilosa-fsck"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError)
+	cfg := &FsckConfig{}
+	cfg.DefineFlags(myflags)
+	cfg.Verbose = true
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+	dirs := myflags.Args()
+	nDir := len(dirs)
+	if nDir <= 0 && !cfg.Colkeydump {
+		fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName)
+		os.Exit(1)
+	}
+
+	cmdline := strings.Join(os.Args, " ")
+
+	// make sure all the dir are distinct
+	dup := make(map[string]bool)
+	for _, dir := range dirs {
+		if dup[dir] {
+			fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline)
+			os.Exit(1)
+		} else {
+			dup[dir] = true
+		}
+	}
+
+	fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo())
+	cwd, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err)
+		os.Exit(1)
+	}
+	fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd)
+	fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline)
+	t0 := time.Now()
+	fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0))
+	defer func() {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+	}()
+	cfg.Dirs = dirs
+
+	fixNeeded, err := cfg.Run()
+	if err != nil {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+	if fixNeeded && !cfg.Fix {
+		fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
+		fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n")
+		os.Exit(1)
+	}
+}
+
+func (cfg *FsckConfig) Run() (fixNeeded bool, err error) {
+
+	//	if cfg.Colkeydump {
+	//	cfg.dumpcols()
+	//}
+
+	perNodeIndexMaps, clusterNodes, ats, err := cfg.read()
+	if err != nil {
+		return false, err
+	}
+
+	if cfg.FixCol {
+		err := cfg.RepairTranslationStores(ats)
+		if err != nil {
+			return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err)
+		}
+	}
+
+	//vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes)
+
+	fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats)
+	if err != nil {
+		return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err)
+	}
+	fixNeeded = ats.RepairNeeded || fixme
+	for _, report := range reports {
+		fmt.Printf("%v\n", report)
+	}
+	if len(reports) == 0 {
+		fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " "))
+	}
+	return
+}
+
+var _ = (&FsckConfig{}).dumpAts
+
+func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) {
+	fmt.Printf("# dumpAts:  	RepairNeeded=%v\n", ats.RepairNeeded)
+	for _, sum := range ats.Sums {
+		fmt.Printf("# sum = '%#v'\n", sum)
+	}
+
+}
+
+type group struct {
+	elem        []*pilosa.TranslatorSummary
+	partitionID int
+}
+
+func (g *group) String() (s string) {
+	for i, e := range g.elem {
+		s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String())
+	}
+	return
+}
+
+func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) {
+	indexMap := make(map[string]bool)
+	for _, sum := range ats.Sums {
+		if !indexMap[sum.Index] {
+			indexMap[sum.Index] = true
+			indexes = append(indexes, sum.Index)
+		}
+	}
+	sort.Strings(indexes)
+	return
+}
+
+func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) {
+
+	verbose := cfg.Verbose
+
+	// group by index first. then repair.
+	indexes := indexesFromAts(ats)
+
+	for _, index := range indexes {
+
+		if !cfg.DoingIndex(index) {
+			continue
+		}
+
+		m := make(map[int]*group)
+		for _, sum := range ats.Sums {
+
+			if !sum.IsColKey || sum.Index != index {
+				continue
+			}
+			grp := m[sum.PartitionID]
+			if grp == nil {
+				grp = &group{
+					partitionID: sum.PartitionID,
+				}
+				m[sum.PartitionID] = grp
+			}
+			grp.elem = append(grp.elem, sum)
+		}
+
+		for partitionID, group := range m {
+			_ = partitionID
+			prim := -1
+			keyCount := 0
+			for k, e := range group.elem {
+				if e.IsPrimary {
+					prim = k
+				}
+				keyCount += e.KeyCount
+			}
+			if prim == -1 {
+				panic(fmt.Sprintf("no primary found for group '%v'", group.String()))
+			}
+
+			primary := group.elem[prim]
+			primaryChecksum := primary.Checksum
+			for _, e := range group.elem {
+				if e.IsPrimary {
+					continue
+				}
+				// is e a replica? not necessarily! have to check.
+				if !e.IsReplica {
+					//if verbose {
+					// since this will happen even on a fix point, where it is already empty,
+					// we don't report it again.
+					//fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath)
+					//}
+					err := os.RemoveAll(e.StorePath)
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath))
+					}
+					store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN)
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath))
+					}
+					err = store.Close()
+					if err != nil {
+						return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath))
+					}
+					continue
+				}
+				// INVAR: e is a replica for this paritionID.
+				// Copy from primary if checksums are different.
+				if e.Checksum != primaryChecksum {
+					from := group.elem[prim].StorePath
+					dest := e.StorePath
+					if verbose {
+						fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest)
+					}
+					err := cp(from, dest)
+					if err != nil {
+						return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+/*
+func (cfg *FsckConfig) dumpcols() {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	dir := cfg.Dir
+	index := cfg.Index
+	partitionID := cfg.PartitionID
+	showKey := cfg.ShowKey
+	showID := cfg.ShowID
+
+	if !quiet {
+		fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir)
+	}
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+	err := holder.Open()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if cfg.ShowHeader {
+		fmt.Println("# columnKey columId")
+	}
+	id_key := make(map[uint64]string)
+	key_id := make(map[string]uint64)
+	for _, idx := range holder.Indexes() {
+		fmt.Printf("# Looking '%v'\n", idx.Name())
+		if idx.Name() == index {
+			store := idx.TranslateStore(partitionID)
+			fmt.Printf("# Key By ID partitionID = %v\n", partitionID)
+			err := store.KeyWalker(func(key string, col uint64) {
+				key_id[key] = col
+				if showKey {
+					fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID)
+				}
+			})
+			panicOn(err)
+		}
+	}
+	for _, idx := range holder.Indexes() {
+		if idx.Name() == index {
+			store := idx.TranslateStore(partitionID)
+			//fmt.Printf("# ID ByKey\n")
+			err := store.IDWalker(func(key string, col uint64) {
+				id_key[col] = key
+				if showID {
+					fmt.Printf("# '%v' %v\n", key, col)
+				}
+			})
+			panicOn(err)
+		}
+	}
+	fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key))
+	fmt.Println("id_key")
+	for k, v := range id_key {
+		l, ok := key_id[v]
+		if ok {
+			if k != l {
+				fmt.Printf("# X: %v %v %v\n", k, l, v)
+			}
+		} else {
+			fmt.Printf("# key not in id %v\n", v)
+		}
+	}
+	fmt.Println("key_id")
+	for k, v := range key_id {
+		l, ok := id_key[v]
+		if ok {
+			if k != l {
+				fmt.Printf("# T: %v %v %v\n", k, l, v)
+			}
+		} else {
+			fmt.Printf("# id not in key %v\n", v)
+		}
+	}
+}
+*/
+
+func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) {
+
+	final = pilosa.NewAllTranslatorSummary()
+
+	dirs := cfg.Dirs
+	for _, dir := range dirs {
+		idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		final.Append(atsNode)
+		clusterNodes = append(clusterNodes, nodeID)
+		perNodeIndexMaps = append(perNodeIndexMaps, idx2frag)
+	}
+	return
+}
+
+func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+
+	if !quiet {
+		fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir)
+	}
+
+	jmphasher := &pilosa.Jmphasher{}
+	partitionN := pilosa.DefaultPartitionN
+	replicaN := cfg.ReplicaN
+	topo, err := loadTopology(dir, jmphasher, partitionN, replicaN)
+	if err != nil {
+		return nil, "", nil, err
+	}
+	cfg.topo = topo
+	//vv("topo = '%#v'", topo)
+	nodeIDs := topo.GetNodeIDs()
+	//vv("nodeIDs = '%#v'", nodeIDs)
+	nNodes := len(nodeIDs)
+	nDir := len(cfg.Dirs)
+	if nDir != nNodes {
+		return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs)
+	}
+
+	holder := pilosa.NewHolder(dir, nil)
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+
+	nodeID, err = holder.LoadNodeID()
+	panicOn(err)
+	//vv("nodeID = '%v'", nodeID)
+	err = holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if !quiet {
+		fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
+	}
+	var indexes []*pilosa.Index
+
+	const checkKeys = true
+	atsNode = pilosa.NewAllTranslatorSummary()
+	for _, idx := range holder.Indexes() {
+
+		if !cfg.DoingIndex(idx.Name()) {
+			continue
+		}
+
+		//vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol)
+
+		asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders)
+		if err != nil {
+			log.Fatal(err)
+		}
+		atsNode.Append(asum)
+		indexes = append(indexes, idx)
+	}
+	atsNode.Sort()
+
+	hasher := blake3.New()
+	if !quiet {
+		fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir)
+	}
+	for _, sum := range atsNode.Sums {
+		if !quiet {
+			fmt.Printf("# index: %v  partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		}
+		_, _ = hasher.Write([]byte(sum.Checksum))
+	}
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	if !quiet {
+		fmt.Printf("# all-checksum = blake3-%x\n", buf)
+	}
+
+	// fragment analysis
+
+	showBits := false
+	showOpsLog := false
+	idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node.
+	for _, idx := range indexes {
+		if verbose {
+			fmt.Printf("# ==============================\n")
+			fmt.Printf("# index: %v\n", idx.Name())
+			fmt.Printf("# ==============================\n")
+		}
+		frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose)
+		frgsum.Dir = dir
+		frgsum.NodeID = nodeID
+		idx2frag[idx.Name()] = frgsum
+	}
+
+	_ = holder.Close()
+
+	//vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple.
+
+	return
+}
+
+func (cfg *FsckConfig) DoingIndex(index string) bool {
+	if cfg.JustThisIndex == "" {
+		// scan all indexes
+		return true
+	}
+	if index == cfg.JustThisIndex {
+		// scan just this one
+		return true
+	}
+	return false
+}
+
+// from cluster.go:1924
+func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) {
+
+	buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology"))
+	if err != nil {
+		return nil, err
+	}
+
+	var pb internal.Topology
+	err = proto.Unmarshal(buf, &pb)
+	if err != nil {
+		return nil, err
+	}
+
+	return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil)
+}
+
+func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	allIndex := make(map[string]bool)
+	for _, mp := range perNodeIndexMaps {
+		for index := range mp {
+			allIndex[index] = true
+		}
+	}
+	if !quiet {
+		vv("allIndex = '%#v'", allIndex)
+	}
+	for index := range allIndex {
+		if !quiet {
+			vv("on index '%v'", index)
+		}
+		nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary)
+		for _, mp := range perNodeIndexMaps {
+			sum := mp[index]
+			if sum == nil {
+				continue
+			}
+			nodes2fragsum[sum.NodeID] = sum
+		}
+		fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats)
+		if err != nil {
+			return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err)
+		}
+		fixNeeded = fixNeeded || fixme
+		reports = append(reports, report)
+	}
+	return fixNeeded, reports, nil
+}
+
+func (cfg *FsckConfig) analyzeThisIndex(
+	index string,
+	nodes2fragsum map[string]*pilosa.IndexFragmentSummary,
+	ats *pilosa.AllTranslatorSummary,
+) (fixNeeded bool, report string, err error) {
+
+	verbose := cfg.Verbose
+	quiet := cfg.Quiet
+	_, _ = verbose, quiet
+
+	var removedBytes int64
+	var copiedBytes int64
+	var changedFiles int64
+	var totalFiles int64
+	var overwrittenBytes int64
+	var totalBytes int64
+
+	if !quiet {
+		vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'",
+			index, len(nodes2fragsum), nodes2fragsum)
+	}
+
+	for node, sum := range nodes2fragsum {
+		if !quiet {
+			fmt.Printf("# on node '%v'\n", node)
+		}
+		// do they disagree on who is the primary?
+		// for each fragment, do they disagree on the checksum?
+
+		// Q: which nodes are supposed to have data, and which
+		// nodes are not supposed to have data?
+
+		//	loopFragSum:
+		for relpath, fragsum := range sum.RelPath2fsum {
+			fragsum.NodeID = node
+			totalFiles++
+			//vv("checking %v on node %v", relpath, node)
+
+			replicas, nonReplicas := cfg.topo.GetReplicasForPrimary(fragsum.Primary)
+			_, _ = replicas, nonReplicas
+			//vv("replicas = '%#v'", replicas)
+			//vv("nonReplicas = '%#v'", nonReplicas)
+
+			err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum)
+			if err != nil {
+				return fixNeeded, "", err
+			}
+
+			// find the primary's checksum
+			primaryChecksum := ""
+			var primaryFragSum *pilosa.FragSum
+			for node, isPrimary := range replicas {
+				if isPrimary {
+					primarySum := nodes2fragsum[node]
+					primaryFragSum = primarySum.RelPath2fsum[relpath]
+					if primaryFragSum == nil {
+
+						// This seems clear indication that we have the topology wrong.
+						// When the topology is right, there are NO errors of this kind.
+						//
+						msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas)
+						vv(msg)
+						fmt.Fprintf(os.Stderr, "%v\n", msg)
+						panic(msg) // stop. the fixes are going to be wrong.
+					} else {
+						primaryChecksum = primaryFragSum.Checksum
+						primaryFragSum.NodeID = node
+						primaryFragSum.ScanDone = true
+					}
+					break
+				}
+			}
+			if primaryChecksum == "" {
+				return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum)
+			}
+
+			// is this a non-replica?
+			_, isNon := nonReplicas[fragsum.NodeID]
+			if isNon {
+				removedBytes += FileSize(fragsum.AbsPath)
+				changedFiles++
+
+				//vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID)
+				if !quiet {
+					fmt.Printf("rm %v             #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum)
+				}
+				if cfg.Fix {
+					err := os.Remove(fragsum.AbsPath)
+					if err != nil {
+						return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err)
+					}
+				}
+			} else {
+				presz := FileSize(fragsum.AbsPath)
+				totalBytes += presz
+
+				checksum := fragsum.Checksum
+				if checksum != primaryChecksum {
+					copiedBytes += FileSize(primaryFragSum.AbsPath)
+					changedFiles++
+					overwrittenBytes += presz
+
+					if !quiet {
+						fmt.Printf("cp %v %v        #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum)
+					}
+					if cfg.Fix {
+						err := cp(primaryFragSum.AbsPath, fragsum.AbsPath)
+						if err != nil {
+							return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'",
+								primaryFragSum.AbsPath, fragsum.AbsPath, err)
+						}
+					}
+				}
+			}
+			fragsum.ScanDone = true
+		}
+	}
+	nDir := len(nodes2fragsum)
+
+	keyCount, idCount := cfg.getKeyIDCounts(index, ats)
+
+	fixNeeded = changedFiles > 0 || ats.RepairNeeded
+	var actionTaken string
+	var wouldBe string
+	if cfg.Fix || cfg.FixCol {
+		if fixNeeded {
+			actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*"
+			wouldBe = "sync repairs made:"
+		} else {
+			wouldBe = ""
+			actionTaken = "NO REPAIR NEEDED."
+		}
+	} else {
+		if fixNeeded {
+			wouldBe = "sync actions that would be taken under -fix:"
+			actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted."
+		} else {
+			wouldBe = ""
+			actionTaken = "NO REPAIR NEEDED."
+		}
+	}
+	var fragUpdate string
+	if changedFiles > 0 {
+		fragUpdate = fmt.Sprintf(`
+#  %v
+#    copied bytes: %v
+#    file bytes overwritten: %v
+#    new bytes added: %v
+#    new bytes is %0.01f%% of %v total bytes
+#    removed %v bytes from non-replicas
+#    changed file count %v (%0.01f%%; total files=%v)
+#
+`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles))
+	}
+
+	report = fmt.Sprintf(`
+# ========================================================
+#   pilosa-fsck final report 
+#
+#     run with    -fix: %v 
+#
+# index examined: '%v'
+#
+# nodes examined: %v
+#       -replicas %v replication factor used
+#
+# feature data examined: %v bytes
+# feature files examined: %v files
+#
+# key-translation-stores examined: %v
+#                       key-count: %v   over all replicas
+#                        id-count: %v   over all replicas
+#
+# %v
+# %v
+# ========================================================
+`,
+		cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate)
+	return
+}
+
+func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error {
+	for node := range replicas {
+		if nodes2fragsum[node] == nil {
+			return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum)
+		}
+	}
+	return nil
+}
+
+func cp(fromPath, toPath string) (err error) {
+	tmpTo := toPath + ".fsck.tmp"
+	toFd, err := os.Create(tmpTo)
+	if err != nil {
+		return err
+	}
+	defer toFd.Close()
+	fromFd, err := os.Open(fromPath)
+	if err != nil {
+		return err
+	}
+	defer fromFd.Close()
+
+	_, err = io.Copy(toFd, fromFd)
+	if err != nil {
+		return err
+	}
+	err = toFd.Close()
+	if err != nil {
+		return err
+	}
+	return os.Rename(tmpTo, toPath)
+}
+
+func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) {
+	for _, sum := range ats.Sums {
+		if sum.Index == index {
+			keyCount += sum.KeyCount
+			idCount += sum.IDCount
+		}
+	}
+	return
+}

cmd/pilosa-fsck/fsck_test.go

@@ -0,0 +1,445 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/hash"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_Repair(t *testing.T) {
+
+	// a) setup 1 primary + 3 replicas of disagree-ing cluster dirs.
+
+	nNodes := 4
+	nReplicas := 3
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[1]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[2]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[3]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	// note: do not defer c.Close() here. We manually close below.
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+
+	ctx := context.Background()
+
+	index := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(index))
+	field := make([]*pilosa.Field, len(index))
+	var err error
+
+	for i := range index {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	rowID := uint64(1)
+	timestamp := int64(0)
+
+	for i := range index {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          index[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+		//qcx.Reset()
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID)
+
+		// Query node0.
+		if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
+		}
+
+		// Query node1.
+		if err := test.RetryUntil(5*time.Second, func() error {
+			if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
+				return err
+			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+				return fmt.Errorf("unexpected column keys: %#v", keys)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// end of setup.
+
+	// partitionID in use: 6, 31, 57, 133, 185, 235
+	targetPartition := 31  // which partitionID we mess with.
+	targetNode := nodes[0] // this is the first replica.
+	targetIndex := index[0]
+	// 0 first replica
+	// 1 second replica
+	// 2  -- not a replica
+	// 3 primary
+
+	cfg := &FsckConfig{
+		Fix:    false,
+		FixCol: false,
+		Quiet:  true,
+		//Verbose:  true,
+		ReplicaN:        nReplicas,
+		Dirs:            dirs,
+		ParallelReaders: 5,
+	}
+	panicOn(cfg.ValidateConfig())
+
+	// for this test, mess up a replica that is not the primary.
+
+	h := targetNode.API.Holder()
+	idx[0] = h.Index(index[0])
+	store := idx[0].TranslateStore(targetPartition)
+	fwd, rev := getFwdRev(store, targetPartition)
+	//vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev)
+
+	// # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001
+	presz := len(rev)
+	delete(rev, fwd["col5"])
+	postsz := len(rev)
+
+	if postsz == presz {
+		panic("did not delete any key!")
+	}
+
+	bolt := store.(*boltdb.TranslateStore)
+	//vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path))
+	//bolt.DumpBolt("pre-corruption")
+
+	if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil {
+		t.Fatal(err)
+	}
+	//vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path))
+	//bolt.DumpBolt("post-corruption")
+
+	//fwd3, rev3 := getFwdRev(store, targetPartition)
+	//vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3)
+
+	targetIndex1 := "morty"
+	targetPartition1 := 226 // for "col11"
+	// # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}'
+	//# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}'
+	idx[1] = h.Index(index[1])
+	store1 := idx[1].TranslateStore(targetPartition1)
+	fwd1, rev1 := getFwdRev(store1, targetPartition1)
+	//vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1)
+
+	presz1 := len(rev1)
+	delete(rev1, fwd1["col11"])
+	postsz1 := len(rev1)
+
+	if postsz1 == presz1 {
+		panic("did not delete any key!")
+	}
+	bolt1 := store1.(*boltdb.TranslateStore)
+	if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil {
+		t.Fatal(err)
+	}
+
+	// done corrupting.
+	for _, nd := range nodes {
+		nd.Command.Close()
+	}
+	//panicOn(bolt.Open())
+	//bolt.DumpBolt("post-corruption, after Close. bolt:")
+	//bolt.Close()
+
+	//chksums := getChecksums(dirs, cfg, targetPartition)
+	//vv("post corruption, pre repair chksums = '%#v'", chksums)
+
+	// first we check that the corruption can be detected
+	// by our test with the checksums.
+
+	chk, err := check(dirs, cfg, targetIndex, targetPartition)
+	_ = chk
+	//vv("pre-fix, chk='%v'; err='%v'", chk, err)
+
+	if err == nil {
+		panic("expected to see checksums not match! but no corruption detected.")
+	}
+
+	chk1, err := check(dirs, cfg, targetIndex1, targetPartition1)
+	_ = chk1
+	//vv("pre-fix, chk1='%v'; err='%v'", chk1, err)
+
+	if err == nil {
+		panic("expected to see checksums not match! but no corruption detected.")
+	}
+
+	// b) running in reporting mode only should report that a fix is needed.
+	fixNeeded, err := cfg.Run()
+	panicOn(err)
+	if !fixNeeded {
+		panic("fix should be needed now, before repair")
+	}
+
+	// c) run the fix.
+	cfg.Fix = true
+	cfg.FixCol = true
+
+	fixNeeded, err = cfg.Run()
+	panicOn(err)
+	if !fixNeeded {
+		panic("fix should be marked needed if repair was made")
+	}
+
+	// d) check that the replicas all look like the primary.
+
+	//chksums = getChecksums(dirs, cfg, targetPartition)
+	//vv("after repair chksums = '%#v'", chksums)
+
+	chk, err = check(dirs, cfg, targetIndex, targetPartition)
+	_ = chk
+	//vv("chk = '%v' after repair; err='%v'", chk, err)
+	panicOn(err)
+
+	chk1, err = check(dirs, cfg, targetIndex1, targetPartition1)
+	_ = chk1
+	//vv("chk = '%v' after repair; err='%v'", chk, err)
+	panicOn(err)
+
+	// e) run again, should see no fix needed.
+	fixNeeded, err = cfg.Run()
+	panicOn(err)
+	if fixNeeded {
+		panic("should see no fix needed after the prior repair")
+	}
+}
+
+func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) {
+	fwd = make(map[string]uint64)
+	rev = make(map[uint64]string)
+	_ = store.KeyWalker(func(key string, col uint64) {
+		//vv("partition %v, key '%v' -> %x", partitionID, key, col)
+		fwd[key] = col
+	})
+	_ = store.IDWalker(func(key string, col uint64) {
+		//vv("partition %v, id %x -> '%v'", partitionID, col, key)
+		rev[col] = key
+	})
+	return
+}
+
+func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) {
+	//vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition)
+	//defer vv("returning from check()")
+
+	firstChecksum := ""
+	firstDir := ""
+	firstStorePath := ""
+	quiet := cfg.Quiet
+	defer func() {
+		cfg.Quiet = quiet
+	}()
+	cfg.Quiet = true
+	for i := range dirs {
+		dir := dirs[i]
+		_, _, ats, err := cfg.readOneDir(dir)
+		panicOn(err)
+		indexes := indexesFromAts(ats)
+		//vv("indexes = '%#v'", indexes)
+
+		for _, index := range indexes {
+
+			if index != targetIndex {
+				continue
+			}
+			for _, s := range ats.Sums {
+				//vv(" s= '%#v'", s)
+				if s.Index != index {
+					//vv("skipping s.Index '%v' != index '%v'", s.Index, index)
+					continue
+				}
+				if s.PartitionID != targetPartition {
+					continue
+				}
+				//vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+
+				//"index '%v';  s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'",
+				//s.PartitionID, targetPartition, s.Index, index,
+				//s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum)
+
+				if s.IsPrimary || s.IsReplica {
+					chksum := s.Checksum
+					if firstChecksum == "" {
+
+						firstChecksum = chksum
+						firstDir = dir
+						firstStorePath = s.StorePath
+
+					} else {
+						//vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum)
+
+						if chksum != firstChecksum {
+							return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath)
+						}
+					}
+				}
+			}
+		}
+	}
+	return firstChecksum, nil
+}
+
+var _ = getChecksums
+
+func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) {
+
+	for i := range dirs {
+		dir := dirs[i]
+		_, _, ats, err := cfg.readOneDir(dir)
+		panicOn(err)
+
+		for _, s := range ats.Sums {
+			if s.PartitionID != targetPartition {
+				continue
+			}
+			chksum = append(chksum, s.Checksum)
+		}
+	}
+	return
+}
+
+/* on shardwidth 20
+# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001
+# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2'
+# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001
+# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5'
+# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001
+# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10'
+# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001
+# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7'
+# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001
+# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3'
+# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001
+# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9'
+*/
+
+var _ = fileChecksum
+
+func fileChecksum(path string) string {
+	by, err := ioutil.ReadFile(path)
+	panicOn(err)
+	return hash.Blake3sum16(by)
+}

cmd/pilosa-fsck/release-pilosa-fsck/.gitignore

@@ -0,0 +1 @@
+pilosa-fsck

cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md

@@ -0,0 +1,252 @@
+Design for pilosa-fsck
+======================
+
+Problem Background
+------------------
+
+Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster.
+
+Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data,
+and Row-Key data are replicated. Only the first two, Roaring data and Column-Key
+data are relevant here. Broadly, the Roaring bitmap data
+forms the central features -- the bits -- of a large, sparse bitmap matrix.
+The Column-Keys are the labels for the columns at the top margin of this matrix.
+
+For speed, the Roaring bitmap data is stored separately from the
+Key data. The Roaring data is stored in sharded files
+within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/...
+The Key translation data is stored in sharded BoltDB databases within
+the PILOSA-DATA-DIR/index_name/_key directory.
+
+The current approach to Roaring file replication involves an
+eventually consistent mechanism that uses an Anti-Entropy agent to
+fix partial or incomplete replication from the primary shard to all
+replica shards.
+
+Unfortunately, the Anti-Entropy agent approach has proved inadequate on two
+fronts. First, it does not provide for immediately consistent reads in the
+event that the primary is lost. Second, the Anti-Entropy agent itself experienced
+out-of-memory issues that have yet to be resolved.
+
+Therefore, work is now underway to replace this replication
+approach with a more consistent design. 
+
+However, in the meantime, for our customers in production with Molecula
+Pilosa, we wish to provide a means to re-establish correct replication.
+Thus even in the event of a node failure followed by a read from a replica, the
+returned read will be correct.
+
+The pilosa-fsck tool can therefore be seen as a temporary, stop-gap
+measure to address immediate issues while the cluster replication
+mechanism is replaced.
+
+The second factor motivating the creation of pilosa-fsck was the discovery
+of a bug in the Key-translation process. Unfortunately this was a hard
+to reproduce bug. It happened only on the customer's premises,
+and only after running the system for a long time, with a
+large amount of data, and with various eccentric node failures
+and recoveries.
+
+However, we were able to reproduce a plausible explanation.
+Non-primary replicas were creating keys when they should have been
+forwarding the request to the primary. Correcting this bug is impetus
+for the v2.1.4 release of Molecula Pilosa.
+
+A fine point here: since we were not able to precisely reproduce the customer's
+issue in the development environment, we cannot guarantee with 100%
+certainty that we have actually addressed the bug that the customer
+was seeing.
+
+Therefore we also desired an additional insurance
+policy. We wished to be able to empower customers to proactively discover any
+future Key-translation issues that happen in their on-premise systems.
+
+To do this, we proposed providing select customers with the pilosa-fsck
+tool which can analyze their offline backups for issues.
+
+Optionally, these issues can also be repaired in-place in the
+offline backup on which pilosa-fsck is run.
+
+The -fix flag repairs both kinds of replication issues.
+
+Solution Approach: mechanism of action
+--------------------------------------
+
+The pilosa-fsck is run offline on a full set of backups taken from
+all nodes in a Pilosa cluster. It runs on a single computer that
+must be separate from the production or staging Pilosa environments.
+
+When run, pilosa-fsck analyzes the differences between the
+primary and its replicas. Both the Roaring
+files and the Key translation databases are analyzed.
+The computer running pilosa-fsck must have the same or more
+memory as the Pilosa nodes in the cluster, as it will
+"pretend" to be each Pilosa node in turn. However, as each
+node's backup is closed before the next node's backup is
+opened, we do not require substantially more memory than a single
+production node. Short Blake3 cryptographic checksums are
+computed for each Roaring fragment and each Key translation
+database. These are held in memory (and printed to the log)
+for comparing nodes. This comparison forms the heart of
+the consistency checks, and is the basis for any subsequent
+repair.
+
+We recommend capturing both stdout and stderr to a log.
+Use `&> log` or  `2>&1 > log` at the end of the
+pilosa-fsck invocation to save a log of the run to disk.
+
+In a typical cluster, the Replication factor R may be less
+than the number of nodes N in the cluster. For example, while
+N may be 4, the R may be only 3. In this example, within
+each replicated shard, one node will be the primary for
+that shard, two nodes will be non-primary replicas, and one
+node will be a non-replica. Note that the designation
+of primary changes for different Roaring shards within an index,
+even on a single node.
+
+The essence of the the -fix repair operation that pilosa-fsck
+can do is this: it will copy from the primary to the
+the non-primary replicas. Further, it will remove data from
+any non-replica node if it was mistakenly present.
+
+The pilosa-fsck output log will contain
+a sequence of command line 'cp' and 'rm' commands.
+These commands are merely a record (with
+accompanying justifcation in the comment following the
+command) of what actions would be performed to repair
+the Roaring file data.
+
+Only with -fix will the repair actions actually happen
+during the pilosa-fsck run.
+
+
+Details: running pilosa-fsck
+----------------------------
+
+Errors in invocation are reported on stderr and the program will exit with a non-zero
+error code if invocation errors are present. A non-zero error code
+is returned if a repair is needed and -fix was not given.
+
+A -fix run will return a zero error code to the shell if the fix was
+successfully made; or if no fix was required.
+
+The log of the run is printed to stdout.
+
+The -h flag to pilosa-fsck prints a summary of its operation
+and a guide to laying out the backup directories.
+
+The help is reproduced below.
+
+~~~
+$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct  5 2020 1:28PM, 9dacbccf)
+
+Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
+
+  -fix
+    	(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
+
+  -replicas R 
+        (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
+        the number of replicas maintained in the cluster. Must be the same as the 
+        [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
+
+  -q
+        be very quiet during analysis and repair
+
+
+Welcome to pilosa-fsck. This is a scan and repair 
+tool that is modeled after the classic unix file 
+system utility fsck. 
+
+WARNING: DO NOT RUN ON A LIVE SYSTEM.
+
+The most important point to remember is that analysis
+and repair must be done *offline*.
+
+Just as fsck must be run on an unmounted disk, 
+pilosa-fsck must be run on a backup. It must 
+not be run on the directories where a live Pilosa system
+is serving queries. Instead, take a backup first.
+A backup is a set of N Pilosa data directories that have been
+copied from your live system. They must all 
+be visible and mounted on one filesystem together.
+
+pilosa-fsck can be run in scan-mode (without -fix),
+or in repair-mode with -fix. The console output
+supplies a log documenting the analysis 
+and showing what data changes would have been made.
+
+REQUIRED COMMAND LINE ARGUMENTS
+
+The paths to all the top-level Pilosa 
+data directories in a cluster must be given on the command
+line. The -replicas R flag is also always required. It
+must be correct for your cluser. Here R is the same as
+the [cluster] stanza "replicas = R" line from your
+pilosa.conf.
+
+Example:
+
+Suppose you are ready to run pilosa-fsck: 
+you have taken a backup of your four node Pilosa 
+cluster and stored it all on one filesystem with 
+all nodes visible and uncompressed. This
+is a pre-requisite to running pilosa-fsck. 
+Let's suppose we have replication R = 3 set.
+In this example, have stored our backed-up directories in 
+
+/backup/molecula
+
+and the four node backups are in 
+subdirectories node1/ node2/ node3/ node4/ under this:
+
+/backup/molecula/node1/
+/backup/molecula/node1/.pilosa/.id
+/backup/molecula/node1/.pilosa/.topology
+/backup/molecula/node1/.pilosa/myindex
+
+/backup/molecula/node2/
+/backup/molecula/node2/.pilosa/.id
+/backup/molecula/node2/.pilosa/.topology
+/backup/molecula/node2/.pilosa/myindex
+
+/backup/molecula/node3/
+/backup/molecula/node3/.pilosa/.id
+/backup/molecula/node3/.pilosa/.topology
+/backup/molecula/node3/.pilosa/myindex
+
+/backup/molecula/node4/
+/backup/molecula/node4/.pilosa/.id
+/backup/molecula/node4/.pilosa/.topology
+/backup/molecula/node4/.pilosa/myindex
+
+NOTE: your .pilosa directories need not be named .pilosa. They can
+be something else, such as when the -d flag to pilosa server was used.
+The .id file, the .topology file, and the index directories must be 
+found directly underneath.
+
+Then a typical invocation to scan a cluster backup for issues:
+
+$ cd /backup/molecula/
+$ pilosa-fsck -replicas 3  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+A typical invocation to repair the replication in the same backup:
+
+$ pilosa-fsck -replicas 3 -fix  node1/.pilosa  node2/.pilosa  node3/.pilosa  node4/.pilosa &> log
+
+In both cases, the .id and .topology files must 
+be present in the backups.
+
+Without -fix, no modifications will be made to the backups. Only
+by running with -fix will repairs be made. The user can safely
+always run with -fix to repair only if needed.
+
+A zero error code will be returned to the shell if no repairs were needed.
+
+A zero error code will be also be returned to the shell if 
+repairs were needed and they were accomplished under -fix.
+
+A non-zero error code indicates that repairs were needed but
+were not made.
+
+~~~

cmd/pilosa-fsck/release-pilosa-fsck/example.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set +x
+export PATH=.:${PATH}
+
+# unpack the sample Molecula Pilosa cluster.
+tar xf backups.tar.gz
+
+
+# check if repair is needed.
+pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
+
+
+# yes, so do the repairs. This can be done first (only) as well.
+#
+pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
+
+
+# check again if you like
+#
+pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa

cmd/pilosa-fsck/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("# %s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) int64 {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return 0
+	}
+	return fi.Size()
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/random-query/main.go

@@ -0,0 +1,564 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math"
+	"math/rand"
+	nethttp "net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/http"
+)
+
+// RandomQueryConfig
+type RandomQueryConfig struct {
+
+	// user facing flags
+	HostPort   string // -hostport
+	TreeDepth  int    // -d
+	QueryCount int    // -n
+	Verbose    bool   // -v
+	VeryVerbose bool  // -V
+	TimeFromArg   string // --time.from
+	TimeToArg     string // --time.to
+	TimeFrom   time.Time // parsed time
+	TimeTo     time.Time // parsed time
+	TimeRange      int64 // hours between parsed times
+
+	IndexMap map[string]*Features
+
+	API  *pilosa.API
+	Info []*pilosa.IndexInfo
+
+	BitmapFunc []string
+
+	Rnd *rand.Rand
+}
+
+type API interface {
+
+	// InternalClient
+	Schema(ctx context.Context) ([]*pilosa.IndexInfo, error)
+	Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error)
+
+	// API for contrast; just a little different:
+	//Schema(ctx context.Context) []*IndexInfo
+	//Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error)
+}
+
+// have to wrap because the ugly little differences between InternalClient and API
+type wrapper struct {
+	api *pilosa.API
+}
+
+func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	return w.api.Schema(ctx), nil
+}
+
+func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	r, err := w.api.Query(ctx, queryRequest)
+	return &r, err
+}
+
+func wrapApiToInternalClient(api *pilosa.API) *wrapper {
+	return &wrapper{api: api}
+}
+
+// These times are copied from the "kitchen sink" data generator to serve as defaults.
+var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
+var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
+
+// call DefineFlags before myflags.Parse()
+func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
+	fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
+	fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
+	fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
+	fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results")
+	fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+	fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *RandomQueryConfig) ValidateConfig() error {
+	if c.TreeDepth < 1 {
+		return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth)
+	}
+	if c.QueryCount < 0 {
+		return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
+	}
+	var err error
+	c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg)
+	if err != nil {
+		return fmt.Errorf("-time.from value couldn't be parsed: %w", err)
+	}
+	c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg)
+	if err != nil {
+		return fmt.Errorf("-time.to value couldn't be parsed: %w", err)
+	}
+	c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours())
+	if c.TimeRange < 1 {
+		return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
+			c.TimeToArg, c.TimeFromArg)
+	}
+	return nil
+}
+
+var ProgramName = "random-query"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
+	cfg := NewRandomQueryConfig()
+	cfg.DefineFlags(myflags)
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+
+	err = cfg.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *RandomQueryConfig) Run() (err error) {
+	remoteClient := nethttp.DefaultClient
+	cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	totalQ := 0
+	loops := 0
+	t0 := time.Now()
+
+	report := func() {
+		dur := time.Since(t0)
+		if dur > 0 {
+			qps := 1e9 * float64(totalQ) / float64(dur)
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: %0.02f", totalQ, dur, qps)
+		} else {
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: N/A", totalQ, dur)
+		}
+	}
+	defer report()
+
+NewSetup:
+	err = cfg.Setup(cli)
+	if err != nil {
+		return err
+	}
+
+	if len(cfg.IndexMap) == 0 {
+		return fmt.Errorf("no rows to query")
+	}
+
+	var indexes []string
+	for index := range cfg.IndexMap {
+		indexes = append(indexes, index)
+	}
+
+	for j := 0; ; j++ {
+		if cfg.QueryCount > 0 {
+			if j >= cfg.QueryCount {
+				break
+			}
+		} else {
+			// else keep doing queries forever...
+			if loops > 0 && loops%500 == 0 {
+				// ...but account for any new data arrived by getting
+				// the schema and rows again every so often.
+				loops++
+				goto NewSetup
+			}
+		}
+		if totalQ > 0 && totalQ%100 == 0 {
+			report()
+		}
+
+		index := indexes[cfg.Rnd.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		if cfg.Verbose {
+			fmt.Printf("pql = '%v'\n", pql)
+		}
+
+		// Query node0.
+		res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
+			return err
+		}
+		if cfg.VeryVerbose {
+			fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+		}
+		totalQ++
+		loops++
+
+	}
+
+	return nil
+}
+
+type Features struct {
+	Slc []IndexFieldRow
+	Ranges []IndexFieldRange
+	Distinctables []IndexFieldRange
+	SlcWeight int
+	RangeWeight int
+}
+
+// Pick either a feature entry or a random query on a range, weighted
+// by number of features and approximate weight of ranges
+func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight)
+	if r < f.SlcWeight {
+		return f.Slc[r].Query(cfg)
+	}
+	r = cfg.Rnd.Intn(len(f.Ranges))
+	return f.Ranges[r].Query(cfg)
+}
+
+func NewRandomQueryConfig() *RandomQueryConfig {
+	return &RandomQueryConfig{
+		IndexMap: make(map[string]*Features),
+	}
+}
+
+type IndexFieldRow struct {
+	Index    string
+	Field    string
+	RowID    uint64
+	RowKey   string
+	IsRowKey bool
+	HasTime  bool
+	IsInt    bool
+}
+
+func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
+	fromTo := ""
+	// 5% of queries on a time field will use the standard view
+	// anyway.
+	if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
+		startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
+		endHours := cfg.Rnd.Int63n(cfg.TimeRange - startHours) + 1 + startHours
+		startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
+		endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
+		fromTo = fmt.Sprintf(", from=%s, to=%s",
+			startTime.Format(pilosaTimeFmt),
+			endTime.Format(pilosaTimeFmt))
+	}
+	if fea.IsRowKey {
+		return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)}
+	}
+	return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
+}
+
+type IndexFieldRange struct {
+	Index string
+	Field string
+	Min, Max, Scale int64
+	ScaleDiv float64
+	Range uint64
+}
+
+// We want to pick one of (1) a single-operation filter, (2) a
+// between-filter of some kind.
+// So, that's one of <=, >=, ==, !=, >, <, or
+// one of [<, <], [<, <=], [<=, <=], [<=, <].
+var binaryOps = []string{
+	"<=", ">=", "==", "!=", "<", ">",
+}
+
+func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Int63n(10)
+	// this is unevenly weighted, but there's no Uint64N, and
+	// Int63n can't represent the whole range.
+	v1 := cfg.Rnd.Uint64() % i.Range
+	v2 := cfg.Rnd.Uint64() % i.Range
+	if v1 > v2 {
+		v1, v2 = v2, v1
+	}
+	v1 = v1 + uint64(i.Min)
+	v2 = v2 + uint64(i.Min)
+	var v1s, v2s string
+	if i.Scale != 0 {
+		v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1)) / i.ScaleDiv)
+		v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2)) / i.ScaleDiv)
+	} else {
+		v1s = strconv.FormatInt(int64(v1), 10)
+		v2s = strconv.FormatInt(int64(v2), 10)
+	}
+	if r < 4 {
+		lte := "<="
+		op1 := lte[:1+(r&1)]
+		op2 := lte[:1+((r>>1)&1)]
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)",
+			v1s, op1, i.Field, op2, v2s)}
+	} else {
+		if cfg.Rnd.Int63n(2) == 1 {
+			v1s = v2s
+		}
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r - 4], v1s)}
+	}
+}
+
+// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
+// and spits back a PQL query
+//
+func (cfg *RandomQueryConfig) Setup(api API) (err error) {
+	ctx := context.Background()
+	cfg.Info, err = api.Schema(ctx)
+	if err != nil {
+		return err
+	}
+	foundIntField := false
+	for i, ii := range cfg.Info {
+		_ = i
+		for k, fld := range ii.Fields {
+			_ = k
+			switch fld.Options.Type {
+			case "set", "mutex", "time":
+				pql := fmt.Sprintf("Rows(%v)", fld.Name)
+
+				res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
+				panicOn(err)
+				if cfg.VeryVerbose {
+					fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+				}
+				// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
+				// e.g.
+				// success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"}
+				// success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}'
+
+				switch x := res.Results[0].(type) {
+				case *pilosa.RowIdentifiers:
+					// internalClient gets this
+					cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time")
+				case pilosa.RowIdentifiers:
+					// test gets this
+					cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time")
+				}
+			case "int":
+				foundIntField = true
+				fallthrough // I bet you thought you'd never see this used
+			case "decimal":
+				// we'll ignore row keys and just use value ranges
+				cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
+			default:
+				AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
+			}
+		}
+	}
+
+	cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
+	if foundIntField {
+		cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
+	}
+	seed := int64(42)
+	cfg.Rnd = rand.New(rand.NewSource(seed))
+
+	return nil
+}
+
+func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) {
+	for _, rowID := range x.Rows {
+		cfg.AddFeature(index, field, rowID, "", false, hasTime)
+	}
+	for _, rowKey := range x.Keys {
+		cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
+	}
+}
+
+const maxEffectiveRange = 1000000
+
+func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) {
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	if min.Scale != scale || max.Scale != scale {
+		panic(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
+			min.Scale, max.Scale, scale))
+	}
+
+	effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
+	// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
+	// wraps to 0. Anything closer together will be fine. We accept the loss
+	// of accuracy in the range from not representing quite the full value
+	// in that edge case.
+	if effectiveRange == 0 {
+		effectiveRange--
+	}
+
+	// we assume that the Value of the field is already scaled, I guess?
+	newRange := IndexFieldRange{
+		Index:    index,
+		Field:    field,
+		Min:      min.Value,
+		Max:      max.Value,
+		Scale:    scale,
+		ScaleDiv: math.Pow(10, float64(scale)),
+		Range:    effectiveRange,
+	}
+	f.Ranges = append(f.Ranges, newRange)
+	if !decimal {
+		f.Distinctables = append(f.Distinctables, newRange)
+	}
+	// We want to add more values for larger int fields, but the
+	// default KitchenSink field has a range of 1<<64 which would make
+	// it completely dominate weights, so...
+	if effectiveRange > maxEffectiveRange {
+		effectiveRange = maxEffectiveRange
+	}
+	f.RangeWeight += int(effectiveRange)
+}
+
+func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
+
+	tree := cfg.GenTree(index, cfg.TreeDepth)
+
+	if cfg.Verbose {
+		fmt.Printf("%v\n", tree.StringIndent(0))
+	}
+	pql = tree.ToPQL()
+
+	// avoid using too much bandwidth, just count the final bitmap.
+	pql = fmt.Sprintf("Count(%v)", pql)
+	return
+}
+
+type Tree struct {
+	Chd []*Tree
+	S string
+	Args []string // Extra args to pass after children, such as a field for Distinct.
+}
+
+func (tr *Tree) StringIndent(ind int) (s string) {
+	spc := strings.Repeat("    ", ind)
+	spc1 := strings.Repeat("    ", ind+1)
+	var chds []string
+	leaf := true
+	if len(tr.Chd) == 0 {
+		// leaf
+	} else {
+		leaf = false
+		for _, chd := range tr.Chd {
+			chds = append(chds, chd.StringIndent(ind+1))
+		}
+	}
+	if leaf {
+		s += fmt.Sprintf("%v %v\n", spc1, tr.S)
+	} else {
+		for i, c := range chds {
+			if i == 0 {
+				s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c)
+			} else {
+				s += fmt.Sprintf("%v", c)
+			}
+		}
+	}
+	return
+}
+
+const pilosaTimeFmt = "2006-01-02T15:04"
+func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
+	features := cfg.IndexMap[index]
+	if depth == 0 {
+		return features.RandomQuery(cfg)
+	}
+
+	r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
+	f := cfg.BitmapFunc[r]
+	tr = &Tree{S: f}
+	numChild := 2
+	switch f {
+	case "Union", "Intersect", "Xor":
+		numChild = cfg.Rnd.Intn(8) + 2
+	case "Not":
+		numChild = 1
+	case "Difference":
+		numChild = 2
+	case "Distinct":
+		numChild = 1
+		// sometimes do a bare distinct without a filter
+		if cfg.Rnd.Intn(10) == 0 {
+			numChild = 0
+		}
+		r = cfg.Rnd.Intn(len(features.Distinctables))
+		tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field))
+	}
+	for i := 0; i < numChild; i++ {
+		tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
+	}
+	return
+}
+
+func (tr *Tree) ToPQL() (s string) {
+
+	if len(tr.Chd) == 0 {
+		// leaf
+		return tr.S
+	}
+
+	var chds []string
+	for _, c := range tr.Chd {
+		chds = append(chds, c.ToPQL())
+	}
+	// If we had no extra args, this does nothing.
+	chds = append(chds, tr.Args...)
+	all := strings.Join(chds, ", ")
+	return fmt.Sprintf("%v(%v)", tr.S, all)
+}
+
+func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) {
+
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	f.Slc = append(f.Slc, IndexFieldRow{
+		Index:    index,
+		Field:    field,
+		RowID:    rowID,
+		RowKey:   rowKey,
+		IsRowKey: isRowKey,
+		HasTime:  hasTime,
+	})
+	f.SlcWeight++
+}

cmd/random-query/main_test.go

@@ -0,0 +1,165 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"math/rand"
+	"strconv"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_RandomQuery(t *testing.T) {
+
+	cfg := NewRandomQueryConfig()
+
+	nNodes := 1
+	nReplicas := 1
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	defer c.Close()
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+	_ = dirs
+
+	ctx := context.Background()
+
+	indexes := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(indexes))
+	field := make([]*pilosa.Field, len(indexes))
+
+	var err error
+
+	for i := range indexes {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	timestamp := int64(0)
+
+	for i := range indexes {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, uint64(j))
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexes[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+		//vv("rowIDs = '%#v'", rowIDs)
+		//vv("colKeys = '%#v'", colKeys)
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+		//qcx.Reset()
+	}
+	// end of setup.
+
+	panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
+
+	for j := 0; j < 4; j++ {
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		//vv("pql = '%v'", pql)
+
+		// Query node0.
+		res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = res
+		//vv("success on pql = '%v'; res='%v'", pql, res.Results[0])
+	}
+}

cmd/random-query/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("# %s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) int64 {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return 0
+	}
+	return fi.Size()
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/rbf.go

@@ -0,0 +1,160 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newRBFCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "rbf",
+		Short: "Inspect RBF data files.",
+		Long: `
+Provides a set of commands for inspecting RBF data files.
+`,
+	}
+	cmd.AddCommand(newRBFCheckCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFDumpCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPagesCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPageCommand(stdin, stdout, stderr))
+	return cmd
+}
+
+func newRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFCheckCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "check [flags] PATH",
+		Short: "Run consistency check on RBF data.",
+		Long: `
+Executes a consistency check on an RBF data directory.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFDumpCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "dump [flags] PATH PGNO [PGNO...]",
+		Short: "Prints RBF raw page data",
+		Long: `
+Dumps the raw hex data for one or more RBF pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPagesCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "pages [flags] PATH",
+		Short: "Prints metadata for the list of all pages",
+		Long: `
+Prints a line for every page in the database with its type/status.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&c.WithTree, "with-tree", false, "Display b-tree name for each row")
+	return cmd
+}
+
+func newRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPageCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "page [flags] PATH PGNO [PGNO...]",
+		Short: "Prints data for a page(s)",
+		Long: `
+Prints the header & cell data for one or more pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}

cmd/root.go

@@ -26,23 +26,18 @@ import (
 )
 
 func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	productName := "Pilosa " + pilosa.Version
-	if pilosa.EnterpriseEnabled {
-		productName = "Pilosa Enterprise " + pilosa.Version
-	}
 	rc := &cobra.Command{
-		Use:   "pilosa",
+		Use: "pilosa",
+		// TODO: These short/long descriptions could use some updating.
 		Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
-		// TODO - is documentation actually there?
 		Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
+tools for administering Pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
 at https://www.pilosa.com/docs/.
 
-` + productName + `
-Build Time: ` + pilosa.BuildTime + "\n",
+` + pilosa.VersionInfo() + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
 			err := setAllConfig(v, cmd.Flags(), "PILOSA")
@@ -74,6 +69,7 @@ Build Time: ` + pilosa.BuildTime + "\n",
 	rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr))
 	rc.AddCommand(newImportCommand(stdin, stdout, stderr))
 	rc.AddCommand(newInspectCommand(stdin, stdout, stderr))
+	rc.AddCommand(newRBFCommand(stdin, stdout, stderr))
 	rc.AddCommand(newServeCmd(stdin, stdout, stderr))
 	rc.AddCommand(newHolderCmd(stdin, stdout, stderr))
 

cmd/root_test.go

@@ -55,6 +55,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +67,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()
@@ -145,6 +148,10 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
 		err = os.Setenv(name, val)
 		failErr(t, err, fmt.Sprintf("setting environment variable '%s' to '%s'", name, val))
 	}
+	// address common case where system might have postgres bind
+	// setting that an existing running Pilosa is using (causing test
+	// failures due to port conflict)
+	os.Setenv("PILOSA_POSTGRES_BIND", "")
 
 	// make command and set args
 	rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard)

cmd/server_test.go

@@ -15,6 +15,7 @@
 package cmd_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"strings"
 	"testing"
@@ -42,9 +43,10 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
+				"PILOSA_LONG_QUERY_TIME":         "1m30s",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
 				"PILOSA_MAX_WRITES_PER_REQUEST":  "2000",
 				"PILOSA_PROFILE_BLOCK_RATE":      "9123",
@@ -53,8 +55,10 @@ func TestServerConfig(t *testing.T) {
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	max-writes-per-request = 3000
-
+	long-query-time = "1m10s"
+	
 	[cluster]
 		disabled = true
 		replicas = 2
@@ -72,6 +76,7 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.Bind, "localhost:42454")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
 				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
@@ -96,6 +101,7 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		disabled = true
@@ -122,6 +128,7 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		hosts = [
@@ -163,28 +170,119 @@ func TestServerConfig(t *testing.T) {
 
 	// run server tests
 	for i, test := range tests {
-		com := test.setupCommand(t)
-		executed := make(chan struct{})
-		var execErr error
-		go func() {
-			execErr = com.Execute()
-			close(executed)
-		}()
-		select {
-		case <-cmd.Server.Started:
-		case <-executed:
-		}
-		if execErr != nil {
-			t.Fatalf("executing server command: %v", execErr)
-		}
-		err := cmd.Server.Close()
-		failErr(t, err, "closing pilosa server command")
-		<-executed
-		failErr(t, execErr, "executing command")
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			com := test.setupCommand(t)
+			executed := make(chan struct{})
+			var execErr error
+			go func() {
+				execErr = com.Execute()
+				close(executed)
+			}()
+			select {
+			case <-cmd.Server.Started:
+			case <-executed:
+			}
+			if execErr != nil {
+				t.Fatalf("executing server command: %v", execErr)
+			}
+			err := cmd.Server.Close()
+			failErr(t, err, "closing pilosa server command")
+			<-executed
+			failErr(t, execErr, "executing command")
 
-		if err := test.validation(); err != nil {
-			t.Fatalf("Failed test %d due to: %v", i, err)
-		}
-		test.reset()
+			if err := test.validation(); err != nil {
+				t.Fatalf("Failed test %d due to: %v", i, err)
+			}
+			test.reset()
+		})
+	}
+}
+func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
+	actualDataDir, err := ioutil.TempDir("", "")
+	failErr(t, err, "making data dir")
+
+	tests := []commandTest{
+		// TEST 0
+		{
+			args: []string{"server", "--long-query-time", "1m10s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = "localhost:0"
+            	bind-grpc = "localhost:0"
+             	data-dir = "` + actualDataDir + `"
+                [gossip]
+                  port = "14321"
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*70))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*70))
+				return v.Error()
+			},
+		},
+		// TEST 1
+		{
+			args: []string{"server", "--cluster.long-query-time", "1m20s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = "localhost:0"
+            	bind-grpc = "localhost:0"
+                [gossip]
+                  port = "14321"
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*80))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*80))
+				return v.Error()
+			},
+		},
+		// TEST 2: Use old value if both are provided because it is the simplest implementation
+		{
+			args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = "localhost:0"
+            	bind-grpc = "localhost:0"
+                [gossip]
+                  port = "14321"
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*50))
+				v.Check(toml.Duration(cmd.Server.Config.Cluster.LongQueryTime), toml.Duration(time.Second*90))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*90))
+				return v.Error()
+			},
+		},
+	}
+
+	// run server tests
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			com := test.setupCommand(t)
+			executed := make(chan struct{})
+			var execErr error
+			go func() {
+				execErr = com.Execute()
+				close(executed)
+			}()
+			select {
+			case <-cmd.Server.Started:
+			case <-executed:
+			}
+			if execErr != nil {
+				t.Fatalf("executing server command: %v", execErr)
+			}
+			err := cmd.Server.Close()
+			failErr(t, err, "closing pilosa server command")
+			<-executed
+			failErr(t, execErr, "executing command")
+
+			if err := test.validation(); err != nil {
+				t.Fatalf("Failed test %d due to: %v", i, err)
+			}
+			test.reset()
+		})
 	}
 }

cmd/slurp/slurp.go

@@ -0,0 +1,261 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+)
+
+// slurp: slurp is a load-tester for importing bulk data.
+//        It allows us to measure write performance.
+
+type stateMachine struct {
+	viewData  map[string][]byte
+	lastIndex string
+	lastField string
+	lastShard uint64
+	state     string
+	client    *http.InternalClient
+	start     time.Time
+
+	profile string
+	host    string
+}
+
+func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
+	parts := strings.Split(h.Name, "/")
+	switch parts[0] {
+	case "roaring":
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		panicOn(err)
+		if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return err
+		}
+		if _, already := r.viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		r.viewData[view] = roaringData
+		r.lastIndex = index
+		r.lastField = field
+		r.lastShard = shard
+	case "bolt":
+		if r.state == "roaring" {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+			vv("Finished import %v", time.Since(r.start))
+
+			if r.profile != "" {
+				stopProfile(r.host, r.profile)
+			}
+		}
+		//
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+
+		switch v := parts[len(parts)-1]; v {
+		case "keys":
+			index := parts[1]
+			fieldName := parts[2]
+			if fieldName == "_keys" {
+				//skip index keys are not not real fields so will have no need for field keys
+				return nil
+
+			}
+
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
+			if err != nil {
+				return err
+			}
+		default:
+			pilosa.VV("%v", h.Name)
+			index := parts[1]
+			partition, err := strconv.ParseUint(v, 10, 64)
+			if err != nil {
+				return err
+			}
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	r.state = parts[0]
+	return nil
+}
+func (r *stateMachine) Upload() error {
+	if len(r.viewData) > 0 {
+		request := &pilosa.ImportRoaringRequest{
+			Views: r.viewData,
+		}
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+		err := r.client.ImportRoaring(context.Background(), uri, r.lastIndex, r.lastField, r.lastShard, false, request)
+		if err != nil {
+			return err
+		}
+		r.viewData = make(map[string][]byte)
+	}
+	return nil
+}
+
+func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error {
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	runner := &stateMachine{
+		viewData: make(map[string][]byte),
+		start:    time.Now(),
+		profile:  profile,
+		host:     host,
+	}
+	runner.client = client
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			_ = runner.Upload()
+			break
+		}
+		if err != nil {
+			panicOn(err)
+		}
+		err = runner.NewHeader(header, tarReader)
+		panicOn(err)
+	}
+	return nil
+}
+
+func main() {
+	var host string
+	var profile string
+	var tarSrcPath string
+	flag.StringVar(&host, "host", "127.0.0.1:10101", "host to import into")
+	flag.StringVar(&profile, "profile", "", "profile and save a cpu profile of the import to this file")
+	flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import")
+	flag.Parse()
+
+	uri, err := pilosa.NewURIFromAddress(host)
+	panicOn(err)
+	globURI = uri
+
+	h := &gohttp.Client{}
+	if profile != "" {
+		startProfile(host)
+	}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	t0 := time.Now()
+	println("uploading", tarSrcPath)
+	panicOn(UploadTar(tarSrcPath, c, profile, host))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+func startProfile(host string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/start",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+}
+
+func stopProfile(host, outfile string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/stop",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+
+	fd, err := os.Create(outfile)
+	panicOn(err)
+	defer fd.Close()
+	_, err = io.Copy(fd, resp.Body)
+	panicOn(err)
+
+}
+
+var globURI *pilosa.URI
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
+	return globURI
+}

cmd/slurp/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

const_amd64.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,11 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-// +build enterprise
 
-package server
+// +build amd64
 
-import (
-	_ "github.com/pilosa/pilosa/v2/enterprise"
-)
+package pilosa
+
+const TxInitialMmapSize = 4 << 30 // 4GB

const_other.go

@@ -0,0 +1,21 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !amd64
+
+package pilosa
+
+// this is a stubbed out file to let 386/arm build.
+
+const TxInitialMmapSize = 1 << 30 // 1GB

ctl/common.go

@@ -31,12 +31,12 @@ type CommandWithTLSSupport interface {
 }
 
 // SetTLSConfig creates common TLS flags
-func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
-	flags.StringVarP(certificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
-	flags.StringVarP(certificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
-	flags.StringVarP(caCertPath, "tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
-	flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
-	flags.BoolVarP(enableClientVerification, "tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
+func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
+	flags.StringVarP(certificatePath, prefix+"tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
+	flags.StringVarP(certificateKeyPath, prefix+"tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
+	flags.StringVarP(caCertPath, prefix+"tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
+	flags.BoolVarP(skipVerify, prefix+"tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
+	flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
 }
 
 // commandClient returns a pilosa.InternalHTTPClient for the command

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
-				return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-			}
+			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
 
-		// Parse FieldValue.
-		value, err := strconv.ParseInt(record[1], 10, 64)
-		if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
-		}
-		val.Value = value
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
 			}
-			a = a[:0]
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
+			value, err := strconv.ParseInt(record[1], 10, 64)
+			if err != nil {
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
+			}
+			req.Values = append(req.Values, value)
+		}
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
@@ -72,7 +73,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -101,7 +102,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -134,7 +135,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -176,7 +177,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -218,7 +219,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -270,8 +271,8 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	c := test.MustRunCluster(t, 2)
 	defer c.Close()
-	cmd0 := c[0]
-	cmd1 := c[1]
+	cmd0 := c.GetNode(0)
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
-		qry := "Count(Row(f=foo0))"
-		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
-		if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
-		}
+		if err := test.RetryUntil(2*time.Second, func() error {
+			qry := "Count(Row(f=foo0))"
+			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
+			if err != nil {
+				return fmt.Errorf("Querying data for validation: %s", err)
+			}
 
-		// Read body and unmarshal response.
-		exp := `{"results":[100]}` + "\n"
-		if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
-		} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+			// Read body and unmarshal response.
+			exp := `{"results":[100]}` + "\n"
+			if body, err := ioutil.ReadAll(resp.Body); err != nil {
+				return fmt.Errorf("reading: %s", err)
+			} else if !reflect.DeepEqual(body, []byte(exp)) {
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.
@@ -332,7 +339,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -358,7 +365,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -446,7 +453,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -522,7 +529,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))

ctl/inspect.go

@@ -16,15 +16,23 @@ package ctl
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
+	"hash/fnv"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
-	"unsafe"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -33,6 +41,12 @@ import (
 type InspectCommand struct {
 	// Path to data file
 	Path string
+	// don't list details of objects
+	Quiet bool
+	// list only this many objects
+	Max int
+	// Filters:
+	InspectOpts pilosa.InspectRequest
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
+type pointerContext struct {
+	from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(cmd.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+		if cmd.Max != 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(cmd.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+		if cmd.Max > 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
 // Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
 	if err != nil {
@@ -58,7 +183,172 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "statting file")
 	}
+	if fi.IsDir() {
+		total := 0
+		infos, err := f.Readdir(0)
+		if err != nil {
+			return err
+		}
+		if len(infos) == 0 {
+			return errors.New("directory contains no files")
+		}
 
+		names := make([]string, len(infos))
+		nameToInfo := make(map[string]os.FileInfo, len(infos))
+		// find numeric-only names; we'll operate on
+		// either those, or the whole holder if we find
+		// a .topology file.
+		n := 0
+		for _, fi := range infos {
+			name := fi.Name()
+			if name == ".topology" {
+				return cmd.InspectHolder(ctx, cmd.Path)
+			}
+			if _, err := strconv.Atoi(name); err == nil {
+				names[n] = name
+				nameToInfo[name] = fi
+				n++
+			}
+		}
+		if n == 0 {
+			return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+		}
+		names = names[:n]
+		fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+		for _, name := range names {
+			f2, err := os.Open(filepath.Join(cmd.Path, name))
+			if err != nil {
+				return fmt.Errorf("opening %q: %v", name, err)
+			}
+			fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+			err = cmd.InspectFile(f2, nameToInfo[name])
+			total++
+			f2.Close()
+			if err != nil {
+				return fmt.Errorf("inspecting %q: %v", name, err)
+			}
+		}
+		return nil
+	}
+	return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology internal.Topology, myID string, err error) {
+	buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, errors.Wrap(err, "reading file")
+	}
+	if err := proto.Unmarshal(buf, &topology); err != nil {
+		return topology, myID, errors.Wrap(err, "unmarshalling")
+	}
+	sort.Slice(topology.NodeIDs,
+		func(i, j int) bool {
+			return topology.NodeIDs[i] < topology.NodeIDs[j]
+		})
+	buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, nil
+	}
+	myID = strings.TrimSpace(string(buf))
+	return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+	var shardMap map[uint64]int
+	var ok bool
+	if shardMap, ok = partitions[index]; !ok {
+		shardMap = make(map[uint64]int)
+		partitions[index] = shardMap
+	}
+	if partition, ok = shardMap[shard]; !ok {
+		var buf [8]byte
+		binary.BigEndian.PutUint64(buf[:], shard)
+
+		// Hash the bytes and mod by partition count.
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write(buf[:])
+		partition = int(h.Sum64() % uint64(partitionN))
+		shardMap[shard] = partition
+	}
+	return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+	parts := strings.Split(path, "/")
+	shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+	holder := pilosa.NewHolder(path, nil)
+	holder.Opts.Inspect = true
+	holder.Opts.ReadOnly = true
+	err := holder.Open()
+	if err != nil {
+		return fmt.Errorf("%s: holder open: %v", path, err)
+	}
+	holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+	if err != nil {
+		return fmt.Errorf("%s: inspect: %v", path, err)
+	}
+	myPartition := 0
+	topology, myID, err := loadTopology(path)
+	if err == nil {
+		fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+		if len(topology.NodeIDs) > 1 {
+			fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+		}
+		found := false
+		for i := range topology.NodeIDs {
+			if topology.NodeIDs[i] == myID {
+				found = true
+				myPartition = i
+				break
+			}
+		}
+		if !found {
+			fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+		}
+	} else {
+		fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+	}
+	for _, name := range holderInfo.FragmentNames {
+		partition, err := findPartitionPath(name, len(topology.NodeIDs))
+		if err != nil {
+			fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+		} else {
+			if partition == myPartition {
+				fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+			} else {
+				fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+			}
+		}
+		details := holderInfo.FragmentInfo[name]
+		cmd.DisplayInfo(details.BitmapInfo)
+		if details.BlockChecksums != nil {
+			fmt.Fprintf(cmd.Stdout, "  Checksums [%d total]:\n", len(details.BlockChecksums))
+			for _, block := range details.BlockChecksums {
+				fmt.Fprintf(cmd.Stdout, "   %8d: %x\n", block.ID, block.Checksum)
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
 	// Memory map the file.
 	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
@@ -72,39 +362,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	}()
 	// Attach the mmap file to the bitmap.
 	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
-	bm := roaring.NewBitmap()
-	if err := bm.UnmarshalBinary(data); err != nil {
-		return errors.Wrap(err, "unmarshalling")
+	fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
+	var info roaring.BitmapInfo
+	_, _, err = roaring.InspectBinary(data, true, &info)
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	cmd.DisplayInfo(info)
+	if err != nil {
+		return errors.Wrap(err, "inspecting")
 	}
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	return nil
+}
 
-	// Retrieve stats.
-	t = time.Now()
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
-	info := bm.Info()
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
+	pC := pointerContext{
+		from: info.From,
+		to:   info.To,
+	}
 
 	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintf(cmd.Stdout, "  Bitmap Info:\n")
+	fmt.Fprintf(cmd.Stdout, "    Bits: %d\n", info.BitCount)
+	fmt.Fprintf(cmd.Stdout, "    Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Operations: %d (%d bits)\n", info.Ops, info.OpN)
 	fmt.Fprintln(cmd.Stdout, "")
 
 	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
-	for _, ci := range info.Containers {
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
-			ci.Key,
-			ci.Type,
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
+	if !cmd.Quiet {
+		if info.ContainerCount > 0 {
+			cmd.PrintContainers(info, pC)
+		}
+		if info.Ops > 0 {
+			cmd.PrintOps(info)
+		}
 	}
-	tw.Flush()
-
-	return nil
 }

ctl/inspect_test.go

@@ -41,7 +41,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	file.Close()
 	cm.Path = file.Name()
 	err = cm.Run(context.Background())
-	expectedError := "unmarshalling: "
+	expectedError := "inspecting: "
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Fatalf("expected error '%s', got '%v'", expectedError, err)
 	}
@@ -52,7 +52,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	if err != nil {
 		t.Fatalf("copying data: %v", err)
 	}
-	if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+	if !strings.Contains(buf.String(), "inspecting bitmap...") {
 		t.Fatalf("Inspect doesn't work: %s", err)
 	}
 

ctl/rbf_check.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFCheckCommand represents a command for running a consistency check on RBF.
+type RBFCheckCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFCheckCommand returns a new instance of RBFCheckCommand.
+func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckCommand {
+	return &RBFCheckCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Run check on the database.
+	if err := db.Check(); err != nil {
+		return err
+	}
+
+	// If successful, print a success message.
+	fmt.Fprintln(cmd.Stdout, "ok")
+
+	return nil
+}

ctl/rbf_dump.go

@@ -0,0 +1,111 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFDumpCommand represents a command for dumping raw data for an RBF page.
+type RBFDumpCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFDumpCommand returns a new instance of RBFDumpCommand.
+func NewRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFDumpCommand {
+	return &RBFDumpCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFDumpCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch each page & dump.
+	for _, pgno := range cmd.Pgnos {
+		buf, err := tx.PageData(pgno)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(cmd.Stdout, "## PAGE %d\n", pgno)
+		fmt.Fprintln(cmd.Stdout, compressedHexDump(buf))
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func compressedHexDump(b []byte) string {
+	const prefixN = len("00000000")
+
+	var output []string
+	var prev string
+	var ellipsis bool
+
+	lines := strings.Split(strings.TrimSpace(hex.Dump(b)), "\n")
+	for i, line := range lines {
+		// Add line to output if it is not repeating or the last line.
+		if i == 0 || i == len(lines)-1 || trimPrefixN(line, prefixN) != trimPrefixN(prev, prefixN) {
+			output = append(output, line)
+			prev, ellipsis = line, false
+			continue
+		}
+
+		// Add an ellipsis for the first duplicate line.
+		if !ellipsis {
+			output = append(output, "...")
+			ellipsis = true
+			continue
+		}
+	}
+
+	return strings.Join(output, "\n")
+}
+
+// trimPrefixN trims n bytes from the beginning of a string.
+func trimPrefixN(s string, n int) string {
+	if len(s) < n {
+		return ""
+	}
+	return s[n:]
+}

ctl/rbf_page.go

@@ -0,0 +1,140 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFPageCommand represents a command for printing data for a single RBF page.
+type RBFPageCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPageCommand returns a new instance of RBFPageCommand.
+func NewRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPageCommand {
+	return &RBFPageCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPageCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch the page.
+	pages, err := tx.Pages(cmd.Pgnos)
+	if err != nil {
+		return err
+	}
+
+	for _, page := range pages {
+		switch page := page.(type) {
+		case *rbf.MetaPage:
+			cmd.printMetaPage(page)
+		case *rbf.RootRecordPage:
+			cmd.printRootRecordPage(page)
+		case *rbf.LeafPage:
+			cmd.printLeafPage(page)
+		case *rbf.BranchPage:
+			cmd.printBranchPage(page)
+		case *rbf.BitmapPage:
+			cmd.printBitmapPage(page)
+		case *rbf.FreePage:
+			cmd.printFreePage(page)
+		default:
+			return fmt.Errorf("unexpected page type %T", page)
+		}
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func (cmd *RBFPageCommand) printMetaPage(page *rbf.MetaPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: meta\n")
+	fmt.Fprintf(cmd.Stdout, "PageN: %d\n", page.PageN)
+	fmt.Fprintf(cmd.Stdout, "WALID: %d\n", page.WALID)
+	fmt.Fprintf(cmd.Stdout, "Root Record Pgno: %d\n", page.RootRecordPageNo)
+	fmt.Fprintf(cmd.Stdout, "Freelist Pgno: %d\n", page.FreelistPageNo)
+}
+
+func (cmd *RBFPageCommand) printRootRecordPage(page *rbf.RootRecordPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: root record\n")
+	fmt.Fprintf(cmd.Stdout, "Next: %d\n", page.Next)
+	fmt.Fprintf(cmd.Stdout, "Records: n=%d\n", len(page.Records))
+	for i, rec := range page.Records {
+		fmt.Fprintf(cmd.Stdout, "[%d]: name=%q pgno=%d\n", i, rec.Name, rec.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printLeafPage(page *rbf.LeafPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: leaf\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		if cell.Type == rbf.ContainerTypeBitmapPtr {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s pgno=%d\n", i, cell.Key, cell.Type, cell.Pgno)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s values=%v\n", i, cell.Key, cell.Type, cell.Values)
+		}
+	}
+}
+
+func (cmd *RBFPageCommand) printBranchPage(page *rbf.BranchPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: branch\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		fmt.Fprintf(cmd.Stdout, "[%d]: key=%d flags=%d pgno=%d\n", i, cell.Key, cell.Flags, cell.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printBitmapPage(page *rbf.BitmapPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: bitmap\n")
+	fmt.Fprintf(cmd.Stdout, "Values: %v\n", page.Values)
+}
+
+func (cmd *RBFPageCommand) printFreePage(page *rbf.FreePage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: free\n")
+}

ctl/rbf_pages.go

@@ -0,0 +1,149 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/txkey"
+)
+
+// RBFPagesCommand represents a command for printing a list of RBF page metadata.
+type RBFPagesCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Print the b-tree key with each row.
+	WithTree bool
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPagesCommand returns a new instance of RBFPagesCommand.
+func NewRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPagesCommand {
+	return &RBFPagesCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Iterate over each page and grab info.
+	infos, err := tx.PageInfos()
+	if err != nil {
+		return err
+	}
+
+	// Write header.
+	fmt.Fprint(cmd.Stdout, "ID       ")
+	fmt.Fprint(cmd.Stdout, "TYPE       ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "TREE                           ")
+	}
+	fmt.Fprintln(cmd.Stdout, "EXTRA")
+
+	fmt.Fprint(cmd.Stdout, "======== ")
+	fmt.Fprint(cmd.Stdout, "========== ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "============================== ")
+	}
+	fmt.Fprintln(cmd.Stdout, "====================")
+
+	// Print one line for each page.
+	for pgno, info := range infos {
+		switch info := info.(type) {
+		case *rbf.MetaPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "meta")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo)
+
+		case *rbf.RootRecordPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "rootrec")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "next=%d\n", info.Next)
+
+		case *rbf.LeafPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "leaf")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BranchPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "branch")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BitmapPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "bitmap")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		case *rbf.FreePageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "free")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		default:
+			panic(fmt.Sprintf("unexpected page info type %T", info))
+		}
+	}
+
+	return nil
+}
+
+func prefixToString(s string) (ret string) {
+	defer func() {
+		if err := recover(); err != nil {
+			ret = s
+		}
+	}()
+	return txkey.PrefixToString([]byte(s))
+}

ctl/server.go

@@ -15,8 +15,10 @@
 package ctl
 
 import (
+	"fmt"
 	"time"
 
+	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/spf13/cobra"
 )
@@ -26,25 +28,30 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
+	flags.StringVar(&srv.Config.AdvertiseGRPC, "advertise-grpc", srv.Config.AdvertiseGRPC, "Address to advertise externally for gRPC.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
 	flags.Uint64Var(&srv.Config.MaxMapCount, "max-map-count", srv.Config.MaxMapCount, "Limits the maximum number of active mmaps. Pilosa will fall back to reading files once this is exhausted. Set below your system's vm.max_map_count.")
 	flags.Uint64Var(&srv.Config.MaxFileCount, "max-file-count", srv.Config.MaxFileCount, "Soft limit on the maximum number of fragment files Pilosa keeps open simultaneously.")
+	flags.DurationVarP((*time.Duration)(&srv.Config.LongQueryTime), "long-query-time", "", time.Duration(srv.Config.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
+	flags.IntVar(&srv.Config.QueryHistoryLength, "query-history-length", srv.Config.QueryHistoryLength, "Number of queries to remember in history.")
 
 	// TLS
-	SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
+	SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
 
 	// Handler
-	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).")
+	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
 
 	// Cluster
 	flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
 	flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
 	flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
 	flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
+	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Duration(srv.Config.Cluster.LongQueryTime), "RENAMED TO 'long-query-time': Duration that will trigger log and stat messages for slow queries.") // negative duration indicates invalid value because 0 is meaningful
+	flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.")
 
 	// Translation
 	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
@@ -70,17 +77,38 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
+	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
 	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
 	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
 	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
+	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
 	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling
 	flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
 	flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
+
+	// Transactional storage engine
+	// Note: the default for --tx must be kept "" empty string. Otherwise we
+	// cannot detect and honor the PILOSA_TXSRC env var over-ride.
+	flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --txsrc option on the command line.", pilosa.DefaultTxsrc))
+
+	// RowcacheOn
+	flags.BoolVarP((&srv.Config.RowcacheOn), "rowcache-on", "", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)")
+
+	// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.
+	srv.Config.RBFConfig.DefineFlags(flags)
+
+	// Postgres endpoint
+	flags.StringVar(&srv.Config.Postgres.Bind, "postgres.bind", srv.Config.Postgres.Bind, "Address to which to bind a postgres endpoint (leave blank to disable)")
+	SetTLSConfig(flags, "postgres.", &srv.Config.Postgres.TLS.CertificatePath, &srv.Config.Postgres.TLS.CertificateKeyPath, &srv.Config.Postgres.TLS.CACertPath, &srv.Config.Postgres.TLS.SkipVerify, &srv.Config.Postgres.TLS.EnableClientVerification)
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.StartupTimeout), "postgres.startup-timeout", time.Duration(srv.Config.Postgres.StartupTimeout), "Timeout for postgres connection startup. (set 0 to disable)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.ReadTimeout), "postgres.read-timeout", time.Duration(srv.Config.Postgres.ReadTimeout), "Timeout for reads on a postgres connection. (set 0 to disable; does not include connection idling)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
+	flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
+	flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
+
 }

ctl/server_test.go

@@ -35,3 +35,14 @@ func TestBuildServerFlags(t *testing.T) {
 		t.Fatal("log-path flag is required")
 	}
 }
+
+func TestServerDefaultTxsrcFlags(t *testing.T) {
+	cm := &cobra.Command{}
+	buf := bytes.Buffer{}
+	stdin, stdout, stderr := GetIO(buf)
+	Server := server.NewCommand(stdin, stdout, stderr)
+	BuildServerFlags(cm, Server)
+	if cm.Flags().Lookup("txsrc").DefValue != "" {
+		t.Fatal("cannot set the txsrc default in ctl/server.go, otherwise we won't know to let the environment override the lack of --txsrc on the command line. We want explicit command line --txsrc to override the env value.")
+	}
+}

dbshard_internal_test.go

@@ -0,0 +1,366 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/shardwidth"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	//txkey "github.com/pilosa/pilosa/v2/txkey"
+)
+
+// Shard per db evaluation
+func TestShardPerDB_SetBit(t *testing.T) {
+	f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
+	_ = idx
+	defer f.Clean(t)
+
+	// Set bits on the fragment.
+	if _, err := f.setBit(tx, 120, 1); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 120, 6); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 121, 0); err != nil {
+		t.Fatal(err)
+	}
+	// should have two containers set in the fragment.
+
+	// Verify counts on rows.
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count: %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count: %d", n)
+	}
+
+	// commit the change, and verify it is still there
+	panicOn(tx.Commit())
+
+	// Close and reopen the fragment & verify the data.
+	err := f.Reopen() // roaring data not being flushed? red on roaring
+	if err != nil {
+		t.Fatal(err)
+	}
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	defer tx.Rollback()
+
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	}
+}
+
+// test that we find all *local* shards
+func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
+
+	tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetShardsForIndex_LocalOnly")
+	panicOn(err)
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+
+	v2s := NewFieldView2Shards()
+	stdShardSet := newShardSet()
+	for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+		stdShardSet.add(shard)
+	}
+	for _, field := range []string{"f", "_exists"} {
+		v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+	}
+
+	for _, src := range []string{"roaring", "bolt", "rbf"} {
+
+		os.Setenv("PILOSA_TXSRC", src)
+
+		// must make Holder AFTER setting src.
+		holder := NewHolder(tmpdir, nil)
+
+		index := "rick"
+		idx := makeSampleRoaringDir(tmpdir, index, src, 1, holder, v2s)
+		if idx == nil {
+			idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
+			panicOn(err)
+		}
+		estd := "rick/_exists/views/standard"
+		std := "rick/f/views/standard"
+
+		shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
+		panicOn(err)
+
+		for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+			if !shards[shard] {
+				panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+			}
+		}
+		if src == "roaring" {
+			// check estd too
+			shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
+			panicOn(err)
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				if !shards[shard] {
+					panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+				}
+			}
+
+			// check GetSortedFieldViewList() and roaringGetFieldView2Shards()
+			vs, err := roaringGetFieldView2Shards(idx)
+			panicOn(err)
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				panicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+
+				for _, fv := range fvs {
+					if !vs.has(fv.Field, fv.View, shard) {
+						panic(fmt.Sprintf("vs did not contain fv='%#v' for shard %v", fv, shard))
+					}
+				}
+				tx.Rollback()
+			}
+		} else {
+			// non-roaring: rbf, bolt
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				panicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+				tx.Rollback()
+			}
+		}
+		holder.Close()
+	}
+}
+
+// data for Test_DBPerShard_GetShardsForIndex
+//
+var sampleRoaringDirList = map[string]string{"roaring": `
+rick/f/views/standard/fragments/215.cache
+rick/f/views/standard/fragments/221.cache
+rick/f/views/standard/fragments/223.cache
+rick/f/views/standard/fragments/93.cache
+rick/f/views/standard/fragments/217.cache
+rick/f/views/standard/fragments/219.cache
+rick/f/views/standard/fragments/217
+rick/f/views/standard/fragments/219
+rick/f/views/standard/fragments/215
+rick/f/views/standard/fragments/221
+rick/f/views/standard/fragments/223
+rick/f/views/standard/fragments/93
+rick/_exists/views/standard/fragments/221
+rick/_exists/views/standard/fragments/215
+rick/_exists/views/standard/fragments/217
+rick/_exists/views/standard/fragments/93
+rick/_exists/views/standard/fragments/219
+rick/_exists/views/standard/fragments/223
+`,
+	"bolt": `
+rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db
+rick.index.txstores@@@/store-boltdb@@/shard.0215-boltdb@/bolt.db
+rick.index.txstores@@@/store-boltdb@@/shard.0217-boltdb@/bolt.db
+rick.index.txstores@@@/store-boltdb@@/shard.0219-boltdb@/bolt.db
+rick.index.txstores@@@/store-boltdb@@/shard.0221-boltdb@/bolt.db
+rick.index.txstores@@@/store-boltdb@@/shard.0223-boltdb@/bolt.db
+`,
+	"rbf": `
+rick.index.txstores@@@/store-rbfdb@@/shard.0093-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0215-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0217-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0219-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0221-rbfdb@
+rick.index.txstores@@@/store-rbfdb@@/shard.0223-rbfdb@
+`,
+}
+
+func makeSampleRoaringDir(root, index, txsrc string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
+
+	shards := []uint64{0, 93, 215, 217, 219, 221, 223}
+	fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
+	firstDone := false
+
+	for i, fn := range fns {
+		if fn == "" {
+			continue
+		}
+		var shard uint64
+		if txsrc != "roaring" {
+			// only have shards for the non-roaring
+			shard = shards[i]
+		}
+		switch txsrc {
+		case "bolt", "rbf":
+			idx = helperCreateDBShard(h, index, shard)
+
+			// first time only, we'll actually make all the shards at this point because
+			// view2shards has them all anyway.
+			if !firstDone {
+				firstDone = true
+				makeTxTestDBWithViewsShards(h, idx, view2shards)
+			}
+			continue
+		}
+
+		path := root + sep + filepath.Dir(fn)
+		panicOn(os.MkdirAll(path, 0755))
+		fd, err := os.Create(root + sep + fn)
+		panicOn(err)
+		if minBytes > 0 {
+			_, err := fd.Write(make([]byte, minBytes))
+			panicOn(err)
+		}
+		fd.Close()
+	}
+	return
+}
+
+func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	panicOn(err)
+	dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
+	panicOn(err)
+	_ = dbs
+	return idx
+}
+
+// keep the ocd linter happy
+var _ = makeBolttestDB
+var _ = makeRBFtestDB
+
+func makeBolttestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+	w, _ := mustOpenEmptyBoltWrapper(path)
+	BoltMustSetBitvalue(w, "index", "field", "view", shard, i)
+	w.Close()
+}
+
+func makeRBFtestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+
+	db := rbf.NewDB(path, nil)
+	err := db.Open()
+	panicOn(err)
+	defer db.Close()
+
+	tx, err := db.Begin(true)
+	panicOn(err)
+
+	err = tx.CreateBitmap("x")
+	panicOn(err)
+
+	_, err = tx.Add("x", i)
+	panicOn(err)
+
+	err = tx.Commit()
+	panicOn(err)
+}
+
+func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shards) {
+
+	// TODO(jea): need date time quantum views!!
+	batched := false
+	for field, viewmap := range exp.m {
+		for view, shset := range viewmap {
+
+			ss := shset.CloneMaybe()
+			for shard := range ss {
+
+				// simply write 1 bit to each shard to force its creation.
+				bits := []uint64{(shard << shardwidth.Exponent) + 1}
+				tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+				changeCount, err := tx.Add(idx.name, field, view, shard, batched, bits...)
+				panicOn(err)
+				if changeCount != len(bits) {
+					panic(fmt.Sprintf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount))
+				}
+
+				panicOn(tx.Commit())
+			}
+		}
+	}
+
+}
+
+// test that rbf can give us a map[view]*shardSet
+func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetFieldView2Shards_map_from_RBF")
+	panicOn(err)
+
+	orig := os.Getenv("PILOSA_TXSRC")
+	defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
+
+	os.Setenv("PILOSA_TXSRC", "rbf")
+
+	// must make Holder AFTER setting src.
+	holder := NewHolder(tmpdir, nil)
+	defer holder.Close()
+
+	index := "rick"
+	field := "f"
+	idx, err := holder.createIndex(index, IndexOptions{})
+	panicOn(err)
+
+	exp := NewFieldView2Shards()
+
+	stdShardSet := newShardSet()
+	stdShardSet.add(12)
+	stdShardSet.add(15)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+
+	hrShardSet := newShardSet()
+	hrShardSet.add(7)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard_2019092416"}, hrShardSet)
+
+	makeTxTestDBWithViewsShards(holder, idx, exp)
+
+	// setup is done
+	view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx)
+	panicOn(err)
+
+	// compare against setup
+	if !view2shard.equals(exp) {
+		panic(fmt.Sprintf("expected '%v' but got view2shard '%v'", exp, view2shard))
+	}
+}

dbshard_test.go

@@ -0,0 +1,111 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+
+	t.Run("RowIDColumnKey", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "rick"
+		fieldName := "f"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+
+		rowID := uint64(1)
+		timestamp := int64(0)
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		for i := 1; i <= 10; i++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
+		}
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		//select {}
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("unexpected column keys: %#v", keys)
+		}
+	})
+
+}

debugstats/stats.go

@@ -0,0 +1,142 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debugstats
+
+import (
+	"fmt"
+	"math"
+	//"os"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+type CallStats struct {
+	// protect elap
+	mu sync.Mutex
+
+	// track how much time each call took.
+	elap map[string]*elapsed
+}
+
+type elapsed struct {
+	dur []float64
+}
+
+func NewCallStats() *CallStats {
+	w := &CallStats{}
+	w.Reset()
+	return w
+}
+
+func (w *CallStats) Reset() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	w.elap = make(map[string]*elapsed)
+}
+
+type LineSorter struct {
+	Line string
+	Tot  float64
+}
+
+type SortByTot []*LineSorter
+
+func (p SortByTot) Len() int {
+	return len(p)
+}
+func (p SortByTot) Less(i, j int) bool {
+	return p[i].Tot < p[j].Tot
+}
+func (p SortByTot) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (c *CallStats) Report(title string) (r string) {
+	//txsrc := os.Getenv("PILOSA_TXSRC")
+	r = fmt.Sprintf("CallStats: (%v)\n", title)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var lines []*LineSorter
+	for id, elap := range c.elap {
+		slc := elap.dur
+		n := len(slc)
+		if n == 0 {
+			continue
+		}
+		mean, sd, totaltm := computeMeanSd(slc)
+		if n == 1 {
+			sd = 0
+			mean = slc[0]
+			totaltm = slc[0]
+		}
+		line := fmt.Sprintf("  %20v  N=%8v   avg/op: %12v   sd: %12v  total: %12v\n", id, n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm))
+		lines = append(lines, &LineSorter{Line: line, Tot: totaltm})
+	}
+	sort.Sort(SortByTot(lines))
+	for i := range lines {
+		r += lines[i].Line
+	}
+
+	if false {
+		var m1 runtime.MemStats
+		runtime.ReadMemStats(&m1)
+		r += fmt.Sprintf("\n m1.TotalAlloc = %v\n", m1.TotalAlloc)
+	}
+
+	return
+}
+
+var NaN = math.NaN()
+
+func computeMeanSd(slc []float64) (mean, sd, tot float64) {
+	if len(slc) < 2 {
+		return NaN, NaN, NaN
+	}
+	for _, v := range slc {
+		tot += v
+	}
+	n := float64(len(slc))
+	mean = tot / n
+
+	variance := 0.0
+	for _, v := range slc {
+		tmp := (v - mean)
+		variance += tmp * tmp
+	}
+	variance = variance / n // biased, but we don't care b/c we can have very small n
+	sd = math.Sqrt(variance)
+	if sd < 1e-8 {
+		// sd is super close to zero, NaN out the z-score rather than +/- Inf
+		sd = NaN
+	}
+	return
+}
+
+func (c *CallStats) Add(k string, dur time.Duration) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.elap[k]
+	if !ok {
+		e = &elapsed{}
+		c.elap[k] = e
+	}
+	e.dur = append(e.dur, float64(dur))
+}

debugstats/stats_test.go

@@ -0,0 +1,54 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debugstats
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestCallStats(t *testing.T) {
+
+	callStats := NewCallStats()
+
+	for j := 0; j < 4; j++ {
+		t0 := time.Now()
+		doOperation0()
+		callStats.Add("op0", time.Since(t0))
+
+		t1 := time.Now()
+		doOperation1()
+		callStats.Add("op1", time.Since(t1))
+
+		t2 := time.Now()
+		doOperation2()
+		callStats.Add("op2", time.Since(t2))
+	}
+
+	fmt.Printf("report = \n%v\n", callStats.Report("test"))
+}
+
+func doOperation0() {
+	time.Sleep(50 * time.Millisecond)
+}
+
+func doOperation1() {
+	time.Sleep(100 * time.Millisecond)
+}
+
+func doOperation2() {
+	time.Sleep(200 * time.Millisecond)
+}

diagnostics.go

@@ -229,11 +229,11 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 	timeQuantumEnabled := false
 
 	for _, index := range d.server.holder.Indexes() {
-		numShards += index.AvailableShards().Count()
+		numShards += index.AvailableShards(includeRemote).Count()
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {
@@ -275,6 +275,7 @@ type SystemInfo interface {
 	CPUCores() (physical int, logical int, err error)
 	CPUMHz() (int, error)
 	CPUArch() string
+	DiskCapacity(string) (uint64, error)
 }
 
 // newNopSystemInfo creates a no-op implementation of SystemInfo.
@@ -345,3 +346,8 @@ func (n *nopSystemInfo) CPUMHz() (int, error) {
 func (n *nopSystemInfo) CPUCores() (physical, logical int, err error) {
 	return 0, 0, nil
 }
+
+// DiskCapacity returns the disk capacity
+func (n *nopSystemInfo) DiskCapacity(path string) (uint64, error) {
+	return 0, nil
+}

diagnostics_internal_test.go

@@ -29,6 +29,7 @@ import (
 func TestDiagnosticsClient(t *testing.T) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)
@@ -121,6 +122,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 			t.Fatalf("couldn't encode version response: %v", err)
 		}
 	}))
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector("localhost:10101")
@@ -158,6 +160,7 @@ func compareJSON(a, b []byte) (bool, error) {
 func BenchmarkDiagnostics(b *testing.B) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)

docs/api-reference.md

@@ -15,7 +15,7 @@ Is equivalent to `GET /schema` and returns the same response.
 
 ### List index schema
 
-`GET /index/<index-name>`
+`GET /index/{index-name}`
 
 Returns the schema of the specified index in JSON.
 
@@ -24,27 +24,31 @@ curl -XGET localhost:10101/index/user
 ```
 ``` response
 {
-    "fields": [
-        {
-            "name": "event",
-            "options": {
-                "keys": false,
-                "timeQuantum": "YMD",
-                "type": "time"
-            }
-        }
-    ],
-    "name": "user",
-    "options": {
-        "keys": false,
-        "trackExistence": true
+  "name": "user",
+  "createdAt": 1591178953061239000,
+  "options": {
+    "keys": false,
+    "trackExistence": true
+  },
+  "fields": [
+    {
+      "name": "event",
+      "createdAt": 1591178962332452000,
+      "options": {
+        "type": "set",
+        "cacheType": "ranked",
+        "cacheSize": 50000,
+        "keys": false
+      }
     }
+  ],
+  "shardWidth": 1048576
 }
 ```
 
 ### Create index
 
-`POST /index/<index-name>`
+`POST /index/{index-name}`
 
 Creates an index with the given name.
 
@@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option
 curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"user","createdAt":1591179042178854000}
 ```
 
 ### Remove index
@@ -75,7 +79,7 @@ curl -XDELETE localhost:10101/index/user
 
 ### Query index
 
-`POST /index/<index-name>/query`
+`POST /index/{index-name}/query`
 
 Sends a [query](../query-language/) to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
 
@@ -133,7 +137,7 @@ By default, all bits and attributes (*for `Row` queries only*) are returned. In
 
 ### Import Data
 
-`POST /index/<index-name>/field/<field-name>/import`
+`POST /index/{index-name}/field/{field-name}/import`
 
 Supports high-rate data ingest to a particular shard of a particular field. The
 official client libraries use this endpoint for their import functionality - it
@@ -151,23 +155,37 @@ represents a particular bit to be set. Timestamps are optional, but if they
 exist must also contain the same number of items as rows and columns. The
 column IDs must all be in the shard specified in the request.
 
+Some endpoints and data structures include a `CreatedAt` fields.
+This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
+but to serve as a unique identifier for use in cache invalidation.
+
+The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
+can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
+This is true except in cases where an index or field gets deleted and then recreated,
+or if Pilosa is restored from a backup.
+So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
+and the ingester will know that it needs to drop its cache.
+
 ```
 message ImportRequest {
-	string Index = 1;
-	string Field = 2;
-	uint64 Shard = 3;
-	repeated uint64 RowIDs = 4;
-	repeated uint64 ColumnIDs = 5;
-	repeated string RowKeys = 7;
-	repeated string ColumnKeys = 8;
-	repeated int64 Timestamps = 6;
+    string Index = 1;
+    string Field = 2;
+    uint64 Shard = 3;
+    repeated uint64 RowIDs = 4;
+    repeated uint64 ColumnIDs = 5;
+    repeated int64 Timestamps = 6;
+    repeated string RowKeys = 7;
+    repeated string ColumnKeys = 8;
+    int64 IndexCreatedAt = 9;
+    int64 FieldCreatedAt = 10;
 }
 ```
 
 
+
 ### Create field
 
-`POST /index/<index-name>/field/<field-name>`
+`POST /index/{index-name}/field/{field-name}`
 
 Creates a field in the given index with the given name.
 
@@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \
      -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"quantity","createdAt":1591180110914425000}
 ```
 
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
@@ -209,21 +227,21 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit,
 curl localhost:10101/index/user/field/language -X POST
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"language","createdAt":1591180128294321000}
 ```
 
 ``` request
 curl localhost:10101/index/repository/field/stats \
     -X POST \
-    -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
+    -d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"stats","createdAt":1591180737881627000}
 ```
 
 ### Remove field
 
-`DELETE /index/<index-name>/field/<field-name>`
+`DELETE /index/{index-name}/field/{field-name}`
 
 Removes the given field.
 
@@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema
 ```
 ``` response
 {
-    "indexes": [
+  "indexes": [
+    {
+      "name": "user",
+      "createdAt": 1591178953061239000,
+      "options": {
+        "keys": false,
+        "trackExistence": true
+      },
+      "fields": [
         {
-            "fields": [
-                {
-                    "name": "event",
-                    "options": {
-                        "keys": false,
-                        "timeQuantum": "YMD",
-                        "type": "time"
-                    }
-                },
-                {
-                    "name": "language",
-                    "options": {
-                        "cacheSize": 50000,
-                        "cacheType": "ranked",
-                        "keys": false,
-                        "type": "set"
-                    }
-                }
-            ],
-            "name": "user",
-            "options": {
-                "keys": false,
-                "trackExistence": true
-            }
+          "name": "event",
+          "createdAt": 1591178962332452000,
+          "options": {
+            "type": "set",
+            "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": false
+          }
+        },
+        {
+          "name": "language",
+          "createdAt": 1591180128294321000,
+          "options": {
+            "type": "set",
+            "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": false
+          }
+        },
+        {
+          "name": "quantity",
+          "createdAt": 1591180110914425000,
+          "options": {
+            "type": "int",
+            "base": 0,
+            "bitDepth": 0,
+            "min": -1000,
+            "max": 2000,
+            "keys": false,
+            "foreignIndex": ""
+          }
         }
-    ]
+      ],
+      "shardWidth": 1048576
+    }
+  ]
 }
 ```
 
@@ -304,7 +340,7 @@ Returns the version of the Pilosa server.
 curl -XGET localhost:10101/version
 ```
 ``` response
-{"version":"v0.6.0"}
+{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
 ```
 
 ### Get status
@@ -318,22 +354,49 @@ curl -XGET localhost:10101/status
 ```
 ```response
 {
-    "localID": "d3369125-29d8-4305-a351-b4474d14a542",
-    "nodes": [
-        {
-            "id": "d3369125-29d8-4305-a351-b4474d14a542",
-            "isCoordinator": true,
-            "uri": {
-                "host": "localhost",
-                "port": 10101,
-                "scheme": "http"
-            }
-        }
-    ],
-    "state": "NORMAL"
+  "state": "NORMAL",
+  "nodes": [
+    {
+      "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
+      "uri": {
+        "scheme": "http",
+        "host": "localhost",
+        "port": 10101
+      },
+      "grpc-uri": {
+        "scheme": "http",
+        "host": "localhost",
+        "port": 20101
+      },
+      "isCoordinator": true,
+      "state": "READY"
+    }
+  ],
+  "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
 }
 ```
 
+### Get active queries
+
+`GET /queries`
+
+Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format.
+Also includes the amount of time that the query has been running (in nanoseconds when using JSON).
+
+```request
+curl -XGET localhost:10101/queries
+```
+```response
+182.412µs  All()
+```
+
+```request
+curl -XGET -H "Accept: application/json" localhost:10101/queries
+```
+```response
+[{"query":"All()","age":135123}]
+```
+
 ### Recalculate Caches
 
 `POST /recalculate-caches`

docs/configuration.md

@@ -114,6 +114,16 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
     ```toml
     verbose = true
     ```
+#### Long Query Time
+
+* Description: Duration that will trigger log and stat messages for slow queries.
+* Flag: `long-query-time="1m0s"`
+* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
+* Config:
+
+    ```toml
+    long-query-time = "1m0s"
+    ```
 
 #### Max Map Count
 
@@ -220,6 +230,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
       key = "/var/secret/gossip.key32"
     ```
 
+#### Cluster Long Query Time
+
+* Description (DEPRICATED, see Long Query Time): Duration that will trigger log and stat messages for slow queries.
+* Flag: `cluster.long-query-time="1m0s"`
+* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
+* Config:
+
+    ```toml
+    [cluster]
+    long-query-time = "1m0s"
+    ```
+
 #### Cluster Coordinator
 
 * Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
@@ -232,18 +254,6 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
     coordinator = true
     ```
 
-#### Cluster Long Query Time
-
-* Description: Duration that will trigger log and stat messages for slow queries.
-* Flag: `cluster.long-query-time="1m0s"`
-* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
-* Config:
-
-    ```toml
-    [cluster]
-    long-query-time = "1m0s"
-    ```
-
 #### Cluster Replicas
 
 * Description: Number of hosts each piece of data should be stored on. 
@@ -302,7 +312,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
     ```toml
     [metric]
-    service = “statsd”
+    service = "statsd"
     ```
 
 #### Metric Host
@@ -319,7 +329,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 #### Metric Poll Interval
 
 * Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
-* Flag: `metric.poll-interval=”0m15s”`
+* Flag: `metric.poll-interval="0m15s"`
 * Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
 * Config:
 
@@ -403,7 +413,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Tracing Sampler Type
 
-* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely.
+* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely. Default is 'off'.
 * Flag: `tracing.sampler-type`
 * Env: `PILOSA_TRACING_SAMPLER_TYPE`
 * Config:

docs/data-model.md

@@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
 #### Set
 
 Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
+Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
+```go
+[]byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+```
+(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
+In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
 
 ``` request
 curl localhost:10101/index/repository/field/info \

docs/examples.md

@@ -81,7 +81,7 @@ lfm := pdk.LinearFloatMapper{
 }
 ```
 
-`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
+`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a "round to nearest integer" behavior. Other predefined mappers have their own specific parameters, usually two or three.
 
 This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). <!-- TODO update so this makes sense -->
 ```go

docs/getting-started.md

@@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
 ``` response
 {"success":true}
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
@@ -325,7 +325,7 @@ Next, let's create the `repository` index:
 	repository := schema.Index("repository")
 ```
 
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -615,7 +615,7 @@ Next, let's create the `repository` index:
 ```
 	Index repository = schema.index("repository");
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -818,7 +818,7 @@ Next, let's create the `repository` index:
 ```
 repository = schema.index("repository")
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```

docs/glossary.md

@@ -58,7 +58,7 @@ nav = []
 
 <strong id="range-bsi">[Row (BSI)](../query-language/#row-bsi):</strong> A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
 
-<strong id="rows">[Rows](../query-language/#rows):<strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
+<strong id="rows">[Rows](../query-language/#rows):</strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
 
 <strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
 

docs/query-language.md

@@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
 * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
 * `UINT` An unsigned integer (e.g. 42839).
 * `BOOL` A boolean value, `true` or `false`.
@@ -52,6 +52,7 @@ curl localhost:10101/index/repository/query \
 * `CALL` Any query.
 * `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not`.
 * `ROWS_CALL` A query that returns a `Rows` result (i.e. a list of row IDs). Currently only the `Rows` query.
+* `ROWSET_CALL` A query that returns a set of rows. Currently only the `Rows` and `TopN` queries.
 * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`).
 
 ### Write Operations
@@ -428,9 +429,9 @@ Union([ROW_CALL ...])
 
 **Description:**
 
-Union performs a logical OR on the results of all `ROW_CALL` queries passed to it.
+Union performs a set union on the column indexes in the results of all `ROW_CALL` queries passed to it. In comparison to a relational query, this is similar to combining clauses in the "OR" sense.
 
-**Result Type:** object with attrs and bits
+**Result Type:** object with attrs and columns
 
 attrs will always be empty
 
@@ -456,7 +457,7 @@ Intersect(<ROW_CALL>, [ROW_CALL ...])
 
 **Description:**
 
-Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it.
+Intersect performs a set intersection on the column indexes in the results of all `ROW_CALL` queries passed to it. In comparison to a relational query, this is similar to combining clauses in the "AND" sense.
 
 **Result Type:** object with attrs and columns
 
@@ -570,6 +571,38 @@ Not(Row(stargazer=1))
 
 * columns are repositories that were not starred by user 1
 
+#### Limit
+
+**Spec:**
+
+```
+Limit(<ROW_CALL>, [limit=<UINT>], [offset=<UINT>])
+```
+
+**Description:**
+
+Limit executes a `ROW_CALL` and returns a subset of the results.
+If a limit of `n` is specified, then this query will return the first `n` results of the row call.
+If an offset of `m` is specified, then this query will skip the first `m` results of the row call.
+If both a limit and offset are specified, the offset is applied before the limit.
+This can be used to implement pagination.
+
+**Result Type:** object with attrs and columns
+
+attrs will always be empty
+
+**Examples:**
+
+Find the second column that has a bit set in the given row.
+```request
+Limit(Row(stargazer=1), limit=1, offset=1)
+```
+```response
+{"results":[{"attrs":{},"columns":[30]}]}
+```
+
+* columns are repositories that were starred by user 1
+
 #### Count
 **Spec:**
 
@@ -595,34 +628,6 @@ Count(Row(stargazer=1))
 
 * Result is the number of repositories that user 1 has starred.
 
-#### Shift
-**Spec:**
-
-```
-Shift(<ROW_CALL>, [n=UINT])
-```
-
-**Description:**
-
-Returns the row specified by `ROW_CALL` shifted by `n` bits.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of the field `stargazer`
-and shift the result by 2:
-```request
-Shift(Row(stargazer=1), n=2)
-```
-```response
-{"attrs":{},"columns":[12, 22]}
-```
-
-* columns are the repositories which user 1 has starred shifted by 2 bits.
-
 #### TopN
 
 **Spec:**
@@ -643,9 +648,15 @@ have the attribute specified by `attrName` with one of the values specified in
 
 **Caveats:**
 
-* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order.
-* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
-* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
+In general, the order of the resulting row keys is not guaranteed to reflect the true order of bit counts across an index. The exact solution to the problem of computing the TopN counts is prohibitively expensive, so TopN is instead implemented as a heuristic. This provides a significant performance improvement, at the cost of uncertainty in the result order.
+
+The implementation is based on a per-shard cache. The accuracy of the results depends on how well the counts for the overall index are reflected in the individual shards (so TopN queries on a single-shard index are exact). If the distribution of bits across shards is uniform, shard counts are representative. This is often a reasonable assumption, especially for the top results for large data sets, in which counts might follow Zipfian, exponential, or other long-tail distributions. However, this assumption may not hold for some applications.
+
+Additional implementation details:
+
+* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. Note that this per-shard tradeoff is independent of the per-index performance/accuracy tradeoff mentioned above.
+* Fields with cache type `ranked` will return the top rows sorted by count in descending order.
+* Fields with cache type `lru` will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
 * Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order.
 * The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
 
@@ -813,12 +824,39 @@ Options(Row(f1=10), shards=[0, 2])
 {"attrs":{},"columns":[100, 2097152]}
 ```
 
+#### Row Constant
+
+**Spec:**
+
+```
+ConstRow(columns=<[]COLUMN>)
+```
+
+**Description:**
+
+`ConstRow` provides a constant bitmap value that can be used in place of a `Row` call.
+The columns can be specified as integer IDs or strings.
+
+**Result Type:** row value columns.
+
+e.g. `{"attrs":{},"columns":[10, 20]}`
+
+**Examples:**
+
+Filter specified columns to only those with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1):
+```request
+Intersect(ConstRow(columns=[10, 20, 30]), Row(stargazer=1))
+```
+```response
+{"attrs":{},"columns":[10, 20]}
+```
+
 #### Rows
 
 **Spec:**
 
 ```
-Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>)
+Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>, like=<STRING>)
 ```
 
 **Description:**
@@ -839,6 +877,10 @@ If the field is of type `time`, the `from` and `to` arguments can be provided
 to restrict the result to a specific time span. If `from` and `to` are
 not provided, the full range of existing data will be queried.
 
+If `like` is given, only keys matching a pattern will be selected.
+A `like` pattern may use `_` as a placeholder to match a single UTF-8 codepoint, and `%` to match 0 or more codepoints.
+All other characters will be matched exactly.
+
 **Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
 
 **Examples:**
@@ -856,7 +898,50 @@ With keys:
 Rows(job)
 ```
 ```response
-{"rows":null,"keys":["engineer","management","student""]}
+{"rows":null,"keys":["engineer","management","student"]}
+```
+
+With `like`:
+```request
+Rows(job, like="%t")
+```
+```response
+{"rows":null,"keys":["management","student"]}
+```
+
+#### Extract
+
+**Spec:**
+```
+Extract(<ROW_CALL>, [<ROWS_CALL>...])
+```
+
+**Description:**
+
+Extract intersects a set of columns with a set of rows in order to extract a subset of the index.
+The result is a table consisting of the matched columns and the rows which they intersect.
+This is similar to a select query in a SQL database.
+
+**Result Type:** Object with an array of the selected fields and an array of the selected columns.
+The column array contains objects containing a column identifier and an array of field values.
+Field values are typed as such:
+- Bool Field - boolean or null
+- Mutex Field (unkeyed) - 64-bit unsigned integer or null
+- Mutex Field (keyed) - string or null
+- Integer Field - 64-bit signed integer or null
+- Decimal Field - Pilosa decimal value or null
+- Set Field (unkeyed) - array of 64-bit unsigned integers
+- Set Field (keyed) - array of strings
+- Time Field - same as the equivalent Set
+
+**Examples:**
+
+List all stargazers who have starred repository 1, and the full set of repositories they have starred:
+```request
+Extract(Row(stargazer=1), Rows(stargazer))
+```
+```response
+{"fields":[{"name":"stargazer","type":"set"}],"columns":[{"column":3,"rows":[[1, 2, 3]]}]}
 ```
 
 #### Group By
@@ -864,7 +949,7 @@ Rows(job)
 **Spec:**
 
 ```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
+GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>, aggregate=<CALL>)
 ```
 
 **Description:**
@@ -874,14 +959,18 @@ taking one row each from the specified `Rows` calls. It returns only those
 combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
- Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
- query.
+Intersect, etc.) which will be intersected with each result prior to returning
+the count. This is analagous to a WHERE clause applied to a relational GROUP BY
+query.
 
 The optional `limit` argument limits the number of results returned. The results
 are ordered, so as long as the data isn't changing, the same query will return
 the same result set.
 
+The optional `aggregate` argument takes a `Sum()` query which will be used to
+calculate the sum & count of each group. This is similar to using a `SUM()` in
+the SELECT clause of a relation GROUP BY query.
+
 Paging through results is supported by passing the `previous` argument to each
 of the `Rows` calls in the GroupBy. Take the last result from your previous
 `GroupBy` query, and pass each row ID in that result as the `previous` argument
@@ -941,3 +1030,31 @@ GroupBy(Rows(age), Rows(job), limit=7, filter=Row(country=USA))
  {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"student"}],"count":3},
  {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"management"}],"count":7}]
 ```
+
+#### UnionRows
+
+**Spec:**
+
+```
+UnionRows([ROWSET_CALL ...])
+```
+
+**Description:**
+
+UnionRows performs a logical OR on the rows matched by the results of all `ROWSET_CALL` queries passed to it.
+
+**Result Type:** object with attrs and columns
+
+attrs will always be empty
+
+**Examples:**
+
+Query columns with a bit set in any row (repositories that are starred by any user):
+```request
+UnionRows(Rows(stargazer))
+```
+```response
+{"attrs":{},"columns":[10, 20, 30]}
+```
+
+* columns are repositories that were starred by any user

enterprise/COPYING

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

enterprise/enterprise.go

@@ -1,23 +0,0 @@
-// Copyright (c) 2018 Pilosa Corp. All rights reserved.
-//
-// This file is part of Pilosa Enterprise Edition.
-//
-// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package enterprise is now deprecated, and the functionality under
-// enterprise/b has been copied into roaring/. It existed to inject enterprise
-// implementations of various Pilosa features when Pilosa was built with
-// "ENTERPRISE=1 make install". These features were dual-licensed separately
-// from Pilosa community edition under the AGPL and Pilosa's commercial license.
-package enterprise

executor_internal_test.go

@@ -15,22 +15,27 @@
 package pilosa
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"strings"
+	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
-func TestExecutor_TranslateGroupByCall(t *testing.T) {
+func TestExecutor_TranslateRowsOnBool(t *testing.T) {
+	path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
+	holder := NewHolder(path, nil)
+	defer holder.Close()
+
 	e := &executor{
-		Holder: NewHolder(),
+		Holder:  holder,
+		Cluster: NewTestCluster(t, 1),
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
-	err := e.Holder.Open()
-	if err != nil {
+	if err := e.Holder.Open(); err != nil {
 		t.Fatalf("opening holder: %v", err)
 	}
 
@@ -39,162 +44,55 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("creating index: %v", err)
 	}
 
-	_, erra := idx.CreateField("ak", OptFieldKeys())
-	_, errb := idx.CreateField("b")
-	_, errc := idx.CreateField("ck", OptFieldKeys())
-	if erra != nil || errb != nil || errc != nil {
-		t.Fatalf("creating fields %v, %v, %v", erra, errb, errc)
+	shard := uint64(0)
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	fb, errb := idx.CreateField("b", OptFieldTypeBool())
+	_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
+	if errb != nil || errbk != nil {
+		t.Fatalf("creating fields %v, %v", errb, errbk)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
-	if err != nil {
-		t.Fatalf("parsing query: %v", err)
-	}
-	c := query.Calls[0]
-	err = e.translateGroupByCall("i", idx, c)
-	if err != nil {
-		t.Fatalf("translating call: %v", err)
-	}
-	if len(c.Args["previous"].([]interface{})) != 3 {
-		t.Fatalf("unexpected length for 'previous' arg %v", c.Args["previous"])
-	}
-	for i, v := range c.Args["previous"].([]interface{}) {
-		if !isInt(v) {
-			t.Fatalf("expected all items in previous to be ints, but '%v' at index %d is %[1]T", v, i)
-		}
+	_, err1 := fb.SetBit(tx, 1, 1, nil)
+	_, err2 := fb.SetBit(tx, 2, 2, nil)
+	_, err3 := fb.SetBit(tx, 3, 3, nil)
+	if err1 != nil || err2 != nil || err3 != nil {
+		t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
 	}
 
-	errTests := []struct {
+	if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
 		pql string
-		err string
 	}{
-		{
-			pql: `GroupBy(Rows(notfound), previous=1)`,
-			err: "'previous' argument must be list",
-		},
-		{
-			pql: `GroupBy(Rows(ak), previous=["la", 0])`,
-			err: "mismatched lengths",
-		},
-		{
-			pql: `GroupBy(Rows(ak), previous=[1])`,
-			err: "prev value must be a string",
-		},
-		{
-			pql: `GroupBy(Rows(notfound), previous=[1])`,
-			err: ErrFieldNotFound.Error(),
-		},
-		// TODO: an unknown key will actually allocate an id. this is probably bad.
-		// {
-		// 	pql: `GroupBy(Rows(ak), previous=["zoop"])`,
-		// 	err: "translating row key '",
-		// },
-		{
-			pql: `GroupBy(Rows(b), previous=["la"])`,
-			err: "which doesn't use string keys",
-		},
+		{pql: "Rows(b)"},
+		{pql: "GroupBy(Rows(b))"},
+		{pql: "Set(4, b=true)"},
 	}
 
-	for i, test := range errTests {
-		t.Run(fmt.Sprintf("#%d_%s", i, test.err), func(t *testing.T) {
+	for _, test := range tests {
+		t.Run(test.pql, func(t *testing.T) {
 			query, err := pql.ParseString(test.pql)
 			if err != nil {
 				t.Fatalf("parsing query: %v", err)
 			}
+
 			c := query.Calls[0]
-			err = e.translateGroupByCall("i", idx, c)
-			if err == nil {
-				t.Fatalf("expected error, but translated call is '%s", c)
+			colTranslations, rowTranslations, err := e.preTranslate(context.Background(), "i", c)
+			if err != nil {
+				t.Fatalf("pre-translating call: %v", err)
 			}
-			if !strings.Contains(err.Error(), test.err) {
-				t.Fatalf("expected '%s', got '%v'", test.err, err)
+			_, err = e.translateCall(c, "i", colTranslations, rowTranslations)
+			if err != nil {
+				t.Fatalf("translating call: %v", err)
 			}
 		})
 	}
 }
 
-func isInt(a interface{}) bool {
-	switch a.(type) {
-	case int, int64, uint, uint64:
-		return true
-	default:
-		return false
-	}
-}
-
-func TestFilterWithLimit(t *testing.T) {
-	f := filterWithLimit(5)
-
-	for i := uint64(0); i < 5; i++ {
-		include, done := f(i, i*(1<<shardVsContainerExponent), nil)
-		if done {
-			t.Fatalf("limit filter ended early on iteration %d", i)
-		}
-		if !include {
-			t.Fatalf("limit filter should always include until done")
-		}
-	}
-	inc, done := f(5, 5*(1<<shardVsContainerExponent)+1, nil)
-	if !done {
-		t.Fatalf("limit filter should have been done, but got inc: %v done: %v", inc, done)
-	}
-}
-
-func TestFilterWithRows(t *testing.T) {
-	tests := []struct {
-		rows     []uint64
-		callWith []uint64
-		expect   [][2]bool
-	}{
-		{
-			rows:     []uint64{},
-			callWith: []uint64{0},
-			expect:   [][2]bool{{false, true}},
-		},
-		{
-			rows:     []uint64{0},
-			callWith: []uint64{0},
-			expect:   [][2]bool{{true, true}},
-		},
-		{
-			rows:     []uint64{1},
-			callWith: []uint64{0, 2},
-			expect:   [][2]bool{{false, false}, {false, true}},
-		},
-		{
-			rows:     []uint64{0},
-			callWith: []uint64{1, 2},
-			expect:   [][2]bool{{false, true}, {false, true}},
-		},
-		{
-			rows:     []uint64{3, 9},
-			callWith: []uint64{1, 2, 3, 10},
-			expect:   [][2]bool{{false, false}, {false, false}, {true, false}, {false, true}},
-		},
-		{
-			rows:     []uint64{0, 1, 2},
-			callWith: []uint64{0, 1, 2},
-			expect:   [][2]bool{{true, false}, {true, false}, {true, true}},
-		},
-	}
-
-	for num, test := range tests {
-		t.Run(fmt.Sprintf("%d_%v_with_%v", num, test.rows, test.callWith), func(t *testing.T) {
-			if len(test.callWith) != len(test.expect) {
-				t.Fatalf("Badly specified test - must expect the same number of values as calls.")
-			}
-			f := filterWithRows(test.rows)
-			for i, id := range test.callWith {
-				inc, done := f(id, 0, nil)
-				if inc != test.expect[i][0] || done != test.expect[i][1] {
-					t.Fatalf("Calling with %d\nexp: %v,%v\ngot: %v,%v", id, test.expect[i][0], test.expect[i][1], inc, done)
-				}
-			}
-		})
-	}
-
-}
-
 func TestFieldRowMarshalJSON(t *testing.T) {
 	fr := FieldRow{
 		Field:  "blah",
@@ -222,3 +120,358 @@ func TestFieldRowMarshalJSON(t *testing.T) {
 		t.Fatalf("unexpected json: %s", b)
 	}
 }
+
+func TestExecutor_GroupCountCondition(t *testing.T) {
+	t.Run("satisfiesCondition", func(t *testing.T) {
+		type condCheck struct {
+			cond string
+			exp  bool
+		}
+		tests := []struct {
+			groupCount GroupCount
+			checks     []condCheck
+		}{
+			{
+				groupCount: GroupCount{Count: 100},
+				checks: []condCheck{
+					{cond: "count == 99", exp: false},
+					{cond: "count != 99", exp: true},
+					{cond: "count < 99", exp: false},
+					{cond: "count <= 99", exp: false},
+					{cond: "count > 99", exp: true},
+					{cond: "count >= 99", exp: true},
+
+					{cond: "count == 100", exp: true},
+					{cond: "count != 100", exp: false},
+					{cond: "count < 100", exp: false},
+					{cond: "count <= 100", exp: true},
+					{cond: "count > 100", exp: false},
+					{cond: "count >= 100", exp: true},
+
+					{cond: "count == 101", exp: false},
+					{cond: "count != 101", exp: true},
+					{cond: "count < 101", exp: true},
+					{cond: "count <= 101", exp: true},
+					{cond: "count > 101", exp: false},
+					{cond: "count >= 101", exp: false},
+
+					{cond: "98 < count < 100", exp: false},
+					{cond: "98 < count <= 100", exp: true},
+					{cond: "98 < count < 101", exp: true},
+					{cond: "100 <= count < 102", exp: true},
+					{cond: "100 < count < 102", exp: false},
+					{cond: "98 <= count <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Agg: 100},
+				checks: []condCheck{
+					{cond: "sum == 99", exp: false},
+					{cond: "sum != 99", exp: true},
+					{cond: "sum < 99", exp: false},
+					{cond: "sum <= 99", exp: false},
+					{cond: "sum > 99", exp: true},
+					{cond: "sum >= 99", exp: true},
+
+					{cond: "sum == 100", exp: true},
+					{cond: "sum != 100", exp: false},
+					{cond: "sum < 100", exp: false},
+					{cond: "sum <= 100", exp: true},
+					{cond: "sum > 100", exp: false},
+					{cond: "sum >= 100", exp: true},
+
+					{cond: "sum == 101", exp: false},
+					{cond: "sum != 101", exp: true},
+					{cond: "sum < 101", exp: true},
+					{cond: "sum <= 101", exp: true},
+					{cond: "sum > 101", exp: false},
+					{cond: "sum >= 101", exp: false},
+
+					{cond: "98 < sum < 100", exp: false},
+					{cond: "98 < sum <= 100", exp: true},
+					{cond: "98 < sum < 101", exp: true},
+					{cond: "100 <= sum < 102", exp: true},
+					{cond: "100 < sum < 102", exp: false},
+					{cond: "98 <= sum <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Agg: -100},
+				checks: []condCheck{
+					{cond: "sum == -99", exp: false},
+					{cond: "sum != -99", exp: true},
+					{cond: "sum < -99", exp: true},
+					{cond: "sum <= -99", exp: true},
+					{cond: "sum > -99", exp: false},
+					{cond: "sum >= -99", exp: false},
+
+					{cond: "sum == -100", exp: true},
+					{cond: "sum != -100", exp: false},
+					{cond: "sum < -100", exp: false},
+					{cond: "sum <= -100", exp: true},
+					{cond: "sum > -100", exp: false},
+					{cond: "sum >= -100", exp: true},
+
+					{cond: "sum == -101", exp: false},
+					{cond: "sum != -101", exp: true},
+					{cond: "sum < -101", exp: false},
+					{cond: "sum <= -101", exp: false},
+					{cond: "sum > -101", exp: true},
+					{cond: "sum >= -101", exp: true},
+
+					{cond: "-100 < sum < -98", exp: false},
+					{cond: "-100 <= sum < -98", exp: true},
+					{cond: "-101 < sum < -98", exp: true},
+					{cond: "-102 < sum <= -100", exp: true},
+					{cond: "-102 < sum < -100", exp: false},
+					{cond: "-102 <= sum <= -98", exp: true},
+				},
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
+				for j, check := range test.checks {
+					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
+
+						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
+						if err != nil {
+							t.Fatalf("parsing query: %v", err)
+						}
+						c := query.Calls[0]
+						having := c.Args["having"].(*pql.Call)
+
+						var got bool
+						for subj, cond := range having.Args {
+							switch subj {
+							case "count", "sum":
+								condition, ok := cond.(*pql.Condition)
+								if !ok {
+									t.Fatalf("not a valid condition")
+								}
+								got = test.groupCount.satisfiesCondition(subj, condition)
+							}
+						}
+
+						if got != check.exp {
+							t.Fatalf("expected: %v, but got: %v", check.exp, got)
+						}
+					})
+				}
+			})
+		}
+	})
+}
+
+func TestValCountComparisons(t *testing.T) {
+	tests := []struct {
+		name       string
+		vc         ValCount
+		other      ValCount
+		expLarger  ValCount
+		expSmaller ValCount
+	}{
+		{
+			name: "zero",
+		},
+		{
+			name:       "ints",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 3, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 1},
+			expSmaller: ValCount{Val: 3, Count: 2},
+		},
+		{
+			name:       "floats",
+			vc:         ValCount{FloatVal: 10.2, Count: 1},
+			other:      ValCount{FloatVal: 3.4, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
+			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
+		},
+		{
+			name:       "intsEquality",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 10, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 3},
+			expSmaller: ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:       "floatsEquality",
+			vc:         ValCount{FloatVal: 10.7, Count: 1},
+			other:      ValCount{FloatVal: 10.7, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
+			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			gotLarger := test.vc.larger(test.other)
+			if gotLarger != test.expLarger {
+				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
+			}
+
+			gotSmaller := test.vc.smaller(test.other)
+			if gotSmaller != test.expSmaller {
+				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
+			}
+		})
+	}
+}
+
+func TestToNegInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1 << 63),
+			i64: int64(-1 << 63),
+		},
+		{
+			u64: uint64(1<<63) - 1,
+			i64: int64(-1<<63) + 1,
+		},
+		{
+			u64:      uint64(1<<63) + 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toNegInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}
+
+func TestToInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1<<63) - 1,
+			i64: 1<<63 - 1,
+		},
+		{
+			u64: uint64(0),
+			i64: 0,
+		},
+		{
+			u64:      uint64(1 << 63),
+			overflow: true,
+		},
+		{
+			u64:      1<<64 - 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}
+
+func TestGetSorter(t *testing.T) {
+	tests := []struct {
+		sortSpec string
+		expGCS   *groupCountSorter
+		expErr   string
+	}{
+		{
+			sortSpec: "count asc",
+			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
+		},
+		{
+			sortSpec: "count    asc",
+			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
+		},
+		{
+			sortSpec: "  count asc",
+			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
+		},
+		{
+			sortSpec: "  count asc    ",
+			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{asc}},
+		},
+		{
+			sortSpec: "count",
+			expGCS:   &groupCountSorter{fields: []int{-1}, order: []order{desc}},
+		},
+		{
+			sortSpec: "sum asc",
+			expGCS:   &groupCountSorter{fields: []int{-2}, order: []order{asc}},
+		},
+		{
+			sortSpec: "aggregate asc",
+			expGCS:   &groupCountSorter{fields: []int{-2}, order: []order{asc}},
+		},
+		{
+			sortSpec: "boondoggle asc",
+			expErr:   "sorting is only supported on count, aggregate, or sum, not 'boondoggle'",
+		},
+		{
+			sortSpec: "sum asc, count desc",
+			expGCS:   &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}},
+		},
+		{
+			sortSpec: "count asc, sum desc",
+			expGCS:   &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
+		},
+		{
+			sortSpec: " count  asc ,  sum  desc ",
+			expGCS:   &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}},
+		},
+		{
+			sortSpec: " count  asc ,  sum  desc blah",
+			expErr:   "parsing sort directive: 'sum  desc blah': too many elements",
+		},
+		{
+			sortSpec: "count asc, sum fesc",
+			expErr:   "unknown sort direction 'fesc'",
+		},
+		{
+			sortSpec: " , sum fesc",
+			expErr:   "invalid sorting directive: ''",
+		},
+		{
+			// weird and useless, but I guess fine?
+			sortSpec: "count  asc,count asc ",
+			expGCS:   &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}},
+		},
+	}
+
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) {
+			gcs, err := getSorter(tst.sortSpec)
+			if err != nil {
+				if tst.expErr == "" {
+					t.Errorf("unexpected error: %v", err)
+					return
+				}
+				if tst.expErr != err.Error() {
+					t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr)
+				}
+				return
+			}
+			if !reflect.DeepEqual(gcs, tst.expGCS) {
+				t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs)
+			}
+
+		})
+	}
+}

field_internal_test.go

@@ -16,15 +16,18 @@ package pilosa
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a bsiGroup can adjust to its baseValue.
@@ -66,12 +69,12 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b0, pql.LT, 5, 105, false},
 			{b0, pql.LT, -8, 92, false},
 			{b0, pql.LT, -108, -8, false},
-			{b0, pql.LT, 1005, 1023, false},
+			{b0, pql.LT, 1005, 1024, false},
 			{b0, pql.LT, 0, 100, false},
 
 			{b1, pql.LT, 5, 5, false},
 			{b1, pql.LT, -8, -8, false},
-			{b1, pql.LT, 1005, 255, false},
+			{b1, pql.LT, 1005, 256, false},
 			{b1, pql.LT, 0, 0, false},
 
 			{b2, pql.LT, 5, -95, false},
@@ -89,6 +92,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, pql.GT, -8, -8, false},
 			{b1, pql.GT, 1005, 0, true},
 			{b1, pql.GT, 0, 0, false},
+			{b1, pql.GT, -300, -256, false},
 
 			{b2, pql.GT, 5, -95, false},
 			{b2, pql.GT, -8, -108, false},
@@ -141,6 +145,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, 5, 20, 5, 20, false},
 			{b1, 20, 1005, 20, 255, false},
 			{b1, 1005, 2000, 0, 0, true},
+			{b1, 0, -1, 0, 0, true},
 
 			{b2, 5, 95, -95, -5, false},
 			{b2, 95, 120, -5, 20, false},
@@ -158,7 +163,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	viewName := viewStandard + "_v"
@@ -192,64 +197,73 @@ func TestField_DeleteView(t *testing.T) {
 // TestField represents a test wrapper for Field.
 type TestField struct {
 	*Field
+	parent *Index
+	tb     testing.TB
 }
 
 // NewTestField returns a new instance of TestField d/0.
-func NewTestField(opts FieldOption) *TestField {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
+func NewTestField(t *testing.T, opts FieldOption) *TestField {
+	path, err := testhook.TempDirInDir(t, *TempDir, "pilosa-field-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := NewHolder(path, nil)
+	panicOn(h.Open())
+
+	idx, err := h.CreateIndex("i", IndexOptions{})
 	if err != nil {
 		panic(err)
 	}
-	field, err := NewField(path, "i", "f", opts)
+	field, err := idx.CreateField("f", opts)
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
-	return &TestField{Field: field}
+	tf := &TestField{Field: field, parent: idx, tb: t}
+	testhook.Cleanup(t, func() {
+		h.Close()
+	})
+	return tf
 }
 
-// MustOpenField returns a new, opened field at a temporary path. Panic on error.
-func MustOpenField(opts FieldOption) *TestField {
-	f := NewTestField(opts)
-	if err := f.Open(); err != nil {
-		panic(err)
-	}
+// OpenField returns a new, opened field at a temporary path.
+func OpenField(t *testing.T, opts FieldOption) *TestField {
+	f := NewTestField(t, opts)
 	return f
 }
 
 // Close closes the field and removes the underlying data.
 func (f *TestField) Close() error {
+	if f.idx != nil {
+		panicOn(f.idx.holder.txf.CloseIndex(f.idx))
+	}
 	defer os.RemoveAll(f.Path())
 	return f.Field.Close()
 }
 
 // Reopen closes the index and reopens it.
 func (f *TestField) Reopen() error {
-	var err error
-	if err := f.Field.Close(); err != nil {
+	name := f.Field.Name()
+	if err := f.parent.Close(); err != nil {
+		f.parent = nil
 		return err
 	}
-
-	path, index, name := f.Path(), f.Index(), f.Name()
-	f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
-	if err != nil {
-		return err
-	}
-
-	if err := f.Open(); err != nil {
+	if err := f.parent.Open(); err != nil {
+		f.parent = nil
 		return err
 	}
+	f.Field = f.parent.Field(name)
 	return nil
 }
 
-func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
+func (f *TestField) MustSetBit(tx Tx, row, col uint64, ts ...time.Time) {
 	if len(ts) == 0 {
-		_, err := f.Field.SetBit(row, col, nil)
+		_, err := f.Field.SetBit(tx, row, col, nil)
 		if err != nil {
 			panic(err)
 		}
 	}
 	for _, t := range ts {
-		_, err := f.Field.SetBit(row, col, &t)
+		_, err := f.Field.SetBit(tx, row, col, &t)
 		if err != nil {
 			panic(err)
 		}
@@ -258,7 +272,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
 
 // Ensure field can open and retrieve a view.
 func TestField_CreateViewIfNotExists(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	// Create view.
@@ -283,7 +297,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
 }
 
 func TestField_SetTimeQuantum(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	// Set & retrieve time quantum.
@@ -302,44 +316,54 @@ func TestField_SetTimeQuantum(t *testing.T) {
 }
 
 func TestField_RowTime(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
+	// Obtain transaction.
+	tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
+	defer tx.Rollback()
+
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
 		t.Fatal(err)
 	}
 
-	f.MustSetBit(1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
-	f.MustSetBit(1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
+	f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
 
-	if r, err := f.RowTime(1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
+	panicOn(tx.Commit())
+
+	// obtain 2nd transaction to read it back.
+	tx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
+	defer tx.Rollback()
+
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{3}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
 	}
 
-	if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil {
+	if r, err := f.RowTime(tx, 1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{5}) {
 		t.Fatalf("wrong columns: %#v", r.Columns())
@@ -348,7 +372,9 @@ func TestField_RowTime(t *testing.T) {
 }
 
 func TestField_PersistAvailableShards(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -356,6 +382,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
@@ -366,10 +393,74 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap()
@@ -383,12 +474,14 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
-		t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+		t.Fatalf("unexpected available shards (reopen). expected: %v, \n but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+
 	}
 
 	bm1 := roaring.NewBitmap()
@@ -438,3 +531,394 @@ func TestBSIGroup_BaseDefaultValue(t *testing.T) {
 		}
 	}
 }
+
+func TestField_ApplyOptions(t *testing.T) {
+	for i, tt := range []struct {
+		opts    FieldOptions
+		expOpts FieldOptions
+	}{
+		{
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+		},
+	} {
+
+		fld := &Field{}
+		fld.options = *applyDefaultOptions(&FieldOptions{})
+
+		if err := fld.applyOptions(tt.opts); err != nil {
+			t.Fatal(err)
+		}
+
+		if fld.options.CacheType != tt.expOpts.CacheType {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheType value. expected: %s, but got: %s", i, tt.expOpts.CacheType, fld.options.CacheType)
+		} else if fld.options.CacheSize != tt.expOpts.CacheSize {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheSize value. expected: %d, but got: %d", i, tt.expOpts.CacheSize, fld.options.CacheSize)
+		}
+	}
+}
+
+// Ensure that importValue handles requiredDepth correctly.
+// This test sets the same column value to 1, then 8, then 1.
+// A previous bug was incorrectly determining bitDepth based
+// on the values in the import, and not taking existing values
+// into consideration. This would cause an import of 1/8/1
+// to result in a value of 9 instead of 1.
+func TestBSIGroup_importValue(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+		panicOn(qcx.Finish())
+		qcx.Reset()
+		if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+		panicOn(qcx.Finish())
+		qcx.Reset()
+	} // loop
+}
+
+func TestIntField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []int64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []int64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []int64{10},
+			expMax:    ValCount{Val: 10, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []int64{10, 20},
+			expMax:    ValCount{Val: 20, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []int64{10, 20, 10, 10, 20},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11, 44, 1},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importValue(qcx, test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+			panicOn(qcx.Finish())
+			qcx.Reset()
+
+			shard := uint64(0)
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			// Rollback below manually, because we are in a loop.
+
+			maxvc, err := f.MaxForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+			tx.Rollback()
+		})
+	}
+}
+
+// Ensure we get errors when they are expected.
+func TestDecimalField_MinMaxBoundaries(t *testing.T) {
+	th := newTestHolder(t)
+
+	for i, test := range []struct {
+		scale  int64
+		min    pql.Decimal
+		max    pql.Decimal
+		expErr bool
+	}{
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 0),
+			max:    pql.NewDecimal(math.MaxInt64, 0),
+			expErr: true,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 3),
+			max:    pql.NewDecimal(math.MaxInt64, 3),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(-44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(1, 0),
+			max:    pql.NewDecimal(2, 0),
+			expErr: true,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(math.MinInt64, 18),
+			max:    pql.NewDecimal(math.MaxInt64, 18),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, 20),
+			max:    pql.NewDecimal(2, 20),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -1),
+			max:    pql.NewDecimal(2, -1),
+			expErr: false,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -19),
+			max:    pql.NewDecimal(2, -19),
+			expErr: true,
+		},
+	} {
+		t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
+			_, err := NewField(th, "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
+			if err != nil && test.expErr {
+				if !strings.Contains(err.Error(), "is not supported") {
+					t.Fatal(err)
+				}
+			} else if err != nil && !test.expErr {
+				t.Fatalf("did not expect error, but got: %s", err)
+			} else if err == nil && test.expErr {
+				t.Fatal("expected error, but got none")
+			}
+		})
+	}
+}
+
+func TestDecimalField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeDecimal(3))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []float64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []float64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []float64{10.1},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []float64{10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "another shard",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11, 44.39, 0.23},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importFloatValue(qcx, test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			shard := uint64(0)
+			tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard})
+			defer tx.Rollback()
+
+			maxvc, err := f.MaxForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if !reflect.DeepEqual(maxvc, test.expMax) {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(tx, shard, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if !reflect.DeepEqual(minvc, test.expMin) {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}
+
+func TestBSIGroup_TxReopenDB(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
+
+	qcx := f.idx.holder.txf.NewQcx()
+	defer qcx.Abort()
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+		panicOn(qcx.Finish())
+		qcx.Reset()
+
+		if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+		panicOn(qcx.Finish())
+		qcx.Reset()
+	} // loop
+
+	// the test: can we re-open a BSI fragment under Tx store
+	_ = f.Reopen()
+}

field_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"math"
 	"testing"
 
@@ -23,12 +22,13 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a field can set & read a bsiGroup value.
 func TestField_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -36,15 +36,18 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value on field.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Read value.
-		if value, exists, err := f.Value(100); err != nil {
+		if value, exists, err := f.Value(tx, 100); err != nil {
 			t.Fatal(err)
 		} else if value != 21 {
 			t.Fatalf("unexpected value: %d", value)
@@ -53,7 +56,7 @@ func TestField_SetValue(t *testing.T) {
 		}
 
 		// Setting value should return no change.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if changed {
 			t.Fatal("expected no change")
@@ -61,7 +64,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("Overwrite", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -69,22 +72,25 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if changed, err := f.SetValue(100, 21); err != nil {
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Set different value.
-		if changed, err := f.SetValue(100, 23); err != nil {
+		if changed, err := f.SetValue(tx, 100, 23); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal("expected change")
 		}
 
 		// Read value.
-		if value, exists, err := f.Value(100); err != nil {
+		if value, exists, err := f.Value(tx, 100); err != nil {
 			t.Fatal(err)
 		} else if value != 23 {
 			t.Fatalf("unexpected value: %d", value)
@@ -94,7 +100,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -102,14 +108,17 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound {
+		if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 
 	t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -117,14 +126,17 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow {
+		if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 
 	t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -132,19 +144,22 @@ func TestField_SetValue(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
 		// Set value.
-		if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
+		if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
 }
 
 func TestField_NameRestriction(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
+	field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault())
 	if field != nil {
 		t.Fatalf("unexpected field name %s", err)
 	}
@@ -158,6 +173,7 @@ func TestField_NameValidation(t *testing.T) {
 		"under_score",
 		"abc123",
 		"trailing_",
+		"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
 	}
 	invalidFieldNames := []string{
 		"",
@@ -168,30 +184,32 @@ func TestField_NameValidation(t *testing.T) {
 		"abc def",
 		"camelCase",
 		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
+		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
 	for _, name := range validFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatalf("unexpected field name: %s %s", name, err)
 		}
 	}
 	for _, name := range invalidFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err == nil {
 			t.Fatalf("expected error on field name: %s", name)
 		}
 	}
 }
 
+const includeRemote = false // for calls to Index.AvailableShards(localOnly bool)
+
 // Ensure can update and delete available shards.
 func TestField_AvailableShards(t *testing.T) {
-	idx := test.MustOpenIndex()
+	idx := test.MustOpenIndex(t)
 	defer idx.Close()
 
 	f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -199,12 +217,15 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// It is okay to pass in a nil tx. f.SetBit will lazily instantiate Tx.
+	var tx pilosa.Tx
+
 	// Set values on shards 0 & 2, and verify.
-	if _, err := f.SetBit(0, 100, nil); err != nil {
+	if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
 		t.Fatal(err)
-	} else if _, err := f.SetBit(0, ShardWidth*2, nil); err != nil {
+	} else if _, err := f.SetBit(tx, 0, ShardWidth*2, nil); err != nil {
 		t.Fatal(err)
-	} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	} else if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -212,7 +233,7 @@ func TestField_AvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
 		t.Fatalf("adding remote shards: %v", err)
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 1, 2, 4}); diff != "" {
 		t.Fatal(diff)
 	}
 
@@ -223,7 +244,50 @@ func TestField_AvailableShards(t *testing.T) {
 			t.Fatalf("removing shard %d: %v", i, err)
 		}
 	}
-	if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
+	if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
 }
+
+func TestField_ClearValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex(t)
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatal(err)
+		}
+		// It is okay to pass in a nil tx. f.SetValue will lazily instantiate Tx.
+		var tx pilosa.Tx
+
+		// Set value on field.
+		if changed, err := f.SetValue(tx, 100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		if changed, err := f.ClearValue(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal(err)
+		}
+
+		// Read value.
+		if _, exists, err := f.Value(tx, 100); err != nil {
+			t.Fatal(err)
+		} else if exists {
+			t.Fatal("expected value to not exist")
+		}
+	})
+}

filesystem.go

@@ -0,0 +1,42 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// Ensure nopFileSystem implements interface.
+var _ FileSystem = &nopFileSystem{}
+
+// FileSystem represents an interface for file system for serving the Lattice UI.
+type FileSystem interface {
+	New() (http.FileSystem, error)
+}
+
+func init() {
+	NopFileSystem = &nopFileSystem{}
+}
+
+// NopFileSystem represents a FileSystem that returns an error if called.
+var NopFileSystem FileSystem
+
+type nopFileSystem struct{}
+
+// New is a no-op implementation of FileSystem New method.
+func (n *nopFileSystem) New() (http.FileSystem, error) {
+	return nil, fmt.Errorf("file system not implemented")
+}