Merge pull request #1563 from 54mir/timestamp-in-orm

Replicate timefield functionality in pilosaclient

.circleci/config.yml

@@ -1,195 +1,289 @@
-version: 2
+version: 2.1
-defaults: &defaults
+
-  working_directory: /go/src/github.com/pilosa/pilosa
+executors:
+  golang:
+    parameters:
+      version:
+        type: string
+        default: "1.15.8"
+      resource_class:
+        type: string
+        default: medium
     docker:
-    - image: circleci/golang:1.13
+      - image: circleci/golang:<< parameters.version >>
-  environment:
+    resource_class: << parameters.resource_class >>
-    GO111MODULE: "on"
+    working_directory: /go/src/github.com/pilosa/pilosa
-fast-checkout: &fast-checkout
+
-  attach_workspace:
+commands:
-    at: .
+  add-github-auth:
-jobs:
+    steps:
-  setup:
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
-    <<: *defaults
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
+  restore-mod-cache:
     steps:
-      - checkout
       - restore_cache:
-          keys:
+          key: mod-cache-{{ checksum "go.sum" }}
-            - mod-cache-{{ checksum "go.sum" }}
+  save-mod-cache:
-      - run: "go mod download"
+    steps:
       - save_cache:
           key: mod-cache-{{ checksum "go.sum" }}
           paths:
             - /go/pkg/mod/
-      - persist_to_workspace:
+  checkout-plus:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
     steps:
-      - *fast-checkout
+      - add-github-auth
+      - checkout
+      - restore-mod-cache
+
+jobs:
+  setup:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod download
+      - save-mod-cache
+  check-license-headers:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
       - run: make golangci-lint
-  test-build-arm:
+  go-mod-tidy:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - run: go mod tidy
+      - run: git diff --exit-code -- go.mod go.sum
+  check-changelog-label:
+    executor:
+      name: golang
+    steps:
+      - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
+      - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
+  test-build-arm:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
+  test:
-    <<: *defaults
+    parameters:
+      resource_class:
+        type: string
+        default: medium
+      golang_version:
+        type: string
+        default: "1.15.8"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
+      resource_class: << parameters.resource_class >>
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: sudo apt-get install lsof
+      - run: sudo apt-get update
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
+  test-external-lookup:
-    <<: *base-test
+    docker:
+      - image: circleci/golang:1.15.8
+      - image: circleci/postgres:13.2-ram
         environment:
-      GO111MODULE: "on"
+          POSTGRES_PASSWORD=password
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: sudo apt-get install lsof
+      - run: sudo apt-get update
-      - run: make test ENTERPRISE=1
+      - run: sudo apt-get install postgresql-client
-  test-golang-1.12:
+      - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
-    <<: *defaults
+      - run:
-    docker:
+          command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
-      - image: circleci/golang:1.12
+          no_output_timeout: 30m
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - setup_remote_docker
       - run: make clustertests-build
-  prerelease:
-    <<: *base-test
-    steps:
-      - *fast-checkout
-      - run: make prerelease
-      - store_artifacts:
-          path: build
-      - persist_to_workspace:
-          root: .
-          paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: make release
+      - attach_workspace:
+          at: .
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker-release
       - store_artifacts:
           path: build
       - persist_to_workspace:
           root: .
           paths: build
-  prerelease-upload:
+  publish_release:
-    docker:
+    executor:
-      - image: circleci/python:2.7-jessie
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - attach_workspace:
-      - *fast-checkout
+          at: .
-      - run: sudo pip install awscli
+      - run: go get github.com/tcnksm/ghr
-      - run: make prerelease-upload
+      - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
-  dockerhub-upload:
+  docker-build:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - checkout-plus
-      - *fast-checkout
+      - setup_remote_docker:
-      - setup_remote_docker
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker GO_VERSION=1.15.8
+      - run: docker run pilosa:$(git describe --tags) help
+  dockerhub-upload-unstable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
+      - run: docker run pilosa:$(git describe --tags) help
-      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
-      - run: docker push pilosa/pilosa:master
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker
+      - run: docker run pilosa:$(git describe --tags) help
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
-  version: 2
+  build:
-  test:
     jobs:
-      - setup
+      - setup:
+          context: molecula
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
+          context: molecula
           requires:
             - setup
       - check-license-headers:
+          context: molecula
+          requires:
+            - setup
+      - go-mod-tidy:
+          context: molecula
+          requires:
+            - setup
+      - check-changelog-label:
+          context: molecula
           requires:
             - setup
       - test-build-arm:
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          resource_class: large
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-race:
+          matrix:
+            parameters:
+              golang_version: ["1.14.15", "1.15.8"]
+      - test:
+          name: << matrix.test_make_target >>
+          resource_class: xlarge
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-386:
+          matrix:
+            parameters:
+              test_make_target: ["test-race", "test-txstore-rbf_bolt"]
+      - test:
+          name: test-shardwidth-22
+          context: molecula
+          shard_width: "22"
+          resource_class: large
           requires:
             - setup
-      - test-golang-1.13:
+      - test-external-lookup:
-          requires:
+          context: molecula
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
           requires:
             - setup
       - cluster-tests:
+          context: molecula
           requires:
             - setup
-      - prerelease:
+      - docker-build:
+          context: molecula
           requires:
-            - linter
+            - setup
-            - check-license-headers
-            - test-golang-1.13
       - release:
+          context: molecula
           requires:
-            - linter
+            - setup
-            - check-license-headers
+          filters:
-            - test-golang-1.13
+            tags:
+              only: /^v.*/
+            branches:
+              only: master
+      - publish_release:
+          context: molecula
+          requires:
+            - release
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - prerelease
+            - setup
-      - dockerhub-upload:
+          filters:
+            branches:
+              only: master
+      - dockerhub-upload-stable:
+          context: molecula
           requires:
-            - linter
+            - setup
-            - check-license-headers
+          filters:
-            - test-golang-1.13
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

.dockerignore

@@ -0,0 +1,5 @@
+lattice/.git
+lattice/node_modules
+lattice/build
+statik/statik.go
+build

.gitignore

@@ -4,3 +4,9 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*
+pilosa
+*.dot

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lattice"]
+	path = lattice
+	url = git@github.com:molecula/lattice.git

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,16 +1,46 @@
-FROM golang:1.13.0 as builder
+ARG GO_VERSION=latest
 
-COPY . pilosa
+#######################
+### Lattice builder ###
+#######################
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+FROM moleculacorp/nodejs:latest as lattice-builder
+WORKDIR /lattice
 
-FROM alpine:3.9.4
+COPY lattice/package.json ./
+COPY lattice/yarn.lock ./
+RUN yarn install
 
-LABEL maintainer "dev@pilosa.com"
+COPY lattice ./
+RUN yarn build
+
+######################
+### Pilosa builder ###
+######################
+
+FROM golang:${GO_VERSION} as pilosa-builder
+ARG MAKE_FLAGS
+WORKDIR /pilosa
+
+RUN go get github.com/rakyll/statik
+
+COPY . ./
+COPY --from=lattice-builder /lattice/build /lattice
+RUN /go/bin/statik -src=/lattice -dest=/pilosa
+
+RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
+
+#####################
+### Pilosa runner ###
+#####################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=builder /go/bin/pilosa /pilosa
+COPY --from=pilosa-builder /pilosa/build/pilosa /
 
 COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
@@ -18,5 +48,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
+    && make install FLAGS="-a -mod=vendor"
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,23 +1,38 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race  test-txstore-rbf lattice
 
 CLONE_URL=github.com/pilosa/pilosa
+MOD_VERSION=v2
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
-BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+VARIANT = Molecula
+GO=go
+GOOS=$(shell $(GO) env GOOS)
+GOARCH=$(shell $(GO) env GOARCH)
+VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-GO_VERSION=latest
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)"
-ENTERPRISE ?= 0
+GO_VERSION=1.15.8
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
+DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
-RELEASE ?= 0
-RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
-BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+TEST_TAGS = roaringparanoia
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+UNAME := $(shell uname -s)
+ifeq ($(UNAME), Darwin)
+    IS_MACOS:=1
+else
+    IS_MACOS:=0
+endif
+
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export CGO_ENABLED=0
 
 # Run tests and compile Pilosa
 default: test build
@@ -28,14 +43,48 @@ clean:
 
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
-	go mod vendor
+	$(GO) mod vendor
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v
+
+# Run test suite with race flag
+test-race:
+	CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v
+
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
+tour:
+	./tournament.sh
 
 bench:
-	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+	$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 
 # Run test suite with coverage enabled
 cover:
@@ -44,17 +93,16 @@ cover:
 
 # Run test suite with coverage enabled and view coverage results in browser
 cover-viz: cover
-	go tool cover -html=build/coverage.out
+	$(GO) tool cover -html=build/coverage.out
 
 # Compile Pilosa
 build:
-	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
 
 # Create a single release build under the build directory
 release-build:
-	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
+	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
+	cp NOTICE README.md LICENSE build/pilosa$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -64,15 +112,16 @@ ifndef SKIP_CHECK_CLEAN
 	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
 endif
 
-# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
+# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
-release: check-clean
+release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
-	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
+# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
+release-sans-ui: check-clean
+	rm -f statik/statik.go
+	$(MAKE) release-build GOOS=darwin GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=amd64
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
 DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
@@ -81,59 +130,143 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
+clustertests-build: vendor
-	docker-compose -f $(DOCKER_COMPOSE) down
+	docker-compose -f $(DOCKER_COMPOSE) down -v
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
-# Create prerelease builds
-prerelease:
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
-	$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
-
-prerelease-upload:
-	aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
-
 # Install Pilosa
 install:
-	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+
+install-bench:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
+
+# Ensure lattice is cloned and the pinned version is checked out
+lattice:
+	git submodule update --init
+
+# Build the lattice assets
+build-lattice: lattice
+	docker build -t lattice:build ./lattice
+	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
+
+# Upgrade lattice to the latest version
+upgrade-lattice: lattice
+	git submodule update --remote
 
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
-	go generate github.com/pilosa/pilosa/v2/internal
+	$(GO) generate github.com/pilosa/pilosa/v2/pb
+
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	$(GO) generate github.com/pilosa/pilosa/v2/statik
+
+# `go generate` statik assets (lattice UI) in Docker
+generate-statik-docker: build-lattice
+	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
 
 # `go generate` stringers
 generate-stringer:
-	go generate github.com/pilosa/pilosa/v2
+	$(GO) generate github.com/pilosa/pilosa/v2
 
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+generate-proto-grpc: require-protoc require-protoc-gen-go
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
+	# TODO: Modify above commands and remove the below mv if possible.
+	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
+	# I couldn't get it to work during development - Cody
+	cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/
+	rm -rf proto/github.com
+
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
+
+# Create release using Docker
+docker-release:
+	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
+
+# Build a release in Docker
+docker-build: vendor lattice
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --target pilosa-builder \
+	    --tag pilosa:build .
+	docker create --name pilosa-build pilosa:build
+	mkdir -p build/pilosa-$(VERSION_ID)
+	docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID)
+	cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID)
+	docker rm pilosa-build
+	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 
 # Create Docker image from Dockerfile
-docker:
+docker-image: vendor lattice
-	docker build -t "pilosa:$(VERSION)" .
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
+	    --tag pilosa:$(VERSION) .
 	@echo Created docker image: pilosa:$(VERSION)
 
-# Compile Pilosa inside Docker container
+# Create docker image (alias)
-docker-build:
+docker: docker-image # alias
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
+pilosa-keydump:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
+
+# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
+pilosa-chk:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
+
+pilosa-fsck:
+	cd ./cmd/pilosa-fsck && make install && make release
 
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
+
+# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
+# run top tests, not subdirs. print summary red/green after.
+# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
+topt:
+	mv log.topt.roar log.topt.roar.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
+	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
+	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
+
+topt-race:
+	mv log.topt.race log.topt.race.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
+	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
+	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +294,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
 
 ######################
 # Build dependencies #
@@ -175,24 +307,39 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go get -u github.com/rakyll/statik
 
 install-stringer:
-	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
+	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
 
 install-protoc-gen-gofast:
-	GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
+	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc-gen-go:
+	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
 
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 
 install-peg:
-	GO111MODULE=off go get github.com/pointlander/peg
+	GO111MODULE=off $(GO) get github.com/pointlander/peg
 
 install-golangci-lint:
-	GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
+	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 
 install-gometalinter:
-	GO111MODULE=off go get -u github.com/alecthomas/gometalinter
+	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
-	GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
+	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
+
+test-txstore-rbf:
+	PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race
+
+test-txstore-rbf_bolt:
+	PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race
+
+test-external-lookup:
+	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)

NOTICE

@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
 Third-party software licenses
 =============================
 
@@ -115,3 +94,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
 
 1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
 
+Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101).
+
 2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
 
     ```shell
@@ -40,7 +42,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
     and verify that it's running:
     
     ```shell
-    curl localhost:10101/nodes
+    curl localhost:10101/status
     ```
 
 3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
@@ -56,6 +58,15 @@ Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) w
 You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
 
 
+## Upgrading UI
+
+Lattice is now a submodule of Pilosa, for the purpose of keeping the frontend and backend components of the UI system synchronized. When making a change to the UI that requires changing both the backend and frontend, follow steps in this order:
+
+1. merge frontend (Lattice) PR
+2. run `make upgrade-lattice`
+3. run `git add lattice`, `git commit -m"Upgrade Lattice"`, `git push`
+4. merge backend (Pilosa) PR
+
 ## Client Libraries
 
 There are supported libraries for the following languages:
@@ -63,13 +74,12 @@ There are supported libraries for the following languages:
 - [Java](https://www.pilosa.com/docs/client-libraries/#java)
 - [Python](https://www.pilosa.com/docs/client-libraries/#python)
 
-## Licenses
+## License
 
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
+Pilosa is licensed under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
+
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
 
 ## Get Support
 

api/client/grpc.go

@@ -0,0 +1,214 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -16,9 +16,12 @@ package pilosa_test
 
 import (
 	"context"
+	"crypto/rand"
+	"errors"
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -28,10 +31,150 @@ import (
 	"github.com/pilosa/pilosa/v2/http"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pilosa/pilosa/v2/test"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 3,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node2"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
+
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "i"
+		fieldName := "f"
+		attrKey := "k"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, indexName, fieldName)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs0,
+			AttrVals:       attrVals0,
+			Shard:          0,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs1,
+			AttrVals:       attrVals1,
+			Shard:          1,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		m := len(res.ColumnAttrSets)
+		if m != 100 {
+			t.Fatalf("incorrect number of column attrs set; m = %v", m)
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
-	c := test.MustRunCluster(t, 2,
+	c := test.MustRunCluster(t, 3,
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node0"),
@@ -46,65 +189,89 @@ func TestAPI_Import(t *testing.T) {
 				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node2"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
-	m1 := c[1]
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "rick"
+		indexName := "rick"
-		field := "f"
+		fieldName := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
 
 		rowID := uint64(1)
 		timestamp := int64(0)
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
-		for i := 1; i <= 10; i++ {
+		N := 10
+		for i := 1; i <= N; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
-		// Import data with keys to the coordinator (node0) and verify that it gets
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		colKeys = colKeys[:N]
+
+		// Import data with keys to the primary and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
-			Index:      index,
+			Index:          indexName,
-			Field:      field,
+			IndexCreatedAt: index.CreatedAt(),
-			Shard:      0,
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making lots of other shards? b/c this is not a restriction.
 			RowIDs:         rowIDs,
 			ColumnKeys:     colKeys,
 			Timestamps:     timestamps,
 		}
-		if err := m0.API.Import(ctx, req); err != nil {
+
+		qcx := m0.API.Txf().NewQcx()
+
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
 			t.Fatal(err)
 		}
+		PanicOn(qcx.Finish())
 
-		pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 			t.Fatal(err)
 		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
-			t.Fatalf("unexpected column keys: %#v", keys)
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
 		}
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 				return err
 			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 				return fmt.Errorf("unexpected column keys: %#v", keys)
@@ -118,7 +285,11 @@ func TestAPI_Import(t *testing.T) {
 	// Relies on the previous test creating an index with TrackExistence and
 	// adding some data.
 	t.Run("SchemaHasNoExists", func(t *testing.T) {
-		schema := m1.API.Schema(context.Background())
+		schema, err := m1.API.Schema(context.Background(), false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		for _, f := range schema[0].Fields {
 			if f.Name == "_exists" {
 				t.Fatalf("found _exists field in schema")
@@ -127,64 +298,11 @@ func TestAPI_Import(t *testing.T) {
 				t.Fatalf("found internal field '%s' in schema output", f.Name)
 			}
 		}
-
-	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
 	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
-	c := test.MustRunCluster(t, 2,
+	c := test.MustRunCluster(t, 3,
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node0"),
@@ -197,35 +315,44 @@ func TestAPI_ImportValue(t *testing.T) {
 				pilosa.OptServerClusterHasher(&offsetModHasher{}),
 				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node2"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	coord := c.GetPrimary()
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
+	m2 := c.GetNode(2)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
 		index := "valck"
 		field := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
+		_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		_, err = coord.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
-		// Import data with keys to the coordinator (node0) and verify that it gets
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the primary and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
 			Index:      index,
@@ -233,9 +360,12 @@ func TestAPI_ImportValue(t *testing.T) {
 			ColumnKeys: colKeys,
 			Values:     values,
 		}
-		if err := m0.API.ImportValue(ctx, req); err != nil {
+
+		qcx := coord.API.Txf().NewQcx()
+		if err := coord.API.ImportValue(ctx, qcx, req); err != nil {
 			t.Fatal(err)
 		}
+		PanicOn(qcx.Finish())
 
 		pql := fmt.Sprintf("Row(%s>0)", field)
 
@@ -258,6 +388,167 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+		_, err := m2.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m2.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		// Generate some records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+		// Import data with keys to node1 and verify that it gets translated and
+		// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+		query := fmt.Sprintf("Row(%s>6)", field)
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: observerd %+v;  expected '%+v'", ids, colIDs[6:])
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValTimestampField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valts"
+		field := "fts"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeTimestamp(pilosa.MinTimestamp, pilosa.MaxTimestamp, pilosa.TimeUnitSeconds))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some records.
+		values := []time.Time{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, pilosa.MinTimestamp.Add(time.Duration(i)*time.Second))
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to node1 and verify that it gets translated and
+		// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:           index,
+			Field:           field,
+			ColumnIDs:       colIDs,
+			TimestampValues: values,
+		}
+
+		qcx := m2.API.Txf().NewQcx()
+		if err := m2.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+
+		query := fmt.Sprintf("Row(%s>='1833-11-24T17:31:50Z')", field) // 6s after MinTimestamp
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: observerd %+v;  expected '%+v'", ids, colIDs[6:])
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = coord.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = coord.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the node0 and verify that it gets translated
+		// and forwarded to the owner of shard 0 (node1; because of
+		// offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node1.
+		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: observerd %+v;  expected '%+v'", ids, []uint64{1})
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.
@@ -266,3 +557,256 @@ type offsetModHasher struct{}
 func (*offsetModHasher) Hash(key uint64, n int) int {
 	return int(key+1) % n
 }
+
+func (*offsetModHasher) Name() string { return "mod" }
+
+func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	// plan:
+	//  1. set a bit
+	//  2. clear with Import() using the ImportRequest.Clear flag
+	//  3. verifiy the clear is done.
+	//  repeat for ImportValueRequest and ImportValues()
+
+	m0 := c.GetNode(0)
+	m0api := m0.API
+
+	ctx := context.Background()
+	index := "i"
+	fieldAcct0 := "acct0"
+
+	opts := pilosa.OptFieldTypeInt(-1000, 1000)
+
+	_, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = m0api.CreateField(ctx, index, fieldAcct0, opts)
+	if err != nil {
+		t.Fatalf("creating fieldAcct0: %v", err)
+	}
+
+	iraField := "ira" // set field.
+	iraRowID := uint64(3)
+	_, err = m0api.CreateField(ctx, index, iraField)
+	if err != nil {
+		t.Fatalf("creating fieldIRA: %v", err)
+	}
+
+	acctOwnerID := uint64(78) // ColumnID
+	shard := acctOwnerID / ShardWidth
+	acct0bal := int64(500)
+
+	ivr0 := &pilosa.ImportValueRequest{
+		Index:     index,
+		Field:     fieldAcct0,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		Values:    []int64{acct0bal},
+	}
+	ir0 := &pilosa.ImportRequest{
+		Index:     index,
+		Field:     iraField,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		RowIDs:    []uint64{iraRowID},
+	}
+
+	qcx := m0api.Txf().NewQcx()
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	PanicOn(qcx.Finish())
+
+	bitIsSet := func() bool {
+		query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		PanicOn(err)
+		cols := res.Results[0].(*pilosa.Row).Columns()
+		for i := range cols {
+			if cols[i] == acctOwnerID {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !bitIsSet() {
+		PanicOn("IRA bit should have been set")
+	}
+
+	queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
+		query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		PanicOn(err)
+
+		if len(res.Results) == 0 {
+			return 0
+		}
+		valCount := res.Results[0].(pilosa.ValCount)
+		return valCount.Val
+	}
+
+	bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+
+	if bal != acct0bal {
+		PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
+	}
+
+	// clear the bit
+	qcx = m0api.Txf().NewQcx()
+	ir0.Clear = true
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	PanicOn(qcx.Finish())
+
+	if bitIsSet() {
+		PanicOn("IRA bit should have been cleared")
+	}
+
+	// clear the BSI
+	qcx = m0api.Txf().NewQcx()
+	ivr0.Clear = true
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	PanicOn(qcx.Finish())
+
+	bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+	if bal != 0 {
+		PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
+	}
+}
+
+func TestAPI_IDAlloc(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	primary := c.GetPrimary().API
+
+	t.Run("Normal", func(t *testing.T) {
+		key := pilosa.IDAllocKey{
+			Index: "normal",
+			Key:   "key",
+		}
+		var session [32]byte
+		_, err := rand.Read(session[:])
+		if err != nil {
+			t.Fatalf("obtaining random bytes: %v", err)
+		}
+
+		const toReserve = 2
+
+		ids, err := primary.ReserveIDs(key, session, ^uint64(0), toReserve)
+		if err != nil {
+			t.Fatalf("reserving IDs: %v", err)
+		}
+
+		var numIds uint64
+		for _, idr := range ids {
+			numIds += (idr.Last - idr.First) + 1
+		}
+		if numIds != toReserve {
+			t.Errorf("expected %d ids but got %d: %v", toReserve, numIds, ids)
+		}
+
+		err = primary.CommitIDs(key, session, numIds)
+		if err != nil {
+			t.Fatalf("committing IDs: %v", err)
+		}
+
+		err = primary.ResetIDAlloc(key.Index)
+		if err != nil {
+			t.Fatalf("resetting ID alloc: %v", err)
+		}
+	})
+	t.Run("Offset", func(t *testing.T) {
+		key := pilosa.IDAllocKey{
+			Index: "offset",
+			Key:   "key",
+		}
+		var session [32]byte
+		_, err := rand.Read(session[:])
+		if err != nil {
+			t.Fatalf("obtaining random bytes: %v", err)
+		}
+
+		ids, err := primary.ReserveIDs(key, session, 0, 2)
+		if err != nil {
+			t.Fatalf("reserving IDs: %v", err)
+		}
+
+		{
+			var numIds uint64
+			for _, idr := range ids {
+				numIds += (idr.Last - idr.First) + 1
+			}
+			if numIds != 2 {
+				t.Errorf("expected %d ids but got %d: %v", 2, numIds, ids)
+			}
+		}
+
+		_, err = rand.Read(session[:])
+		if err != nil {
+			t.Fatalf("obtaining random bytes: %v", err)
+		}
+		ids2, err := primary.ReserveIDs(key, session, 1, 2)
+		if err != nil {
+			t.Fatalf("reserving IDs with partially increased offset: %v", err)
+		}
+
+		var numIds uint64
+		for _, idr := range ids2 {
+			numIds += (idr.Last - idr.First) + 1
+		}
+		if numIds != 2 {
+			t.Errorf("expected %d ids but got %d: %v", 2, numIds, ids2)
+		}
+
+		if prevEnd, newStart := ids[len(ids)-1].Last, ids2[0].First; prevEnd != newStart {
+			t.Errorf("expected reuse of last ID (%d), but started with %d", prevEnd, newStart)
+		}
+
+		err = primary.CommitIDs(key, session, numIds)
+		if err != nil {
+			t.Errorf("committing IDs: %v", err)
+		}
+
+		_, err = rand.Read(session[:])
+		if err != nil {
+			t.Fatalf("obtaining random bytes: %v", err)
+		}
+		ids3, err := primary.ReserveIDs(key, session, 0, 2)
+		var esync pilosa.ErrIDOffsetDesync
+		if errors.As(err, &esync) {
+			if esync.Requested != 0 {
+				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)
+			}
+			if esync.Base != 3 {
+				t.Errorf("incorrect base offset: expected %d but got %d", 3, esync.Base)
+			}
+		} else if err == nil {
+			t.Errorf("successfully re-reserved at a committed offset: %v", ids3)
+		} else {
+			t.Fatalf("unexpected error when reserving committed IDs: %v", err)
+		}
+
+		err = primary.ResetIDAlloc(key.Index)
+		if err != nil {
+			t.Fatalf("resetting ID alloc: %v", err)
+		}
+	})
+}

apimethod_string.go

@@ -19,25 +19,36 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
+	_ = x[apiTranslateData-11]
-	_ = x[apiFieldAttrDiff-12]
+	_ = x[apiField-12]
-	_ = x[apiImport-13]
+	_ = x[apiFieldAttrDiff-13]
-	_ = x[apiImportValue-14]
+	_ = x[apiImport-14]
-	_ = x[apiIndex-15]
+	_ = x[apiImportValue-15]
-	_ = x[apiIndexAttrDiff-16]
+	_ = x[apiIndex-16]
-	_ = x[apiQuery-17]
+	_ = x[apiIndexAttrDiff-17]
-	_ = x[apiRecalculateCaches-18]
+	_ = x[apiQuery-18]
-	_ = x[apiRemoveNode-19]
+	_ = x[apiRecalculateCaches-19]
-	_ = x[apiResizeAbort-20]
+	_ = x[apiRemoveNode-20]
-	_ = x[apiSetCoordinator-21]
+	_ = x[apiResizeAbort-21]
-	_ = x[apiShardNodes-22]
+	_ = x[apiSchema-22]
-	_ = x[apiViews-23]
+	_ = x[apiShardNodes-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiState-24]
+	_ = x[apiViews-25]
+	_ = x[apiApplySchema-26]
+	_ = x[apiStartTransaction-27]
+	_ = x[apiFinishTransaction-28]
+	_ = x[apiTransactions-29]
+	_ = x[apiGetTransaction-30]
+	_ = x[apiActiveQueries-31]
+	_ = x[apiPastQueries-32]
+	_ = x[apiIDReserve-33]
+	_ = x[apiIDCommit-34]
+	_ = x[apiIDReset-35]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 324, 337, 345, 353, 367, 386, 406, 421, 438, 454, 468, 480, 491, 501}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

attr.go

@@ -19,7 +19,7 @@ import (
 	"sort"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/v2/internal"
+	"github.com/pilosa/pilosa/v2/pb"
 )
 
 // Attribute data type enum.
@@ -119,21 +119,21 @@ func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
 	}
 }
 
-func encodeAttrs(m map[string]interface{}) []*internal.Attr {
+func encodeAttrs(m map[string]interface{}) []*pb.Attr {
 	keys := make([]string, 0, len(m))
 	for k := range m {
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)
 
-	a := make([]*internal.Attr, len(keys))
+	a := make([]*pb.Attr, len(keys))
 	for i := range keys {
 		a[i] = encodeAttr(keys[i], m[keys[i]])
 	}
 	return a
 }
 
-func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
+func decodeAttrs(pb []*pb.Attr) map[string]interface{} {
 	m := make(map[string]interface{}, len(pb))
 	for i := range pb {
 		key, value := decodeAttr(pb[i])
@@ -142,9 +142,9 @@ func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
 	return m
 }
 
-// encodeAttr converts a key/value pair into an Attr internal representation.
+// encodeAttr converts a key/value pair into an Attr pb.representation.
-func encodeAttr(key string, value interface{}) *internal.Attr {
+func encodeAttr(key string, value interface{}) *pb.Attr {
-	pb := &internal.Attr{Key: key}
+	pb := &pb.Attr{Key: key}
 	switch value := value.(type) {
 	case string:
 		pb.Type = attrTypeString
@@ -165,8 +165,8 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
 	return pb
 }
 
-// decodeAttr converts from an Attr internal representation to a key/value pair.
+// decodeAttr converts from an Attr pb.representation to a key/value pair.
-func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
+func decodeAttr(attr *pb.Attr) (key string, value interface{}) {
 	switch attr.Type {
 	case attrTypeString:
 		return attr.Key, attr.StringValue
@@ -192,12 +192,12 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} {
 
 // EncodeAttrs encodes an attribute map into a byte slice.
 func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
-	return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
+	return proto.Marshal(&pb.AttrMap{Attrs: encodeAttrs(attr)})
 }
 
 // DecodeAttrs decodes a byte slice into an  attribute map.
 func DecodeAttrs(v []byte) (map[string]interface{}, error) {
-	var pb internal.AttrMap
+	var pb pb.AttrMap
 	if err := proto.Unmarshal(v, &pb); err != nil {
 		return nil, err
 	}

attr_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"os"
 	"reflect"
 	"runtime"
@@ -24,11 +23,12 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure database can set and retrieve column attributes.
 func TestAttrStore_Attrs(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -57,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) {
 
 // Ensure database returns a non-nil empty map if unset.
 func TestAttrStore_Attrs_Empty(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	if m, err := s.Attrs(100); err != nil {
@@ -69,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
 
 // Ensure database can unset attributes if explicitly set to nil.
 func TestAttrStore_Attrs_Unset(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -89,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
 
 // Ensure attribute block checksums can be returned.
 func TestAttrStore_Blocks(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -135,19 +135,22 @@ type AttrStore struct {
 }
 
 // NewAttrStore returns a new instance of AttrStore.
-func NewAttrStore(string) pilosa.AttrStore {
+func NewAttrStore(tb testing.TB) pilosa.AttrStore {
-	f, err := ioutil.TempFile("", "pilosa-attr-")
+	f, err := testhook.TempFile(tb, "pilosa-attr-")
 	if err != nil {
 		panic(err)
 	}
+	// Note, even though the file is closed, TempFile will still avoid
+	// creating the same name again if we leave it existing. The boltdb
+	// code may already be deleting this, so the TestHook deletion
+	// may not matter but it's more reliable this way.
 	f.Close()
-	os.Remove(f.Name())
 
 	return &AttrStore{boltdb.NewAttrStore(f.Name())}
 }
 
 func BenchmarkAttrStore_Duplicate(b *testing.B) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(b)
 	defer s.Close()
 
 	// Set attributes.
@@ -186,8 +189,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) {
 }
 
 // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
-func MustOpenAttrStore() pilosa.AttrStore {
+func MustOpenAttrStore(tb testing.TB) pilosa.AttrStore {
-	s := NewAttrStore("")
+	s := NewAttrStore(tb)
 	if err := s.Open(); err != nil {
 		panic(err)
 	}

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

barrier.go

@@ -0,0 +1,219 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"github.com/glycerine/idem"
+)
+
+// Barrier allows us to temporarily halt all readers, so that
+// a writer can commit alone and thus compact the db.
+// The Barrier starts unblocked, alllowing passage to any
+// caller of WaitAtGate().
+type Barrier struct {
+	wait       chan *appointment // send upon entering the waiting room.
+	halt       *idem.Halter
+	blockReqCh chan *blockReq
+	unblockCh  chan *unblock
+}
+
+type blockReq struct {
+	count int
+	done  chan struct{}
+}
+
+func newBlockReq(count int) *blockReq {
+	return &blockReq{
+		count: count,
+		done:  make(chan struct{}),
+	}
+}
+
+type appointment struct {
+	id   int
+	done chan struct{}
+}
+
+func newAppointment(id int) *appointment {
+	return &appointment{
+		id:   id,
+		done: make(chan struct{}),
+	}
+}
+
+// NewBarrier is either open, allowing immediate passage,
+// or blocked, halting all callers at WaitAtGate()
+// until the barrier is opened. By default it is open.
+//
+// Barrier.Close() must be called when the barrier
+// is no longer needed to avoid a goroutine leak.
+func NewBarrier() (b *Barrier) {
+	b = &Barrier{
+		wait:       make(chan *appointment), // waiters indicate they are waiting for the gate by sending here.
+		halt:       idem.NewHalter(),
+		blockReqCh: make(chan *blockReq),
+		unblockCh:  make(chan *unblock),
+	}
+	go func() {
+		defer b.halt.Done.Close()
+
+		var waitlist []*appointment
+		var curBlockReq *blockReq
+
+		for {
+			select {
+			case br := <-b.blockReqCh:
+				if br.count == 0 {
+					close(br.done)
+					continue
+				}
+				if curBlockReq == nil {
+					// good, changing state from open to closed barrier.
+				} else {
+					panic("got 2nd block request atop of first")
+				}
+				curBlockReq = br
+				//vv("barrier: request to block for %v waiters", br.count)
+				if len(waitlist) != 0 {
+					panic("had waiters when we were open, internal/client bug")
+				}
+			case appt := <-b.wait:
+				//vv("barrier.wait sees appt = '%#v' and  curBlockReq = '%#v'", appt, curBlockReq)
+				if curBlockReq == nil {
+					close(appt.done)
+					continue
+				}
+				waitlist = append(waitlist, appt)
+				n := len(waitlist)
+				th := curBlockReq.count
+				if th < 0 {
+					// infinite waiters. we block everybody until we
+					// see an unblock request.
+					continue
+				}
+				if n >= th {
+					close(curBlockReq.done)
+					curBlockReq = nil
+				}
+			case ub := <-b.unblockCh:
+				for _, appt := range waitlist {
+					close(appt.done)
+				}
+				waitlist = nil
+				curBlockReq = nil
+				close(ub.done)
+			case <-b.halt.ReqStop.Chan:
+				return
+			}
+		}
+	}()
+	return
+}
+
+// WaitAtGate will return immediately
+// if the barrier is unblocked. Otherwise
+// it will not return until another
+// goroutine unblocks the barrier.
+func (b *Barrier) WaitAtGate(id int) {
+	appt := newAppointment(id)
+	select {
+	case b.wait <- appt:
+		select {
+		case <-appt.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// Close should be called to stop the
+// barrier's background goroutine when
+// you are done using the barrier.
+func (b *Barrier) Close() {
+	b.halt.ReqStop.Close()
+	<-b.halt.Done.Chan
+}
+
+type unblock struct {
+	done chan struct{}
+}
+
+func newUnblock() *unblock {
+	return &unblock{
+		done: make(chan struct{}),
+	}
+}
+
+// Unblock lets all waiting goroutines resume execution.
+func (b *Barrier) UnblockReaders() {
+	ub := newUnblock()
+	select {
+	case b.unblockCh <- ub:
+		select {
+		case <-ub.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// BlockUntil is called with a count, the
+// number of waiters required to be present and waiting
+// at the gate before call returns.
+// A count of < 0 will return immediately and raise
+// the barrier to any number of arriving readers.
+// A count of 0 is a no-op.
+//
+// Otherwise we raise the barrier
+// and wait until we have seen count other goroutines waiting
+// on it.
+//
+// We return without releasing the waiters. Call
+// Open when you want them to resume.
+func (b *Barrier) BlockUntil(count int) {
+	if count == 0 {
+		return
+	}
+	req := newBlockReq(count)
+	b.blockReqCh <- req
+	if count > 0 {
+		<-req.done
+	}
+}
+
+// BlockAllReadersNoWait raises the barrier to
+// an infinite number of waiters and returns immediately
+// to the caller.
+func (b *Barrier) BlockAllReadersNoWait() {
+	req := newBlockReq(-1) // -1 means block any number of readers.
+	b.blockReqCh <- req
+	// don't wait. <-req.done
+}

barrier_test.go

@@ -0,0 +1,90 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestBarrierHolds(t *testing.T) {
+	b := NewBarrier()
+	defer b.Close()
+
+	released := int64(0)
+
+	waiter := func(i int) {
+		b.WaitAtGate(i)
+		//vv("goro %v is released", i)
+		atomic.AddInt64(&released, 1)
+	}
+
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+	time.Sleep(time.Second)
+	r := atomic.SwapInt64(&released, 0)
+	if r != 3 {
+		panic("open barrier held back goro")
+	}
+	//vv("good: barrier started open")
+
+	//seenAll := make(chan bool)
+	b.BlockAllReadersNoWait()
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic("bad: barrier did not hold back goro")
+	}
+	//vv("good: barrier of 4 did not release on 3")
+	go waiter(4)
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r))
+	}
+
+	b.UnblockReaders()
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 4 {
+		panic("bad: unblock should have released 4 goro")
+	}
+
+}

bluegreentx_test.go

@@ -0,0 +1,100 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	cryrand "crypto/rand"
+
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+var _ = context.Background
+var _ = os.Open
+var _ = strings.Split
+
+func TestMultiReaderB(t *testing.T) {
+	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
+	// member io.Readers, else it should panic. This should hold for
+	// varying sizes of inputs.
+
+	for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
+		src := io.LimitReader(cryrand.Reader, int64(n))
+
+		a := make([]byte, n)
+		nr := 0
+		for nr < n {
+			na, err := src.Read(a)
+			PanicOn(err)
+			nr += na
+		}
+		if nr != n {
+			panic("short read")
+		}
+
+		b := make([]byte, n)
+		copy(b, a)
+		if !bytes.Equal(a, b) {
+			panic("test prep failed")
+		}
+
+		m := &MultiReaderB{
+			a: ioutil.NopCloser(bytes.NewBuffer(a)),
+			b: ioutil.NopCloser(bytes.NewBuffer(b)),
+		}
+
+		// should not trigger the internal panic of MultiReadB
+		ncp, err := io.Copy(ioutil.Discard, m)
+		PanicOn(err)
+		if ncp != int64(n) {
+			panic("short copy")
+		}
+
+		for victim := 0; victim < n; victim += 7 {
+
+			copy(b, a)
+			if victim%2 == 0 {
+				// corrupt b
+				b[victim] = (b[victim] + 1) % 255
+			} else {
+				// corrupt a
+				a[victim] = (a[victim] + 1) % 255
+			}
+			m = &MultiReaderB{
+				a: ioutil.NopCloser(bytes.NewBuffer(a)),
+				b: ioutil.NopCloser(bytes.NewBuffer(b)),
+			}
+			helperShouldPanicOnCopy(m)
+		}
+	}
+}
+
+func helperShouldPanicOnCopy(m *MultiReaderB) {
+	// differences in bytes read should be noticed
+	defer func() {
+		r := recover()
+		if r == nil {
+			panic("expected panic on byte difference but didn't see it")
+		}
+	}()
+	_, _ = io.Copy(ioutil.Discard, m)
+}

boltdb/attrstore.go

@@ -25,9 +25,9 @@ import (
 
 	"github.com/cespare/xxhash"
 
-	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 // attrBlockSize is the size of attribute blocks for anti-entropy.

boltdb/translate.go

@@ -11,29 +11,51 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
+
+	"runtime/pprof"
 )
 
+var _ = pprof.StartCPUProfile
+
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
+
+	// ErrTranslateKeyNotFound is returned when translating key
+	// and the underlying store returns an empty set
+	ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set")
+
+	bucketKeys = []byte("keys")
+	bucketIDs  = []byte("ids")
+)
+
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+
+	errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found"
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +67,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
 	index       string
 	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +95,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -74,6 +108,15 @@ func NewTranslateStore(index, field string) *TranslateStore {
 
 // Open opens the translate file.
 func (s *TranslateStore) Open() (err error) {
+
+	// add the path to the problem database if we panic handling it.
+	defer func() {
+		r := recover()
+		if r != nil {
+			panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r))
+		}
+	}()
+
 	if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
 	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
@@ -82,9 +125,9 @@ func (s *TranslateStore) Open() (err error) {
 
 	// Initialize buckets.
 	if err := s.db.Update(func(tx *bolt.Tx) error {
-		if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil {
+		if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
 			return err
-		} else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil {
+		} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
 			return err
 		}
 		return nil
@@ -108,6 +151,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,105 +183,155 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKey converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
+// If key does not have an associated id then one is created, unless writable is false,
-func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
-	// Find id by key under read lock.
+func (s *TranslateStore) TranslateKey(key string, writable bool) (uint64, error) {
-	if err := s.db.View(func(tx *bolt.Tx) error {
+	ids, err := s.translateKeys([]string{key}, writable)
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
+	if err != nil {
-		return nil
-	}); err != nil {
 		return 0, err
-	} else if id != 0 {
+	}
-		return id, nil
+	if len(ids) == 0 {
+		return 0, ErrTranslateKeyNotFound
+	}
+	return ids[0], nil
+}
+
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
+// If a key does not have an associated id then one is created, unless writable is false,
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
+func (s *TranslateStore) TranslateKeys(keys []string, writable bool) ([]uint64, error) {
+	return s.translateKeys(keys, writable)
+}
+
+// FindKeys looks up the ID for each key.
+// Keys are not created if they do not exist.
+// Missing keys are not considered errors, so the length of the result may be less than that of the input.
+func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
+	result := make(map[string]uint64, len(keys))
+	err := s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
+		}
+		for _, key := range keys {
+			id, _ := findIDByKey(bkt, key)
+			if id == 0 {
+				// The key does not exist.
+				continue
 			}
 
-	if s.ReadOnly() {
+			result[key] = id
-		return 0, pilosa.ErrTranslateStoreReadOnly
 		}
-
-	// Find or create id under write lock.
-	var written bool
-	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
 		return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+// CreateKeys maps all keys to IDs, creating the IDs if they do not exist.
+// If the translator is read-only, this will return an error.
+func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
+	if s.ReadOnly() {
+		return nil, pilosa.ErrTranslateStoreReadOnly
+	}
+
+	written := false
+	result := make(map[string]uint64, len(keys))
+	err := s.db.Update(func(tx *bolt.Tx) error {
+		keyBucket := tx.Bucket(bucketKeys)
+		if keyBucket == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
+		}
+		idBucket := tx.Bucket(bucketIDs)
+		if keyBucket == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
+		}
+		for _, key := range keys {
+			id, boltKey := findIDByKey(keyBucket, key)
+			if id == 0 {
+				// The key does not exist.
+				id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+				if err := keyBucket.Put(boltKey, u64tob(id)); err != nil {
 					return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
+				} else if err := idBucket.Put(u64tob(id), boltKey); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
 					return err
 				}
 				written = true
-		return nil
-	}); err != nil {
-		return 0, err
 			}
 
+			result[key] = id
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
 	if written {
 		s.notifyWrite()
 	}
 
-	return id, nil
+	return result, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
+func (s *TranslateStore) translateKeys(keys []string, writable bool) ([]uint64, error) {
-// If key does not have an associated id then one is created.
+	ids := make([]uint64, 0, len(keys))
-func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
-	if len(keys) == 0 {
-		return nil, nil
-	}
 
-	// Allocate slice for ID mapping.
+	if s.ReadOnly() || !writable {
-	ids = make([]uint64, len(keys))
+		found := 0
-
-	// Find ids by key under read lock.
-	var found int
 		if err := s.db.View(func(tx *bolt.Tx) error {
-		bkt := tx.Bucket([]byte("keys"))
+			bkt := tx.Bucket(bucketKeys)
-		for i, key := range keys {
+			if bkt == nil {
-			if id := findIDByKey(bkt, key); id != 0 {
+				return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
-				ids[i] = id
+			}
+			for _, key := range keys {
+				if id, _ := findIDByKey(bkt, key); id != 0 {
+					ids = append(ids, id)
 					found++
 				}
 			}
 			return nil
 		}); err != nil {
 			return nil, err
-	} else if found == len(keys) {
+		}
+		if found == len(keys) {
 			return ids, nil
 		}
-
 		if s.ReadOnly() {
 			return ids, pilosa.ErrTranslateStoreReadOnly
 		}
+		if !writable {
+			return nil, pilosa.ErrTranslatingKeyNotFound
+		}
+		return nil, nil
+	}
 
 	// Find or create ids under write lock if any keys were not found.
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
+		bkt := tx.Bucket(bucketKeys)
-		for i, key := range keys {
+		for _, key := range keys {
-			if ids[i] != 0 {
+			id, boltKey := findIDByKey(bkt, key)
+			if id != 0 {
+				ids = append(ids, id)
 				continue
 			}
-
+			id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
+			if err := bkt.Put(boltKey, u64tob(id)); err != nil {
-				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
+			} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), boltKey); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
 				return err
 			}
+			ids = append(ids, id)
 			written = true
 		}
 		return nil
 	}); err != nil {
 		return nil, err
 	}
-
 	if written {
 		s.notifyWrite()
 	}
@@ -248,7 +346,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) {
 		return "", err
 	}
 	defer func() { _ = tx.Rollback() }()
-	return findKeyByID(tx.Bucket([]byte("ids")), id), nil
+	return findKeyByID(tx.Bucket(bucketIDs), id), nil
 }
 
 // TranslateIDs converts a list of integer IDs to a list of string keys.
@@ -263,9 +361,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 	}
 	defer func() { _ = tx.Rollback() }()
 
+	bucket := tx.Bucket(bucketIDs)
+
 	keys := make([]string, len(ids))
 	for i, id := range ids {
-		keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id)
+		keys[i] = findKeyByID(bucket, id)
 	}
 	return keys, nil
 }
@@ -273,9 +373,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 // ForceSet writes the id/key pair to the store even if read only. Used by replication.
 func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil {
+		if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
 			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil {
 			return err
 		}
 		return nil
@@ -286,7 +386,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +411,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		max = maxID(tx)
-			max = btou64(key)
-		}
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +419,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -353,7 +506,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 		var found bool
 		if err := r.store.db.View(func(tx *bolt.Tx) error {
 			// Find ID/key lookup at offset or later.
-			cur := tx.Bucket([]byte("ids")).Cursor()
+			cur := tx.Bucket(bucketIDs).Cursor()
 			key, value := cur.Seek(u64tob(r.offset))
 			if key == nil {
 				return nil
@@ -387,13 +540,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
+// emptyKey is a sentinel byte slice which stands for "" as a key.
-	if value := bkt.Get([]byte(key)); value != nil {
+var emptyKey = []byte{
-		return btou64(value)
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }

boltdb/translate_test.go

@@ -14,112 +14,241 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
-	"io/ioutil"
+	"fmt"
-	"os"
+	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pilosa/pilosa/v2/topology"
 )
 
+//var vv = pilosa.VV
+
 func TestTranslateStore_TranslateKey(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
+	// Ensure initial key translates to first ID for shard
-	if id, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
-	if id, err := s.TranslateKey("bar"); err != nil {
+	if id, err := s.TranslateKey("bar", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	if id, err := s.TranslateKey("foo", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
 
 func TestTranslateStore_TranslateKeys(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial keys translate to incrementing IDs.
+	ids, err := s.TranslateKeys([]string{"abc", "abc"}, true)
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[1], ids[0]; got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
+		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	}
+
+	// Ensure initial keys translate to incrementing IDs.
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
-	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
 
+func TestTranslateStore_CreateKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.CreateKeys("abc", "abc")
+	if err != nil {
+		t.Fatal(err)
+	} else if _, ok := ids["abc"]; !ok {
+		t.Fatalf(`missing "abc"; got %v`, ids)
+	} else if len(ids) > 1 {
+		t.Fatalf("expected one key, got %d in %v", len(ids), ids)
+	}
+
+	// Ensure different keys translate to different IDs.
+	ids1, err := s.CreateKeys("foo", "bar")
+	if err != nil {
+		t.Fatal(err)
+	} else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar {
+		t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo)
+	}
+
+	// Ensure retranslation returns original IDs.
+	if ids, err := s.CreateKeys("bar", "foo"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, ids1) {
+		t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1)
+	}
+
+	// Ensure retranslating with existing and non-existing keys returns correctly.
+	if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil {
+		t.Fatal(err)
+	} else if got, want := ids["foo"], ids1["foo"]; got != want {
+		t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want)
+	} else if _, ok := ids["baz"]; !ok {
+		t.Fatalf(`missing translation for "baz"; got %v`, ids)
+	} else if got, want := ids["bar"], ids1["bar"]; got != want {
+		t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want)
+	}
+}
+
+func TestTranslateStore_ReadKey(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	id, err := s.TranslateKey("foo", false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+
+	s.SetReadOnly(true)
+	id, err = s.TranslateKey("foo", true)
+	if err == nil {
+		t.Fatalf("got error: %+v, want: 'translate store read only'", err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+	s.SetReadOnly(false)
+
+	// Ensure next key autoincrements.
+	if id, err = s.TranslateKey("foo", true); err != nil {
+		t.Fatal(err)
+	}
+	id1, err := s.TranslateKey("foo", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id1 != id {
+		t.Fatalf("TranslateKey()=%d, want %d", id1, id)
+	}
+}
+
+func TestTranslateStore_ReadKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	for _, id := range ids {
+		if id != 0 {
+			t.Fatalf("TranslateKeys()=%d, want %d", id, 0)
+		}
+	}
+
+	// Ensure next key autoincrements.
+	if ids, err = s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, true); err != nil {
+		t.Fatal(err)
+	}
+	ids1, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range ids1 {
+		if ids1[i] != ids[i] {
+			t.Fatalf("TranslateKeys()=%d, want %d", ids1[i], ids[i])
+		}
+	}
+}
 func TestTranslateStore_TranslateID(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("", true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -130,13 +259,116 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	}
 }
 
+func TestTranslateStore_FindKeys(t *testing.T) {
+	cases := []struct {
+		name   string
+		data   []string
+		lookup []string
+	}{
+		{
+			name:   "All",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h"},
+		},
+		{
+			name:   "Extra",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h", "65"},
+		},
+		{
+			name:   "None",
+			data:   []string{"a", "b", "c"},
+			lookup: []string{"d", "e"},
+		},
+		{
+			name:   "Empty",
+			lookup: []string{"h"},
+		},
+		{
+			name: "LookupNothing",
+		},
+	}
+
+	for _, c := range cases {
+		c := c
+		t.Run(c.name, func(t *testing.T) {
+			s := MustOpenNewTranslateStore(t)
+			defer MustCloseTranslateStore(s)
+
+			var naiveMap map[string]uint64
+			if c.data != nil {
+				// Load in key data.
+				keys := c.data
+				ids, err := s.TranslateKeys(keys, true)
+				if err != nil {
+					t.Errorf("failed to import keys: %v", err)
+					return
+				}
+				if len(ids) != len(keys) {
+					t.Errorf("mapped %d keys to %d ids", len(keys), len(ids))
+					return
+				}
+				naiveMap = make(map[string]uint64, len(keys))
+				for i, key := range keys {
+					naiveMap[key] = ids[i]
+				}
+			}
+
+			// Compute expected lookup result.
+			result := map[string]uint64{}
+			for _, key := range c.lookup {
+				id, ok := naiveMap[key]
+				if !ok {
+					// The key is expected to be missing.
+					continue
+				}
+
+				result[key] = id
+			}
+
+			// Find the keys.
+			found, err := s.FindKeys(c.lookup...)
+			if err != nil {
+				t.Errorf("failed to find keys: %v", err)
+			} else if !reflect.DeepEqual(result, found) {
+				t.Errorf("expected %v but found %v", result, found)
+			}
+		})
+	}
+}
+
+func TestTranslateStore_MaxID(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	// Generate a bunch of keys.
+	var lastk uint64
+	for i := 0; i < 1026; i++ {
+		k, err := s.TranslateKey(strconv.Itoa(i), true)
+		if err != nil {
+			t.Fatalf("translating %d: %v", i, err)
+		}
+		lastk = k
+	}
+
+	// Verify the max ID.
+	max, err := s.MaxID()
+	if err != nil {
+		t.Fatalf("checking max ID: %v", err)
+	}
+	if max != lastk {
+		t.Fatalf("last key is %d but max is %d", lastk, max)
+	}
+}
+
 func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +383,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +392,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz", true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -188,7 +421,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure reader will read as soon as a new write comes in using WriteNotify().
 	t.Run("WriteNotify", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -198,20 +431,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo", true)
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -226,7 +464,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on close.
 	t.Run("Close", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -260,7 +498,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on store close.
 	t.Run("StoreClose", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -294,22 +532,87 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 }
 
 // MustNewTranslateStore returns a new TranslateStore with a temporary path.
-func MustNewTranslateStore() *boltdb.TranslateStore {
+func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
-	f, err := ioutil.TempFile("", "")
+	f, err := testhook.TempFile(tb, "translate-store")
 	if err != nil {
 		panic(err)
 	} else if err := f.Close(); err != nil {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore(t)
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, true); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
-func MustOpenNewTranslateStore() *boltdb.TranslateStore {
+func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
-	s := MustNewTranslateStore()
+	s := MustNewTranslateStore(tb)
 	if err := s.Open(); err != nil {
 		panic(err)
 	}
@@ -320,7 +623,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 	if err := s.Close(); err != nil {
 		panic(err)
-	} else if err := os.Remove(s.Path); err != nil {
-		panic(err)
 	}
 }

broadcast.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"fmt"
 
+	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pkg/errors"
 )
 
@@ -26,11 +27,22 @@ type Serializer interface {
 	Unmarshal([]byte, Message) error
 }
 
+// NopSerializer represents a Serializer that doesn't do anything.
+var NopSerializer Serializer = &nopSerializer{}
+
+type nopSerializer struct{}
+
+// Marshal is a no-op implementation of Serializer Marshal method.
+func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
+
+// Unmarshal is a no-op implementation of Serializer Unmarshal method.
+func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
+
 // broadcaster is an interface for broadcasting messages.
 type broadcaster interface {
 	SendSync(Message) error
 	SendAsync(Message) error
-	SendTo(*Node, Message) error
+	SendTo(*topology.Node, Message) error
 }
 
 // Message is the interface implemented by all core pilosa types which can be serialized to messages.
@@ -49,7 +61,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
 func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
+func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
@@ -63,12 +75,14 @@ const (
 	messageTypeClusterStatus
 	messageTypeResizeInstruction
 	messageTypeResizeInstructionComplete
-	messageTypeSetCoordinator
-	messageTypeUpdateCoordinator
 	messageTypeNodeState
 	messageTypeRecalculateCaches
+	messageTypeLoadSchemaMessage
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
+	messageTypeResizeNodeMessage
+	messageTypeResizeAbortMessage
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -104,18 +118,22 @@ func getMessage(typ byte) Message {
 		return &ResizeInstruction{}
 	case messageTypeResizeInstructionComplete:
 		return &ResizeInstructionComplete{}
-	case messageTypeSetCoordinator:
-		return &SetCoordinatorMessage{}
-	case messageTypeUpdateCoordinator:
-		return &UpdateCoordinatorMessage{}
 	case messageTypeNodeState:
 		return &NodeStateMessage{}
 	case messageTypeRecalculateCaches:
 		return &RecalculateCaches{}
+	case messageTypeLoadSchemaMessage:
+		return &LoadSchemaMessage{}
 	case messageTypeNodeEvent:
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
+	case messageTypeResizeNodeMessage:
+		return &ResizeNodeMessage{}
+	case messageTypeResizeAbortMessage:
+		return &ResizeAbortMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -143,18 +161,22 @@ func getMessageType(m Message) byte {
 		return messageTypeResizeInstruction
 	case *ResizeInstructionComplete:
 		return messageTypeResizeInstructionComplete
-	case *SetCoordinatorMessage:
-		return messageTypeSetCoordinator
-	case *UpdateCoordinatorMessage:
-		return messageTypeUpdateCoordinator
 	case *NodeStateMessage:
 		return messageTypeNodeState
 	case *RecalculateCaches:
 		return messageTypeRecalculateCaches
+	case *LoadSchemaMessage:
+		return messageTypeLoadSchemaMessage
 	case *NodeEvent:
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
+	case *ResizeNodeMessage:
+		return messageTypeResizeNodeMessage
+	case *ResizeAbortMessage:
+		return messageTypeResizeAbortMessage
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

bsi.go

@@ -0,0 +1,296 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"math/bits"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// bsiData contains BSI-structured data.
+type bsiData []*Row
+
+// pivotDescending loops over nonzero BSI values in descending order.
+// For each value, the provided function is called with the value and a slice of the associated columns.
+// If limit or offset are not-nil, they will be applied.
+// Applying a limit or offset may modify the pointed-to value.
+func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+	// This "pivot" algorithm works by treating the BSI data as a tree.
+	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
+	// Each range is subdivided into 2 ranges of half size, which form lower branches.
+	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
+	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
+	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
+	// This function uses a depth-first search over this virtual tree.
+
+	switch {
+	case !filter.Any():
+		// There are no remaining data.
+
+	case offset != nil && *offset >= filter.Count():
+		// Skip this entire branch.
+		*offset -= filter.Count()
+
+	case limit != nil && *limit == 0:
+		// The limit has been reached.
+		// No more data is necessary.
+
+	case len(bsi) == 0:
+		// This is a leaf node.
+		cols := filter.Columns()
+		if offset != nil {
+			cols = cols[*offset:]
+			*offset = 0
+		}
+		if limit != nil {
+			if *limit < uint64(len(cols)) {
+				cols = cols[:*limit]
+			}
+			*limit -= uint64(len(cols))
+		}
+		fn(branch, cols...)
+
+	default:
+		// Pivot over the highest bit.
+		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
+		splitBit := bsi[len(bsi)-1]
+		lowerBits := bsi[:len(bsi)-1]
+		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+	}
+}
+
+/*
+// distribution generates a BSI histogram for the input.
+// TODO: I forgot what I was going to use this for.
+// Could probbably use this for:
+// - quartile queries
+// - TopN on int
+func (bsi bsiData) distribution(filter *Row) bsiData {
+	var dist bsiData
+	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+		dist.insert(count, uint64(len(values)))
+	})
+	return dist
+}
+*/
+
+var placeholderBitmap = roaring.NewBitmap()
+
+// addBSI adds two BSI bitmaps together.
+// It does not handle sign and has no concept of overflow.
+func addBSI(x, y bsiData) bsiData {
+	// Accumulate row segments.
+	segments := make([][]rowSegment, len(x)+len(y))
+	xsegs, ysegs := segments[:len(x)], segments[len(x):]
+	for i, r := range x {
+		xsegs[i] = r.segments
+	}
+	for i, r := range y {
+		ysegs[i] = r.segments
+	}
+
+	var dst bsiData
+	var xbitmaps, ybitmaps []*roaring.Bitmap
+	for {
+		// Find the next shard.
+		next := ^uint64(0)
+		for _, s := range segments {
+			if len(s) == 0 {
+				continue
+			}
+			shard := s[0].shard
+			if shard < next {
+				next = shard
+			}
+		}
+		if next == ^uint64(0) {
+			// There are no remaining shards.
+			break
+		}
+
+		// Accumulate bitmaps for this shard.
+		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
+		for i, segs := range xsegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			xsegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(xbitmaps) < i {
+				xbitmaps = append(xbitmaps, placeholderBitmap)
+			}
+			xbitmaps = append(xbitmaps, bm)
+		}
+		for i, segs := range ysegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			ysegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(ybitmaps) < i {
+				ybitmaps = append(ybitmaps, placeholderBitmap)
+			}
+			ybitmaps = append(ybitmaps, bm)
+		}
+
+		// Add the shard values together.
+		var out []*roaring.Bitmap
+		switch {
+		case len(xbitmaps) == 0:
+			// There are no values in x.
+			out = ybitmaps
+		case len(ybitmaps) == 0:
+			// There are no values in y.
+			out = xbitmaps
+		default:
+			out = roaring.Add(xbitmaps, ybitmaps)
+		}
+
+		// Convert the bitmaps to output segments.
+		for i, b := range out {
+			if !b.Any() {
+				continue
+			}
+			for len(dst) <= i {
+				dst = append(dst, NewRow())
+			}
+			dst[i].segments = append(dst[i].segments, rowSegment{
+				shard:    next,
+				writable: true,
+				data:     b,
+				n:        b.Count(),
+			})
+		}
+	}
+
+	return dst
+}
+
+// rowBuilder builds a row quickly from individual values.
+// It is optimized for the case in which values are generated sequentially.
+type rowBuilder struct {
+	bm    *roaring.Bitmap
+	mask  *[1024]uint64
+	array []uint16
+	key   uint64
+	n     int32
+}
+
+// flushKey flushes the data at the current key to the bitmap.
+func (b *rowBuilder) flushKey() {
+	var c *roaring.Container
+	switch {
+	case b.mask != nil:
+		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
+		b.mask = nil
+	case len(b.array) > 0:
+		c = roaring.NewContainerArrayCopy(b.array)
+		b.array = b.array[:0]
+	default:
+		return
+	}
+
+	if b.bm == nil {
+		b.bm = roaring.NewBitmap()
+	}
+	if old := b.bm.Containers.Get(b.key); old != nil {
+		c = roaring.Union(c, old)
+	}
+	b.bm.Containers.Put(b.key, c)
+}
+
+// Add a value to the bitmap.
+// Values must be added sequentially.
+func (b *rowBuilder) Add(v uint64) {
+	vkey := v / (1 << 16)
+	if b.key != vkey {
+		// This is a new key, so flush the old one.
+		b.flushKey()
+		b.key = vkey
+	}
+
+	if b.mask != nil {
+		// Add to the mask.
+		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
+		b.mask[uint16(v)/64] |= 1 << (v % 64)
+		return
+	}
+
+	// Add to an array.
+	b.array = append(b.array, uint16(v))
+	if len(b.array) >= roaring.ArrayMaxSize {
+		// The array is too big.
+		// Convert it to a bitmask.
+		m := [1024]uint64{}
+		for _, v := range b.array {
+			m[v/64] |= 1 << (v % 64)
+		}
+		b.n = int32(len(b.array))
+		b.array = b.array[:0]
+		b.mask = &m
+	}
+}
+
+// Build a Row from stored data.
+// This resets the builder.
+func (b *rowBuilder) Build() *Row {
+	// Flush the active key to the bitmap.
+	b.flushKey()
+
+	// Remove the bitmap and convert it to a Row.
+	bm := b.bm
+	b.bm = nil
+	if bm == nil {
+		return NewRow()
+	}
+	return NewRowFromBitmap(bm)
+}
+
+// bsiBuilder assembles BSI data.
+// It is optimized for the case in which values are generated sequentially.
+type bsiBuilder []rowBuilder
+
+// Insert a value into the BSI data.
+// Columns must be inserted sequentially, and duplicates are not allowed.
+func (b *bsiBuilder) Insert(col, val uint64) {
+	for val != 0 {
+		i := bits.TrailingZeros64(val)
+		val &^= 1 << i
+		for len(*b) <= i {
+			*b = append(*b, rowBuilder{})
+		}
+		(*b)[i].Add(col)
+	}
+}
+
+// Build BSI data.
+// This resets the builder.
+func (b *bsiBuilder) Build() bsiData {
+	builders := *b
+	*b = builders[:0]
+	rows := make(bsiData, len(builders))
+	for i := range builders {
+		rows[i] = builders[i].Build()
+	}
+	return rows
+}

bsi_test.go

@@ -0,0 +1,172 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+)
+
+// TestBSIAdd does a number of iterations. For each iteration, it
+// generates a random number of ids, and two random values for each id
+// to add together.
+func TestBSIAdd(t *testing.T) {
+	// TODO wouldn't it be cool if our test suite had a randomized
+	// burn-in mode where you could run any test which supported it
+	// with a random seed and way more iterations?
+	rnd := rand.New(rand.NewSource(99))
+	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
+	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
+	var builderA, builderB bsiBuilder
+	// a and b are generated slices of numbers to add together
+	var a, b []uint64
+	// idToIndex maps record ids to indexes in a and b
+	idToIndex := make(map[int]int)
+	// indexToID has the record id for each value in a and b
+	indexToID := []uint64{}
+
+	min := 999999999
+	max := 0
+
+	for iteration := 0; iteration < 1; iteration++ {
+		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
+			// reset generated data
+			a, b = a[:0], b[:0]
+			indexToID = indexToID[:0]
+			for k := range idToIndex {
+				delete(idToIndex, k)
+			}
+
+			// z generates the values, they can be fairly large, but are usually small
+			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
+			id := -1
+			for i := 0; true; i++ {
+				// get the next id, skipping a random amount
+				id = id + int(idZipf.Uint64()+1)
+				if id >= ShardWidth {
+					if i < min {
+						min = i
+					}
+					if max < i {
+						max = i
+					}
+					break
+				}
+				idToIndex[id] = int(i)
+				indexToID = append(indexToID, uint64(id))
+
+				// append a random value to each data slice
+				a = append(a, z.Uint64())
+				b = append(b, z.Uint64())
+			}
+
+			// build the BSIs based on the data slices and generated IDs
+			for index, id := range indexToID {
+				va, vb := a[index], b[index]
+				builderA.Insert(uint64(id), va)
+				builderB.Insert(uint64(id), vb)
+			}
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+
+			// build results from added bsiData; results[i] should hold a[i]+b[i]
+			results := make([]uint64, len(a))
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[idToIndex[int(id)]] = count
+				}
+			})
+
+			for i, res := range results {
+				if res != a[i]+b[i] {
+					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
+				}
+			}
+		})
+	}
+}
+
+type bsiAddCase struct {
+	positions []uint64
+	a         []uint64
+	b         []uint64
+}
+
+func (b bsiAddCase) Len() int {
+	return len(b.positions)
+}
+
+// Less reports whether the element with
+// index i should sort before the element with index j.
+func (b bsiAddCase) Less(i, j int) bool {
+	return b.positions[i] < b.positions[j]
+}
+
+// Swap swaps the elements with indexes i and j.
+func (b bsiAddCase) Swap(i, j int) {
+	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
+	b.a[i], b.a[j] = b.a[j], b.a[i]
+	b.b[i], b.b[j] = b.b[j], b.b[i]
+}
+
+// TestBSIAddCases tests specific cases of bsiAdd (would generally be
+// pulled from randomly generated ones from TestBSIAdd upon failure).
+func TestBSIAddCases(t *testing.T) {
+	tests := []bsiAddCase{
+		{
+			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
+			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
+			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
+		},
+		{
+			positions: []uint64{17570, 54607},
+			a:         []uint64{1010, 71},
+			b:         []uint64{25, 150},
+		},
+	}
+
+	var builderA, builderB bsiBuilder
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
+				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
+			}
+			sort.Sort(tst)
+
+			for i := 0; i < len(tst.a); i++ {
+				builderA.Insert(tst.positions[i], tst.a[i])
+				builderB.Insert(tst.positions[i], tst.b[i])
+			}
+
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+			// maps id to count
+			results := make(map[uint64]uint64)
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[id] = count
+				}
+			})
+
+			for i, id := range tst.positions {
+				if results[id] != tst.a[i]+tst.b[i] {
+					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
+				}
+			}
+		})
+	}
+}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +121,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -136,7 +140,9 @@ var _ cache = &lruCache{}
 type rankCache struct {
 	mu           sync.Mutex
 	entries      map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -168,10 +174,16 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	// Ignore if the column count is below the threshold,
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -184,7 +196,13 @@ func (c *rankCache) Add(id uint64, n uint64) {
 func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -209,12 +227,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
+	if len(c.entries) == 0 {
-	for id := range c.entries {
+		return nil
-		a = append(a, id)
 	}
-	sort.Sort(uint64Slice(a))
+	ids := make([]uint64, 0, len(c.entries))
-	return a
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +249,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -236,27 +257,40 @@ func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,11 +306,14 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
+
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // SetStats defines the stats client used in the cache.
@@ -285,7 +322,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +358,68 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
+func (p *bitmapPairs) Len() int           { return len(*p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
+	}
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +495,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
+	}
+	copy(r.Pairs, p.Pairs)
+	return
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -431,13 +607,7 @@ func (p uint64Slice) merge(other []uint64) []uint64 {
 	return ret
 }
 
-// bitmapCache provides an interface for caching full bitmaps.
+// simpleCache implements a bitmap Rowcache.
-type bitmapCache interface {
-	Fetch(id uint64) (*Row, bool)
-	Add(id uint64, b *Row)
-}
-
-// simpleCache implements BitmapCache
 // it is meant to be a short-lived cache for cases where writes are continuing to access
 // the same row within a short time frame (i.e. good for write-heavy loads)
 // A read-heavy use case would cause the cache to get bigger, potentially causing the
@@ -452,6 +622,12 @@ func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
 	return m, ok
 }
 
+func newSimpleCache() *simpleCache {
+	return &simpleCache{
+		cache: make(map[uint64]*Row),
+	}
+}
+
 // Add adds the bitmap to the cache, keyed on the id. A nil row means
 // deleting the row from the cache.
 func (s *simpleCache) Add(id uint64, b *Row) {

cache_test.go

@@ -15,13 +15,14 @@
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
+// Ensure cache stays constrained to its configured size.
-func TestCache_Rank(t *testing.T) {
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +32,54 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
 }

catcher.go

@@ -0,0 +1,323 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	. "github.com/pilosa/pilosa/v2/vprint"
+)
+
+// catcher is useful to report error locations with a
+// Stack dump before the complexity
+// of the executor_test swallows up
+// the location of a PanicOn.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
+	c.b.IncrementOpN(index, field, view, shard, changedN)
+}
+
+func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	return c.b.NewTxIterator(index, field, view, shard)
+}
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
+}
+
+func (c *catcherTx) Dump(short bool, shard uint64) {
+	c.b.Dump(short, shard)
+}
+
+func (c *catcherTx) Readonly() bool {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Readonly()
+}
+
+func (tx *catcherTx) Pointer() string {
+	return fmt.Sprintf("%p", tx)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) UseRowCache() bool {
+	return c.b.UseRowCache()
+}
+
+func (c *catcherTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, batched, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.ForEach(index, field, view, shard, fn)
+}
+
+func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.UnionInPlace(index, field, view, shard, others...)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
+			PanicOn(r)
+		}
+	}()
+	return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+
+func (c *catcherTx) Group() *TxGroup {
+	return c.b.Group()
+}
+
+func (c *catcherTx) Options() Txo {
+	return c.b.Options()
+}
+
+// Sn retreives the serial number of the Tx.
+func (c *catcherTx) Sn() int64 {
+	return c.b.Sn()
+}
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
+	return c.b.GetSortedFieldViewList(idx, shard)
+}
+
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

client.go

@@ -17,6 +17,10 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+	"github.com/pilosa/pilosa/v2/topology"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -43,15 +47,19 @@ type FieldValue struct {
 // something hasn't been architected correctly.
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
+// Another note from Travis: I think we eventually want to unify `InternalClient` with
+// the `github.com/pilosa/pilosa/v2/client` client.
+// Doing that may obviate the need to refactor this.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
-	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
+	PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error
 	CreateIndex(ctx context.Context, index string, opt IndexOptions) error
-	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
+	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error)
-	Nodes(ctx context.Context) ([]*Node, error)
+	Nodes(ctx context.Context) ([]*topology.Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,40 +67,91 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
-	FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
+	FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
-	BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
+	BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
-	ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
+	ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
+	RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	SendMessage(ctx context.Context, uri *URI, msg []byte) error
+	SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
-	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
+	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
-	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
+	ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
+
+	GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
+	GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
 }
 
 //===============
 
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error)
+
+	QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+
+	// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
+	TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error)
+
+	FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
+	FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
+
+	CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
+	CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
 }
 
 type nopInternalQueryClient struct{}
 
-func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
+func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) {
 	return nil, nil
 }
 
-func newNopInternalQueryClient() *nopInternalQueryClient {
+func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return &nopInternalQueryClient{}
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func newNopInternalQueryClient() nopInternalQueryClient {
+	return nopInternalQueryClient{}
 }
 
 var _ InternalQueryClient = newNopInternalQueryClient()
 
 //===============
 
-type nopInternalClient struct{}
+type nopInternalClient struct{ nopInternalQueryClient }
 
 func newNopInternalClient() nopInternalClient {
 	return nopInternalClient{}
@@ -104,34 +163,40 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64,
 	return nil, nil
 }
 func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
-func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error {
+func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error {
 	return nil
 }
 
 func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
 	return nil
 }
-func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
+func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
 	return nil, nil
 }
-func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
+func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
 	return nil, nil
 }
 func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
-func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
-func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
+	return nil
+}
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -154,21 +219,45 @@ func (n nopInternalClient) CreateField(ctx context.Context, index, field string)
 func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
 	return nil
 }
-func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
+func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
 	return nil, nil
 }
-func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
+func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
 	return nil, nil, nil
 }
-func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
 	return nil, nil
 }
-func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
 	return nil, nil
 }
-func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
+func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
 	return nil
 }
-func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
+func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
 	return nil, nil
 }

client/README.md

@@ -0,0 +1,85 @@
+# Go Client for Pilosa
+
+Go client for Pilosa high performance distributed index.
+
+## Usage
+
+If you have the pilosa repo in your `GOPATH`,
+you can import the library in your code using:
+
+```go
+import "github.com/pilosa/pilosa/v2/client"
+```
+
+
+### Quick overview
+
+Assuming [Pilosa](https://github.com/pilosa/pilosa) server is running at `localhost:10101` (the default):
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/pilosa/pilosa/v2/client"
+)
+
+func main() {
+	// Create the default client
+	cli := client.DefaultClient()
+
+	// Retrieve the schema
+	schema, err := cli.Schema()
+
+	// Create an Index object
+	myindex := schema.Index("myindex")
+
+	// Create a Field object
+	myfield := myindex.Field("myfield")
+
+	// make sure the index and the field exists on the server
+	err := cli.SyncSchema(schema)
+
+	// Send a Set query. If err is non-nil, response will be nil.
+	response, err := cli.Query(myfield.Set(5, 42))
+
+	// Send a Row query. If err is non-nil, response will be nil.
+	response, err = cli.Query(myfield.Row(5))
+
+	// Get the result
+	result := response.Result()
+	// Act on the result
+	if result != nil {
+		columns := result.Row().Columns
+		fmt.Println("Got columns: ", columns)
+	}
+
+	// You can batch queries to improve throughput
+	response, err = cli.Query(myindex.BatchQuery(
+		myfield.Row(5),
+		myfield.Row(10)))
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	for _, result := range response.Results() {
+		// Act on the result
+		fmt.Println(result.Row().Columns)
+	}
+}
+```
+
+## Documentation
+
+### Data Model and Queries
+
+See: [Data Model and Queries](docs/data-model-queries.md)
+
+### Executing Queries
+
+See: [Server Interaction](docs/server-interaction.md)
+
+### Other Documentation
+
+* [Tracing](docs/tracing.md)

client/client_it_test.go

@@ -0,0 +1,816 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/disco"
+	pnet "github.com/pilosa/pilosa/v2/net"
+	"github.com/pilosa/pilosa/v2/shardwidth"
+	"github.com/pilosa/pilosa/v2/test"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	testIndex                *Index
+	testIndexWithKeys        *Index
+	testIndexWithKeysNoTrack *Index
+	testIndexAtomicRecord    *Index
+
+	testField            *Field
+	testFieldTimeQuantum *Field
+	testFieldInt0        *Field
+	testFieldInt1        *Field
+)
+
+func setup(t *testing.T, r *require.Assertions, cli *Client) {
+	t.Helper()
+
+	testSchema := NewSchema()
+	testIndex = testSchema.Index("test-index")
+	testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true))
+	testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack",
+		OptIndexKeys(true),
+		OptIndexTrackExistence(false),
+	)
+	testField = testIndex.Field("test-field")
+	testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear))
+
+	testIndexAtomicRecord = testSchema.Index("test-index-atomic-record")
+	testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000))
+	testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000))
+
+	r.NoErrorf(cli.SyncSchema(testSchema), "SyncSchema")
+}
+
+func tearDown(t *testing.T, r *require.Assertions, cli *Client) {
+	t.Helper()
+
+	for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord} {
+		r.NoErrorf(cli.DeleteIndex(i), "DeleteIndex(%s)", i.name)
+	}
+}
+
+func TestClientAgainstCluster(t *testing.T) {
+	require := require.New(t)
+
+	for size, replicaN := 3, 1; replicaN <= 2; replicaN++ {
+		testName := fmt.Sprintf("%d.%d", size, replicaN)
+		t.Run(testName, func(t *testing.T) {
+
+			// Start size.replicaN cluster
+			c := test.MustNewCluster(t, size)
+			for _, n := range c.Nodes {
+				n.Config.Cluster.ReplicaN = replicaN
+			}
+			err := c.Start()
+			require.NoError(err, "Start cluster "+testName)
+
+			urls := make([]string, len(c.Nodes))
+			for i, n := range c.Nodes {
+				urls[i] = n.URL()
+			}
+			defer c.Close()
+
+			// Create a new client for the cluster
+			cli, err := newClientFromAddresses(urls, &ClientOptions{})
+			require.NoErrorf(err, "newClientFromAddresses(%v): %v", urls, err)
+			defer cli.Close()
+
+			t.Run("GetStatus", func(t *testing.T) {
+				status, err := cli.Status()
+				require.NoErrorf(err, "GET /status")
+
+				require.Equalf(disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status")
+			})
+
+			t.Run("QueryRow", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				resp, err := cli.Query(testField.Row(1))
+				require.NoErrorf(err, "Query Row")
+				require.NotNil(resp, "Response should not be nil")
+			})
+
+			t.Run("QueryWithShards", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				shardWidth := uint64(1 << shardwidth.Exponent)
+
+				_, err := cli.Query(testField.Set(1, 1))
+				require.NoErrorf(err, "Set(1, %d)", 1)
+
+				_, err = cli.Query(testField.Set(1, shardWidth))
+				require.NoErrorf(err, "Set(1, %d)", shardWidth)
+
+				_, err = cli.Query(testField.Set(1, shardWidth*3))
+				require.NoErrorf(err, "Set(1, %d)", shardWidth*3)
+
+				resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3))
+				require.NoErrorf(err, "Row(1) OptQueryShards(0, 3)")
+
+				cols := resp.Result().Row().Columns
+				require.Equalf([]uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols)
+			})
+
+			t.Run("QueryWithColumns", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				targetAttrs := map[string]interface{}{
+					"name":       "some string",
+					"age":        int64(95),
+					"registered": true,
+					"height":     1.83,
+				}
+				_, err := cli.Query(testField.Set(1, 100))
+				require.NoErrorf(err, "Set(1, 100)")
+
+				resp, err := cli.Query(testIndex.SetColumnAttrs(100, targetAttrs))
+				require.NoErrorf(err, "SetColumnAttrs(100, %v)", targetAttrs)
+				require.Equalf(ColumnItem{}, resp.Column(), "No columns should be returned if it wasn't explicitly requested")
+
+				resp, err = cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true})
+				require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}")
+				require.Equalf(1, len(resp.ColumnAttrs()), "ColumnAttrs count should be == 1")
+
+				cols := resp.Columns()
+				require.Equalf(1, len(cols), "Column count")
+				require.Equalf(uint64(100), cols[0].ID, "Column ID")
+
+				require.Equalf(targetAttrs, cols[0].Attributes, "Column attrs.")
+
+				require.Equalf(cols[0], resp.Column(), "Column() should be equivalent to first column in the response")
+			})
+
+			t.Run("SetRowAttrs", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				targetAttrs := map[string]interface{}{
+					"name":       "some string",
+					"age":        int64(95),
+					"registered": true,
+					"height":     1.83,
+				}
+
+				_, err := cli.Query(testField.Set(1, 100))
+				require.NoErrorf(err, "Set(1, 100)")
+
+				_, err = cli.Query(testField.SetRowAttrs(1, targetAttrs))
+				require.NoErrorf(err, "SetRowAttrs(1, %v)", targetAttrs)
+
+				resp, err := cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true})
+				require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}")
+
+				require.Equalf(targetAttrs, resp.Result().Row().Attributes, "Row attributes should be set")
+			})
+
+			t.Run("OrmCount", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldCount := testIndex.Field("test-field-count")
+				err := cli.EnsureField(testFieldCount)
+				require.NoError(err)
+
+				qry := testIndex.BatchQuery(
+					testFieldCount.Set(10, 20),
+					testFieldCount.Set(10, 21),
+					testFieldCount.Set(15, 25),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(err, "BatchQuery")
+
+				resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10)))
+				require.NoErrorf(err, "Count")
+				require.Equalf(int64(2), resp.Result().Count(), "Count")
+			})
+
+			t.Run("DecimalField", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3))
+				err := cli.EnsureField(testFieldDec)
+				require.NoError(err)
+
+				sch, err := cli.Schema()
+				require.NoErrorf(err, "Schema")
+
+				idx := sch.indexes[testIndex.name]
+				opts := idx.Field(testFieldDec.name).Options()
+				require.Equalf(int64(3), opts.scale, "%s scale", testFieldDec.name)
+			})
+
+			t.Run("IntersectReturns", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldSegments := testIndex.Field("test-field-segments")
+				err := cli.EnsureField(testFieldSegments)
+				require.NoError(err)
+
+				qry1 := testIndex.BatchQuery(
+					testFieldSegments.Set(2, 10),
+					testFieldSegments.Set(2, 15),
+					testFieldSegments.Set(3, 10),
+					testFieldSegments.Set(3, 20),
+				)
+				_, err = cli.Query(qry1)
+				require.NoErrorf(err, "BatchQuery")
+
+				qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3))
+				resp, err := cli.Query(qry2)
+				require.NoErrorf(err, "Intersect")
+
+				require.Equalf(1, len(resp.Results()), "Intersect number of results")
+				require.Equalf([]uint64{10}, resp.Result().Row().Columns, "Intersect columns results")
+			})
+
+			t.Run("TopNReturns", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldTopN := testIndex.Field("test-field-topn")
+				err := cli.EnsureField(testFieldTopN)
+				require.NoError(err)
+
+				qry := testIndex.BatchQuery(
+					testFieldTopN.Set(10, 5),
+					testFieldTopN.Set(10, 10),
+					testFieldTopN.Set(10, 15),
+					testFieldTopN.Set(20, 5),
+					testFieldTopN.Set(30, 5),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(err, "BatchQuery")
+
+				// XXX: The following is required to make this test pass. See: https://github.com/pilosa/pilosa/issues/625
+				_, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil)
+				require.NoErrorf(err, "POST /recalculate-caches")
+
+				resp, err := cli.Query(testFieldTopN.TopN(2))
+				require.NoErrorf(err, "TopN(2)")
+
+				items := resp.Result().CountItems()
+				require.Equalf(2, len(items), "TopN result CountItems")
+
+				item := items[0]
+				require.Equalf(uint64(10), item.ID, "TopN result item[0].ID")
+				require.Equalf(uint64(3), item.Count, "TopN result item[0].Count")
+
+				_, err = cli.Query(testFieldTopN.SetRowAttrs(10, map[string]interface{}{"foo": "bar"}))
+				require.NoErrorf(err, "SetRowAttrs(10)")
+
+				resp, err = cli.Query(testFieldTopN.FilterAttrTopN(5, nil, "foo", "bar"))
+				require.NoErrorf(err, `FilterAttrTopN(5, nil, "foo", "bar")`)
+
+				items = resp.Result().CountItems()
+				require.Equalf(1, len(items), "FilterAttrTopN result CountItems")
+
+				item = items[0]
+				require.Equalf(uint64(10), item.ID, "FilterAttrTopN result item[0].ID")
+				require.Equalf(uint64(3), item.Count, "FilterAttrTopN result item[0].Count")
+			})
+
+			t.Run("MinMaxRow", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldMinMax := testIndex.Field("test-field-minmax")
+				err := cli.EnsureField(testFieldMinMax)
+				require.NoError(err)
+
+				qry := testIndex.BatchQuery(
+					testFieldMinMax.Set(10, 5),
+					testFieldMinMax.Set(10, 10),
+					testFieldMinMax.Set(10, 15),
+					testFieldMinMax.Set(20, 5),
+					testFieldMinMax.Set(30, 5),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(err, "Setting bits")
+
+				resp, err := cli.Query(testFieldMinMax.MinRow())
+				require.NoErrorf(err, "MinRow")
+
+				min := resp.Result().CountItem().ID
+				require.Equalf(uint64(10), min, "Min")
+
+				resp, err = cli.Query(testFieldMinMax.MaxRow())
+				require.NoErrorf(err, "MaxRow")
+
+				max := resp.Result().CountItem().ID
+				require.Equalf(uint64(30), max, "Max")
+			})
+
+			t.Run("SetMutexField", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0))
+				err := cli.EnsureField(testFieldMutex)
+				require.NoError(err)
+
+				// can set mutex
+				_, err = cli.Query(testFieldMutex.Set(1, 100))
+				require.NoErrorf(err, "Set(1, 100)")
+
+				resp, err := cli.Query(testFieldMutex.Row(1))
+				require.NoErrorf(err, "Row(1)")
+
+				target := []uint64{100}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+
+				// setting another row removes the previous
+				_, err = cli.Query(testFieldMutex.Set(42, 100))
+				require.NoErrorf(err, "Set(42, 100)")
+
+				resp, err = cli.Query(testIndex.BatchQuery(
+					testFieldMutex.Row(1),
+					testFieldMutex.Row(42),
+				))
+				require.NoErrorf(err, "BatchQuery")
+
+				target1 := []uint64(nil)
+				target42 := []uint64{100}
+				require.Equalf(target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns")
+				require.Equalf(target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns")
+			})
+
+			t.Run("SetBoolField", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool())
+				err := cli.EnsureField(testFieldBool)
+				require.NoError(err)
+
+				// can set bool
+				_, err = cli.Query(testFieldBool.Set(true, 100))
+				require.NoErrorf(err, "Set(true, 100)")
+
+				resp, err := cli.Query(testFieldBool.Row(true))
+				require.NoErrorf(err, "Row(true)")
+
+				target := []uint64{100}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("ClearRowQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldClear := testIndex.Field("test-field-clear")
+				err := cli.EnsureField(testFieldClear)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldClear.Set(1, 100),
+					testFieldClear.Set(1, 200),
+				))
+				require.NoErrorf(err, "Set(1, 100) Set(1, 200)")
+
+				resp, err := cli.Query(testFieldClear.Row(1))
+				require.NoErrorf(err, "Row(1)")
+
+				target := []uint64{100, 200}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+
+				_, err = cli.Query(testFieldClear.ClearRow(1))
+				require.NoErrorf(err, "ClearRow(1)")
+
+				resp, err = cli.Query(testFieldClear.Row(1))
+				require.NoErrorf(err, "Row(1)")
+
+				target = []uint64(nil)
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("RowsQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldRows := testIndex.Field("test-field-rows")
+				err := cli.EnsureField(testFieldRows)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRows.Set(1, 100),
+					testFieldRows.Set(1, 200),
+					testFieldRows.Set(2, 200),
+				))
+				require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testFieldRows.Rows())
+				require.NoErrorf(err, "Rows")
+
+				target := RowIdentifiersResult{
+					IDs: []uint64{1, 2},
+				}
+				require.Equalf(target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
+			})
+
+			t.Run("UnionRowsQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldRows := testIndex.Field("test-field-rows")
+				err := cli.EnsureField(testFieldRows)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRows.Set(1, 100),
+					testFieldRows.Set(1, 200),
+					testFieldRows.Set(2, 200),
+				))
+				require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testFieldRows.Rows().Union())
+				require.NoErrorf(err, "Rows Union")
+
+				target := []uint64{100, 200}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("LikeQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true))
+				err := cli.EnsureField(testFieldLike)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldLike.Set("a", 100),
+					testFieldLike.Set("b", 200),
+					testFieldLike.Set("bc", 200),
+				))
+				require.NoErrorf(err, "Set(a, 100) Set(b, 200) Set(bc, 200)")
+
+				resp, err := cli.Query(testFieldLike.Like("b%"))
+				require.NoErrorf(err, `Like(b%)`)
+
+				target := RowIdentifiersResult{
+					Keys: []string{"b", "bc"},
+				}
+				require.Equalf(target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
+			})
+
+			t.Run("GroupByQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldGroupBy := testIndex.Field("test-field-group-by")
+				err := cli.EnsureField(testFieldGroupBy)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldGroupBy.Set(1, 100),
+					testFieldGroupBy.Set(1, 200),
+					testFieldGroupBy.Set(2, 200),
+				))
+				require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
+				require.NoErrorf(err, `Like(b%)`)
+
+				target := []GroupCount{
+					{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1},
+				}
+
+				assertGroupBy(t, require, target, resp.Result().GroupCounts())
+			})
+
+			t.Run("GroupByQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10))
+				err := cli.EnsureField(testFieldGroupBy)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.RawQuery(`
+		Set(0,      test-field-group-by-int=1)
+		Set(1,      test-field-group-by-int=2)
+
+		Set(2,      test-field-group-by-int=-2)
+		Set(3,      test-field-group-by-int=-1)
+
+		Set(4,      test-field-group-by-int=4)
+
+		Set(10,     test-field-group-by-int=0)
+		Set(100,    test-field-group-by-int=0)
+		Set(1000,   test-field-group-by-int=0)
+		Set(10000,  test-field-group-by-int=0)
+		Set(100000, test-field-group-by-int=0)
+		`))
+				require.NoError(err, "Set(0..100000)")
+
+				resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
+				require.NoErrorf(err, `GroupBy(Rows)`)
+
+				var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4
+				target := []GroupCount{
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1},
+				}
+				assertGroupBy(t, require, target, resp.Result().GroupCounts())
+			})
+
+			t.Run("CreateDeleteIndexField", func(t *testing.T) {
+				tmpIndex := NewIndex("tmp-index")
+				tmpField := tmpIndex.Field("tmp-field")
+
+				err := cli.CreateIndex(tmpIndex)
+				require.NoError(err)
+
+				err = cli.CreateField(tmpField)
+				require.NoError(err)
+
+				err = cli.DeleteField(tmpField)
+				require.NoError(err)
+
+				err = cli.DeleteIndex(tmpIndex)
+				require.NoError(err)
+			})
+
+			t.Run("ErrorCreatingIndexField", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				require.ErrorIs(cli.CreateIndex(testIndex), ErrIndexExists)
+				require.ErrorIs(cli.CreateField(testField), ErrFieldExists)
+			})
+
+			t.Run("Failover", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar")
+				tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0))
+
+				attrs := map[string]interface{}{"a": 1}
+				_, err := tmpcli.Query(testIndex.SetColumnAttrs(0, attrs))
+				require.Error(err, ErrTriedMaxHosts)
+			})
+
+			t.Run("InvalidQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				_, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil)
+				require.Error(err)
+
+				_, err = cli.Query(testIndex.RawQuery("Invalid query"))
+				require.Error(err)
+			})
+
+			t.Run("Sync", func(t *testing.T) {
+				testIndexRemote := NewIndex("test-index-remote")
+				err := cli.EnsureIndex(testIndexRemote)
+				require.NoError(err)
+
+				testFieldRemote := testIndexRemote.Field("test-field-remote")
+				err = cli.EnsureField(testFieldRemote)
+				require.NoError(err)
+
+				schema := NewSchema()
+				idx1 := schema.Index("index-1")
+				idx1.Field("field-1-1")
+				idx1.Field("field-1-2")
+
+				idx2 := schema.Index("index-2")
+				idx2.Field("field-2-1")
+				schema.Index(testIndexRemote.Name())
+
+				err = cli.SyncSchema(schema)
+				require.NoError(err)
+
+				err = cli.DeleteIndex(testIndexRemote)
+				require.NoError(err)
+
+				err = cli.DeleteIndex(idx1)
+				require.NoError(err)
+
+				err = cli.DeleteIndex(idx2)
+				require.NoError(err)
+			})
+
+			t.Run("FetchFragmentNodes", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0)
+				require.NoErrorf(err, "fetchFragmentNodes(%s, 0)", testIndex.name)
+				require.Equalf(replicaN, len(nodes), "len(nodes)")
+
+				// running the same for coverage
+				nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0)
+				require.NoErrorf(err, "fetchFragmentNodes(%s, 0)", testIndex.name)
+				require.Equalf(replicaN, len(nodes), "len(nodes)")
+			})
+
+			t.Run("RowRangeQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour))
+				err := cli.EnsureField(testFieldRange)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)),
+					testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)),
+					testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)),
+				))
+				require.NoErrorf(err, "BatchQuery SetTimestamp")
+
+				start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC)
+				end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC)
+				resp, err := cli.Query(testFieldRange.RowRange(10, start, end))
+				require.NoErrorf(err, "RowRange(10, %v, %v)", start, end)
+
+				target := []uint64{100}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("StoreQuery", func(t *testing.T) {
+				schema := NewSchema()
+				testIndexStore := schema.Index("test-index-store")
+				testFieldFrom := testIndexStore.Field("test-field-from")
+				testFieldTo := testIndexStore.Field("test-field-to")
+				err := cli.SyncSchema(schema)
+				require.NoError(err)
+
+				defer func() {
+					cerr := cli.DeleteIndex(testIndexStore)
+					require.NoErrorf(cerr, "failed to delete index: %v", testIndexStore.name)
+				}()
+
+				_, err = cli.Query(testIndexStore.BatchQuery(
+					testFieldFrom.Set(10, 100),
+					testFieldFrom.Set(10, 200),
+					testFieldTo.Store(testFieldFrom.Row(10), 1),
+				))
+				require.NoErrorf(err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)")
+
+				resp, err := cli.Query(testFieldTo.Row(1))
+				require.NoErrorf(err, "Row(1)")
+
+				target := []uint64{100, 200}
+				require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("MultipleClientKeyQuery", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient")
+				err := cli.EnsureField(testFieldMultiClient)
+				require.NoError(err)
+
+				eg := &errgroup.Group{}
+				for i := 0; i < 10; i++ {
+					rowID := uint64(i)
+					eg.Go(func() error {
+						_, e := cli.Query(testFieldMultiClient.Set(rowID, "col"))
+						return e
+					})
+				}
+				require.NoError(eg.Wait())
+			})
+
+			t.Run("ExportRowIDColumnID", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldExport := testIndex.Field("test-field-export")
+				err := cli.EnsureField(testFieldExport)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldExport.Set(1, 1),
+					testFieldExport.Set(1, 10),
+					testFieldExport.Set(2, 1048577),
+				), nil)
+				require.NoErrorf(err, "Set(1, 1) Set(1, 10) Set(2, 1048577)")
+
+				r, err := cli.ExportField(testFieldExport)
+				require.NoErrorf(err, "ExportField")
+
+				b, err := ioutil.ReadAll(r)
+				require.NoError(err)
+
+				target := "1,1\n1,10\n2,1048577\n"
+				require.Equalf(target, string(b), "Export Field Response")
+			})
+
+			t.Run("ExportRowIDColumnKey", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldExport := testIndexWithKeys.Field("test-field-export")
+				err := cli.EnsureField(testFieldExport)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndexWithKeys.BatchQuery(
+					testFieldExport.Set(1, "one"),
+					testFieldExport.Set(1, "ten"),
+					testFieldExport.Set(2, "big-number"),
+				), nil)
+				require.NoErrorf(err, "Set(1, one) Set(1, ten) Set(2, big-number)")
+
+				r, err := cli.ExportField(testFieldExport)
+				require.NoErrorf(err, "ExportField")
+
+				b, err := ioutil.ReadAll(r)
+				require.NoError(err)
+
+				target := "1,one\n1,ten\n2,big-number\n"
+				require.Equalf(target, string(b), "Export Field Response")
+			})
+
+			t.Run("TranslateRowKeys", func(t *testing.T) {
+				setup(t, require, cli)
+				defer tearDown(t, require, cli)
+
+				testFieldTranslate := testIndex.Field("test-field-translate-rowkeys", OptFieldKeys(true))
+				err := cli.EnsureField(testFieldTranslate)
+				require.NoError(err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldTranslate.Set("key1", 10),
+					testFieldTranslate.Set("key2", 1000),
+				))
+				require.NoErrorf(err, "Set(key1, 10) Set(key2, 1000)")
+
+				rowIDs, err := cli.TranslateRowKeys(testFieldTranslate, []string{"key1", "key2"})
+				require.NoErrorf(err, "TranslateRowKeys")
+
+				target := []uint64{1, 2}
+				require.Equalf(target, rowIDs, "TranslateRowKeys")
+			})
+
+			t.Run("TranslateColKeys", func(t *testing.T) {
+				trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute)
+				require.NoErrorf(err, "StartTransaction(blah)")
+				require.Equalf("blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(trns.Active, "TranslateColumnKeys Active")
+
+				trnsMap, err := cli.Transactions()
+				require.NoErrorf(err, "Transactions")
+				require.Equalf(1, len(trnsMap), "Transactions len")
+				require.Truef(trnsMap["blah"].Active, "Transactions Active")
+
+				trns, err = cli.GetTransaction("blah")
+				require.NoErrorf(err, "GetTransaction(blah)")
+				require.Equalf("blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(trns.Active, "TranslateColumnKeys Active")
+
+				trns, err = cli.FinishTransaction("blah")
+				require.NoErrorf(err, "FinishTransaction(blah)")
+				require.Equalf("blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(trns.Active, "TranslateColumnKeys Active")
+			})
+		})
+	}
+}
+
+func assertGroupBy(t *testing.T, r *require.Assertions, expected, results []GroupCount) {
+	t.Helper()
+
+	r.Equalf(len(expected), len(results), "number of groupings mismatch")
+
+	for i, result := range results {
+		r.Equalf(expected[i], result, "unexpected result at %d", i)
+	}
+}

client/client_test.go

@@ -0,0 +1,293 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"crypto/tls"
+	"errors"
+	"reflect"
+	"testing"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+)
+
+func TestQueryWithError(t *testing.T) {
+	var err error
+	client := DefaultClient()
+	index := NewIndex("foo")
+	field := index.Field("foo")
+	invalid := field.FilterAttrTopN(12, field.Row(7), "$invalid$", 80, 81)
+	_, err = client.Query(invalid, nil)
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestClientOptions(t *testing.T) {
+	targets := []*ClientOptions{
+		{SocketTimeout: 10},
+		{ConnectTimeout: 5},
+		{PoolSizePerRoute: 7},
+		{TotalPoolSize: 17},
+		{TLSConfig: &tls.Config{InsecureSkipVerify: true}},
+	}
+	optionsList := [][]ClientOption{
+		{OptClientSocketTimeout(10)},
+		{OptClientConnectTimeout(5)},
+		{OptClientPoolSizePerRoute(7)},
+		{OptClientTotalPoolSize(17)},
+		{OptClientTLSConfig(&tls.Config{InsecureSkipVerify: true})},
+	}
+
+	for i := 0; i < len(targets); i++ {
+		options := &ClientOptions{}
+		err := options.addOptions(optionsList[i]...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		target := targets[i]
+		if !reflect.DeepEqual(target, options) {
+			t.Fatalf("%v != %v", target, options)
+		}
+	}
+}
+
+func TestNewClientWithErrorredOption(t *testing.T) {
+	_, err := NewClient(":8888", ClientOptionErr(0))
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestNewClient(t *testing.T) {
+	client, err := NewClient(":9999", OptClientManualServerAddress(true))
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI, err := pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(targetURI, client.manualServerURI) {
+		t.Fatalf("%v != %v", targetURI, client.manualServerURI)
+	}
+	targetFragmentNode := &fragmentNode{
+		Scheme: "http",
+		Host:   "localhost",
+		Port:   9999,
+	}
+	if !reflect.DeepEqual(targetFragmentNode, client.manualFragmentNode) {
+		t.Fatalf("%v != %v", targetFragmentNode, client.manualFragmentNode)
+	}
+	client, err = NewClient(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	targetURI, err = pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	target := []*pnet.URI{targetURI}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+	client, err = NewClient([]string{":9999"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	targetURI1, err := pnet.NewURIFromAddress(":8888")
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI2, err := pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client, err = NewClient([]*pnet.URI{targetURI1, targetURI2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	target = []*pnet.URI{targetURI1, targetURI2}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	client, err = NewClient([]*pnet.URI{targetURI})
+	if err != nil {
+		t.Fatal(err)
+	}
+	target = []*pnet.URI{targetURI}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	client, err = NewClient(DefaultCluster())
+	if err != nil {
+		t.Fatal(err)
+	}
+	target = []*pnet.URI{}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+}
+
+func TestNewClientWithInvalidAddr(t *testing.T) {
+	_, err := NewClient(10)
+	if err != ErrAddrURIClusterExpected {
+		t.Fatalf("%v != %v", ErrAddrURIClusterExpected, err)
+	}
+	_, err = NewClient(":invalid")
+	if err == nil {
+		t.Fatalf("should have failed: %+v", err)
+	}
+	_, err = NewClient([]string{"valid:8000", ":invalid"})
+	if err != pnet.ErrInvalidAddress {
+		t.Fatalf("Should have failed '%v, got '%v'", pnet.ErrInvalidAddress, err)
+	}
+}
+
+func TestNewClientManualAddressWithNoURIs(t *testing.T) {
+	_, err := NewClient([]string{}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+	_, err = NewClient([]*pnet.URI{}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+}
+
+func TestNewClientManualAddressWithMultipleURIs(t *testing.T) {
+	_, err := NewClient([]string{":9000", ":5000"}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+
+	targetURI1, err := pnet.NewURIFromAddress(":9000")
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI2, err := pnet.NewURIFromAddress(":5000")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewClient([]*pnet.URI{targetURI1, targetURI2}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+}
+
+func ClientOptionErr(int) ClientOption {
+	return func(*ClientOptions) error {
+		return errors.New("Some error")
+	}
+}
+
+func TestQueryOptions(t *testing.T) {
+	targets := []*QueryOptions{
+		{ColumnAttrs: true},
+		{ColumnAttrs: false},
+		{ExcludeRowAttrs: true},
+		{ExcludeRowAttrs: false},
+		{ExcludeColumns: true},
+		{ExcludeColumns: false},
+	}
+
+	optionsList := [][]interface{}{
+		{OptQueryColumnAttrs(true)},
+		{OptQueryColumnAttrs(false)},
+		{OptQueryExcludeAttrs(true)},
+		{OptQueryExcludeAttrs(false)},
+		{OptQueryExcludeColumns(true)},
+		{OptQueryExcludeColumns(false)},
+	}
+
+	for i := 0; i < len(targets); i++ {
+		options := &QueryOptions{}
+		err := options.addOptions(optionsList[i]...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		target := targets[i]
+		if !reflect.DeepEqual(target, options) {
+			t.Fatalf("%v != %v", target, options)
+		}
+	}
+
+	target := &QueryOptions{
+		ColumnAttrs:     true,
+		ExcludeRowAttrs: true,
+		ExcludeColumns:  true,
+	}
+	options := &QueryOptions{}
+	err := options.addOptions(&QueryOptions{
+		ColumnAttrs:     true,
+		ExcludeRowAttrs: true,
+		ExcludeColumns:  true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(target, options) {
+		t.Fatalf("%v != %v", target, options)
+	}
+}
+
+func TestQueryOptionsWithError(t *testing.T) {
+	options := &QueryOptions{}
+	err := options.addOptions(1)
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+	err = options.addOptions(OptQueryColumnAttrs(true), nil)
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+	err = options.addOptions(OptQueryColumnAttrs(true), &QueryOptions{})
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+	err = options.addOptions(QueryOptionErr(0))
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func TestQueryOptionsError(t *testing.T) {
+	client := DefaultClient()
+	index := NewIndex("foo")
+	_, err := client.Query(index.RawQuery(""), QueryOptionErr(0))
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func QueryOptionErr(int) QueryOption {
+	return func(*QueryOptions) error {
+		return errors.New("Some error")
+	}
+}

client/cluster.go

@@ -0,0 +1,112 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"sync"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+)
+
+// Cluster contains hosts in a Pilosa cluster.
+type Cluster struct {
+	hosts       []*pnet.URI
+	okList      []bool
+	mutex       *sync.RWMutex
+	lastHostIdx int
+}
+
+// DefaultCluster returns the default Cluster.
+func DefaultCluster() *Cluster {
+	return &Cluster{
+		hosts:  make([]*pnet.URI, 0),
+		okList: make([]bool, 0),
+		mutex:  &sync.RWMutex{},
+	}
+}
+
+// NewClusterWithHost returns a cluster with the given URIs.
+func NewClusterWithHost(hosts ...*pnet.URI) *Cluster {
+	cluster := DefaultCluster()
+	for _, host := range hosts {
+		cluster.AddHost(host)
+	}
+	return cluster
+}
+
+// AddHost adds a host to the cluster.
+func (c *Cluster) AddHost(address *pnet.URI) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.hosts = append(c.hosts, address)
+	c.okList = append(c.okList, true)
+}
+
+// Host returns a host in the cluster.
+func (c *Cluster) Host() *pnet.URI {
+	c.mutex.Lock()
+	var host *pnet.URI
+	for i := range c.okList {
+		idx := (i + c.lastHostIdx) % len(c.okList)
+		ok := c.okList[idx]
+		if ok {
+			host = c.hosts[idx]
+			break
+		}
+	}
+	c.lastHostIdx++
+	c.mutex.Unlock()
+	if host != nil {
+		return host
+	}
+	c.reset()
+	return host
+}
+
+// RemoveHost black lists the host with the given pnet.URI from the cluster.
+func (c *Cluster) RemoveHost(address *pnet.URI) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	for i, uri := range c.hosts {
+		if uri.Equals(address) {
+			c.okList[i] = false
+			break
+		}
+	}
+}
+
+// Hosts returns all available hosts in the cluster.
+func (c *Cluster) Hosts() []pnet.URI {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	hosts := make([]pnet.URI, 0, len(c.hosts))
+	for i, host := range c.hosts {
+		if c.okList[i] {
+			hosts = append(hosts, *host)
+		}
+	}
+	return hosts
+}
+
+func (c *Cluster) reset() {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	for i := range c.okList {
+		c.okList[i] = true
+	}
+}

client/cluster_test.go

@@ -0,0 +1,83 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"testing"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+)
+
+func TestNewClusterWithHost(t *testing.T) {
+	c := NewClusterWithHost(pnet.DefaultURI())
+	hosts := c.Hosts()
+	if len(hosts) != 1 || !hosts[0].Equals(pnet.DefaultURI()) {
+		t.Fail()
+	}
+}
+
+func TestAddHost(t *testing.T) {
+	const addr = "http://localhost:3000"
+	c := DefaultCluster()
+	if c.Hosts() == nil {
+		t.Fatalf("Hosts should not be nil")
+	}
+	uri, err := pnet.NewURIFromAddress(addr)
+	if err != nil {
+		t.Fatalf("Cannot parse address")
+	}
+	target, err := pnet.NewURIFromAddress(addr)
+	if err != nil {
+		t.Fatalf("Cannot parse address")
+	}
+	c.AddHost(uri)
+	hosts := c.Hosts()
+	if len(hosts) != 1 || !hosts[0].Equals(target) {
+		t.Fail()
+	}
+}
+
+func TestHosts(t *testing.T) {
+	c := DefaultCluster()
+	if c.Host() != nil {
+		t.Fatalf("Hosts with empty cluster should return nil")
+	}
+	c = NewClusterWithHost(pnet.DefaultURI())
+	if !c.Host().Equals(pnet.DefaultURI()) {
+		t.Fatalf("Host should return a value if there are hosts in the cluster")
+	}
+}
+
+func TestRemoveHost(t *testing.T) {
+	uri, err := pnet.NewURIFromAddress("index1.pilosa.com:9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := NewClusterWithHost(uri)
+	if len(c.hosts) != 1 {
+		t.Fatalf("The cluster should contain the host")
+	}
+	uri, err = pnet.NewURIFromAddress("index1.pilosa.com:9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.RemoveHost(uri)
+	if len(c.Hosts()) != 0 {
+		t.Fatalf("The cluster should not contain the host")
+	}
+}

client/csv/csv.go

@@ -0,0 +1,194 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package csv
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/client"
+)
+
+// Format is the format of the data in the CSV file.
+type Format uint
+
+const (
+	// RowIDColumnID formatted data is ROW_ID,COLUMN_ID.
+	RowIDColumnID Format = iota
+	// RowIDColumnKey formatted data is ROW_ID,COLUMN_KEY.
+	RowIDColumnKey
+	// RowKeyColumnID formatted data is ROW_KEY,COLUMN_ID.
+	RowKeyColumnID
+	// RowKeyColumnKey formatted data is ROW_KEY,COLUMN_ID.
+	RowKeyColumnKey
+	// ColumnID formatted data is COLUMN_ID. Valid only for value import.
+	ColumnID
+	// ColumnKey formatted data is COLUMN_KEY. Valud only for value import.
+	ColumnKey
+)
+
+// ColumnUnmarshaller creates a RecordUnmarshaller for importing columns with the given format.
+func ColumnUnmarshaller(format Format) RecordUnmarshaller {
+	return ColumnUnmarshallerWithTimestamp(format, "")
+}
+
+// ColumnUnmarshallerWithTimestamp creates a RecordUnmarshaller for importing columns with the given format and timestamp format.
+func ColumnUnmarshallerWithTimestamp(format Format, timestampFormat string) RecordUnmarshaller {
+	return func(text string) (client.Record, error) {
+		var err error
+		column := client.Column{}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			return nil, errors.New("Invalid CSV line")
+		}
+
+		hasRowKey := format == RowKeyColumnID || format == RowKeyColumnKey
+		hasColumnKey := format == RowIDColumnKey || format == RowKeyColumnKey
+
+		if hasRowKey {
+			column.RowKey = parts[0]
+		} else {
+			column.RowID, err = strconv.ParseUint(parts[0], 10, 64)
+			if err != nil {
+				return nil, errors.New("Invalid row ID")
+			}
+		}
+
+		if hasColumnKey {
+			column.ColumnKey = parts[1]
+		} else {
+			column.ColumnID, err = strconv.ParseUint(parts[1], 10, 64)
+			if err != nil {
+				return nil, errors.New("Invalid column ID")
+			}
+		}
+
+		timestamp := int64(0)
+		if len(parts) == 3 {
+			if timestampFormat == "" {
+				if tsInt, err := strconv.Atoi(parts[2]); err != nil {
+					return nil, err
+				} else {
+					timestamp = int64(tsInt)
+				}
+			} else {
+				t, err := time.Parse(timestampFormat, parts[2])
+				if err != nil {
+					return nil, err
+				}
+				timestamp = t.Unix() * int64(time.Second) // Casting a duration to int64 gives the number of nanoseconds in that duration.
+			}
+		}
+		column.Timestamp = timestamp
+
+		return column, nil
+	}
+}
+
+// RecordUnmarshaller is a function which creates a Record from a CSV file line with column data.
+type RecordUnmarshaller func(text string) (client.Record, error)
+
+// Iterator reads records from a Reader.
+// Each line should contain a single record in the following form:
+// field1,field2,...
+type Iterator struct {
+	reader       io.Reader
+	line         int
+	scanner      *bufio.Scanner
+	unmarshaller RecordUnmarshaller
+}
+
+// NewIterator creates a CSVIterator from a Reader.
+func NewIterator(reader io.Reader, unmarshaller RecordUnmarshaller) *Iterator {
+	return &Iterator{
+		reader:       reader,
+		line:         0,
+		scanner:      bufio.NewScanner(reader),
+		unmarshaller: unmarshaller,
+	}
+}
+
+// NewColumnIterator creates a new iterator for column data.
+func NewColumnIterator(format Format, reader io.Reader) *Iterator {
+	return NewIterator(reader, ColumnUnmarshaller(format))
+}
+
+// NewColumnIteratorWithTimestampFormat creates a new iterator for column data with timestamp.
+func NewColumnIteratorWithTimestampFormat(format Format, reader io.Reader, timestampFormat string) *Iterator {
+	return NewIterator(reader, ColumnUnmarshallerWithTimestamp(format, timestampFormat))
+}
+
+// NewValueIterator creates a new iterator for value data.
+func NewValueIterator(format Format, reader io.Reader) *Iterator {
+	return NewIterator(reader, FieldValueUnmarshaller(format))
+}
+
+// NextRecord iterates on lines of a Reader.
+// Returns io.EOF on end of iteration.
+func (c *Iterator) NextRecord() (client.Record, error) {
+	if ok := c.scanner.Scan(); ok {
+		c.line++
+		text := strings.TrimSpace(c.scanner.Text())
+		if text != "" {
+			rc, err := c.unmarshaller(text)
+			if err != nil {
+				return nil, fmt.Errorf("%s at line: %d", err.Error(), c.line)
+			}
+			return rc, nil
+		}
+	}
+	err := c.scanner.Err()
+	if err != nil {
+		return nil, err
+	}
+	return nil, io.EOF
+}
+
+// FieldValueUnmarshaller is a function which creates a Record from a CSV file line with value data.
+func FieldValueUnmarshaller(format Format) RecordUnmarshaller {
+	return func(text string) (client.Record, error) {
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			return nil, errors.New("Invalid CSV")
+		}
+		value, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, errors.New("Invalid value")
+		}
+		switch format {
+		case ColumnID:
+			columnID, err := strconv.ParseUint(parts[0], 10, 64)
+			if err != nil {
+				return nil, errors.New("Invalid column ID at line: %d")
+			}
+			return client.FieldValue{
+				ColumnID: uint64(columnID),
+				Value:    value,
+			}, nil
+		case ColumnKey:
+			return client.FieldValue{
+				ColumnKey: parts[0],
+				Value:     value,
+			}, nil
+		default:
+			return nil, fmt.Errorf("Invalid format: %d", format)
+		}
+	}
+}

client/csv/csv_it_test.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//+build integration
+
+package csv_test
+
+import (
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/client"
+	"github.com/pilosa/pilosa/v2/client/csv"
+)
+
+func TestCSVIterate(t *testing.T) {
+	text := `10,7
+		10,5
+		2,3
+		7,1`
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
+	recs := consumeIterator(t, iterator)
+	target := []client.Record{
+		client.Column{RowID: 10, ColumnID: 7},
+		client.Column{RowID: 10, ColumnID: 5},
+		client.Column{RowID: 2, ColumnID: 3},
+		client.Column{RowID: 7, ColumnID: 1},
+	}
+	if !reflect.DeepEqual(target, recs) {
+		t.Fatalf("%v != %v", target, recs)
+	}
+}
+
+func consumeIterator(t *testing.T, it *csv.Iterator) []client.Record {
+	recs := []client.Record{}
+	for {
+		r, err := it.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		recs = append(recs, r)
+	}
+	return recs
+}

client/csv/csv_test.go

@@ -0,0 +1,267 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package csv_test
+
+import (
+	"errors"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/client"
+	"github.com/pilosa/pilosa/v2/client/csv"
+)
+
+func TestCSVColumnIterator(t *testing.T) {
+	reader := strings.NewReader(`1,10,683793200
+		5,20,683793300
+		3,41,683793385`)
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, reader)
+	columns := []client.Record{}
+	for {
+		column, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		columns = append(columns, column)
+	}
+	if len(columns) != 3 {
+		t.Fatalf("There should be 3 columns")
+	}
+	target := []client.Column{
+		{RowID: 1, ColumnID: 10, Timestamp: 683793200},
+		{RowID: 5, ColumnID: 20, Timestamp: 683793300},
+		{RowID: 3, ColumnID: 41, Timestamp: 683793385},
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], columns[i]) {
+			t.Fatalf("%v != %v", target[i], columns[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatRowIDColumnID(t *testing.T) {
+	format := "2006-01-02T03:04"
+	reader := strings.NewReader(`1,10,1991-09-02T09:33
+		5,20,1991-09-02T09:35
+		3,41,1991-09-02T09:36`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
+	records := []client.Record{}
+	for {
+		record, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		records = append(records, record)
+	}
+	target := []client.Column{
+		{RowID: 1, ColumnID: 10, Timestamp: 683803980000000000},
+		{RowID: 5, ColumnID: 20, Timestamp: 683804100000000000},
+		{RowID: 3, ColumnID: 41, Timestamp: 683804160000000000},
+	}
+	if len(records) != len(target) {
+		t.Fatalf("There should be %d columns", len(target))
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], records[i]) {
+			t.Fatalf("%v != %v", target[i], records[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatRowKeyColumnKey(t *testing.T) {
+	format := "2006-01-02T03:04"
+	reader := strings.NewReader(`one,ten,1991-09-02T09:33
+		five,twenty,1991-09-02T09:35
+		three,forty-one,1991-09-02T09:36`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowKeyColumnKey, reader, format)
+	records := []client.Record{}
+	for {
+		record, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		records = append(records, record)
+	}
+	target := []client.Column{
+		{RowKey: "one", ColumnKey: "ten", Timestamp: 683803980000000000},
+		{RowKey: "five", ColumnKey: "twenty", Timestamp: 683804100000000000},
+		{RowKey: "three", ColumnKey: "forty-one", Timestamp: 683804160000000000},
+	}
+	if len(records) != len(target) {
+		t.Fatalf("There should be %d columns", len(target))
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], records[i]) {
+			t.Fatalf("%v != %v", target[i], records[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatFail(t *testing.T) {
+	format := "2014-07-16"
+	reader := strings.NewReader(`1,10,X`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestCSVValueIteratorWithColumnID(t *testing.T) {
+	reader := strings.NewReader(`1,10
+		5,-20
+		3,41
+	`)
+	iterator := csv.NewValueIterator(csv.ColumnID, reader)
+	values := []client.Record{}
+	for {
+		value, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		values = append(values, value)
+	}
+	target := []pilosa.FieldValue{
+		{ColumnID: 1, Value: 10},
+		{ColumnID: 5, Value: -20},
+		{ColumnID: 3, Value: 41},
+	}
+	if len(values) != len(target) {
+		t.Fatalf("There should be %d values, got %d", len(target), len(values))
+	}
+	for i := range target {
+		v := values[i].(client.FieldValue)
+		if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
+			t.Fatalf("'%+v' != '%+v'", target[i], values[i])
+		}
+	}
+}
+
+func TestCSVValueIteratorWithColumnKey(t *testing.T) {
+	reader := strings.NewReader(`one,10
+		five,-20
+		three,41
+	`)
+	iterator := csv.NewValueIterator(csv.ColumnKey, reader)
+	values := []client.Record{}
+	for {
+		value, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		values = append(values, value)
+	}
+	target := []pilosa.FieldValue{
+		{ColumnKey: "one", Value: 10},
+		{ColumnKey: "five", Value: -20},
+		{ColumnKey: "three", Value: 41},
+	}
+	if len(values) != len(target) {
+		t.Fatalf("There should be %d values, got %d", len(target), len(values))
+	}
+	for i := range target {
+		v := values[i].(client.FieldValue)
+		if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
+			t.Fatalf("%v != %v", target[i], values[i])
+		}
+	}
+}
+
+func TestCSValueIteratorWithInvalidFormat(t *testing.T) {
+	reader := strings.NewReader("1,2")
+	iterator := csv.NewValueIterator(csv.RowIDColumnID, reader)
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func TestCSVColumnIteratorInvalidInput(t *testing.T) {
+	invalidInputs := []string{
+		// less than 2 columns
+		"155",
+		// invalid row ID
+		"a5,155",
+		// invalid column ID
+		"155,a5",
+		// invalid timestamp
+		"155,255,a5",
+	}
+	for _, text := range invalidInputs {
+		iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
+		_, err := iterator.NextRecord()
+		if err == nil {
+			t.Fatalf("CSVColumnIterator input: %s should fail", text)
+		}
+	}
+}
+
+func TestCSVValueIteratorInvalidInput(t *testing.T) {
+	invalidInputs := []string{
+		// less than 2 columns
+		"155",
+		// invalid column ID
+		"a5,155",
+		// invalid value
+		"155,a5",
+	}
+	for _, text := range invalidInputs {
+		iterator := csv.NewValueIterator(csv.ColumnID, strings.NewReader(text))
+		_, err := iterator.NextRecord()
+		if err == nil {
+			t.Fatalf("CSVValueIterator input: %s should fail", text)
+		}
+	}
+}
+
+func TestCSVColumnIteratorError(t *testing.T) {
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, &BrokenReader{})
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatal("CSVColumnIterator should fail with error")
+	}
+}
+
+func TestCSVValueIteratorError(t *testing.T) {
+	iterator := csv.NewValueIterator(csv.ColumnID, &BrokenReader{})
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatal("CSVValueIterator should fail with error")
+	}
+}
+
+type BrokenReader struct{}
+
+func (r BrokenReader) Read(p []byte) (n int, err error) {
+	return 0, errors.New("broken reader")
+}

client/doc.go

@@ -0,0 +1,68 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+/*
+Package client enables querying a Pilosa server.
+
+This client uses Pilosa's http+protobuf API.
+
+Usage:
+
+	import (
+		"fmt"
+		"github.com/pilosa/pilosa/v2/client"
+	)
+
+	// Create a Client instance
+	cli := client.DefaultClient()
+
+	// Create a Schema instance
+	schema, err := cli.Schema()
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an Index instance
+	index, err := schema.Index("repository")
+	if err != nil {
+		panic(err)
+	}
+
+	// Create a Field instance
+	stargazer, err := index.Field("stargazer")
+	if err != nil {
+		panic(err)
+	}
+
+	// Sync the schema with the server-side, so non-existing indexes/fields are created on the server-side.
+	err = cli.SyncSchema(schema)
+	if err != nil {
+		panic(err)
+	}
+
+	// Execute a query
+	response, err := cli.Query(stargazer.Row(5))
+	if err != nil {
+		panic(err)
+	}
+
+	// Act on the result
+	fmt.Println(response.Result())
+
+See also https://www.pilosa.com/docs/api-reference/ and https://www.pilosa.com/docs/query-language/.
+*/
+package client

client/docs/data-model-queries.md

@@ -0,0 +1,152 @@
+# Data Model and Queries
+
+## Indexes and Fields
+
+*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs/latest/data-model/) for more detail about the data model.
+
+The `schema.Index` function is used to create an index instance. Note that this does not create an index on the server; the index object simply defines the schema.
+
+```go
+schema := client.NewSchema()
+repository := schema.Index("repository")
+```
+
+You can pass options while creating index instances:
+```go
+repository := schema.Index("repository", pilosa.OptIndexKeys(true))
+```
+
+Field definitions are created with a call to the `Field` function of an index:
+
+```go
+stargazer := repository.Field("stargazer")
+```
+
+You can pass options to fields:
+
+```go
+stargazer := repository.Field("stargazer", pilosa.OptFieldTypeTime(TimeQuantumYearMonthDay))
+```
+
+In case the schema already exists on the server, you can retrieve that instead of creating the schema:
+```go
+cli := client.DefaultClient()
+schema, err := cli.Schema()
+if err != nil {
+    // act on the error
+}
+repository := schema.Index("repository")
+```
+
+## Queries
+
+Once you have indexes and field definitions, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows with related methods attached to fields.
+
+For instance, `Row` queries work on rows; use a `Field` object to create those queries:
+
+```go
+rowQuery := stargazer.Row(1)  // corresponds to PQL: Row(stargazer=1)
+```
+
+`Union` queries work on columns; use the index to create them:
+
+```go
+query := repository.Union(rowQuery1, rowQuery2)
+```
+
+In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.BatchQuery` function is used for that purpose:
+
+```go
+query := repository.BatchQuery(
+    stargazer.Row(1),
+    repository.Union(stargazer.Row(100), stargazer.Row(5)))
+```
+
+The recommended way of creating query instances is using dedicated functions attached to index and field objects, but sometimes it would be desirable to send raw queries to Pilosa. You can use `index.RawQuery` method for that. Note that query string is not validated before sending to the server:
+
+```go
+query := repository.RawQuery("Row(stargazer=5)")
+```
+
+Raw queries are only sent to the coordinator node of a Pilosa cluster, so currently there's a possible performance hit using them instead of ORM functions attached to index or field instances.
+
+This client supports [range queries using bit sliced indexes (BSI)](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read the [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa.
+
+In order to use BSI range queries, an integer field should be created. The field should have its minimum and maximum set. Here's how you would do that:
+```go
+index := schema.Index("animals")
+captivity := index.Field("captivity", pilosa.OptFieldTypeInt(0, 956))
+```
+
+If the field with the necessary field already exists on the server, you don't need to create the field instance, `cli.SyncSchema(schema)` would load that to `schema`. You can then add some data:
+```go
+// Add the captivity values to the field.
+data := []int{3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318}
+query := index.BatchQuery()
+for i, x := range data {
+	column := uint64(i + 1)
+	query.Add(captivity.SetIntValue(column, x))
+}
+cli.Query(query)
+```
+
+Let's write a range query:
+```go
+// Query for all animals with more than 100 specimens
+response, _ := cli.Query(captivity.GT(100))
+fmt.Println(response.Result().Row().Columns)
+
+// Query for the total number of animals in captivity
+response, _ = cli.Query(captivity.Sum(nil))
+fmt.Println(response.Result().Value())
+```
+
+If you pass a row query to `Sum` as a filter, then only the columns matching the filter will be considered in the `Sum` calculation:
+```go
+// Let's run a few set queries first
+cli.Query(index.BatchQuery(
+    field.Set(42, 1),
+    field.Set(42, 6)))
+// Query for the total number of animals in captivity where row 42 is set
+response, _ = cli.Query(captivity.Sum(field.Row(42)))
+fmt.Println(response.Result().Value())
+```
+
+See the functions further below for the list of functions that can be used with a `Field`.
+
+Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details. Here is a list of methods corresponding to PQL calls:
+
+Index:
+
+* `Union(rows *PQLRowQuery...) *PQLRowQuery`
+* `Intersect(rows *PQLRowQuery...) *PQLRowQuery`
+* `Difference(rows *PQLRowQuery...) *PQLRowQuery`
+* `Xor(rows ...*PQLRowQuery) *PQLRowQuery`
+* `Not(row) *PQLRowQuery`
+* `Count(row *PQLRowQuery) *PQLBaseQuery`
+* `SetColumnAttrs(columnID uint64, attrs map[string]interface{}) *PQLBaseQuery`
+* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery`
+
+Field:
+
+* `Row(rowID uint64) *PQLRowQuery`
+* `Set(rowID uint64, columnID uint64) *PQLBaseQuery`
+* `SetTimestamp(rowID uint64, columnID uint64, timestamp time.Time) *PQLBaseQuery`
+* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery`
+* `TopN(n uint64) *PQLRowQuery`
+* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery`
+* `FilterFieldTopN(n uint64, row *PQLRowQuery, field string, values ...interface{}) *PQLRowQuery`
+* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
+* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
+* `SetRowAttrs(rowID uint64, attrs map[string]interface{}) *PQLBaseQuery`
+* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery`
+* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery`
+* `LT(n int) *PQLRowQuery`
+* `LTE(n int) *PQLRowQuery`
+* `GT(n int) *PQLRowQuery`
+* `GTE(n int) *PQLRowQuery`
+* `Between(a int, b int) *PQLRowQuery`
+* `Sum(row *PQLRowQuery) *PQLBaseQuery`
+* `Min(row *PQLRowQuery) *PQLBaseQuery`
+* `Max(row *PQLRowQuery) *PQLBaseQuery`
+* `SetIntValue(columnID uint64, value int) *PQLBaseQuery`

client/docs/server-interaction.md

@@ -0,0 +1,178 @@
+# Server Interaction
+
+## Pilosa URI
+
+A Pilosa URI has the `${SCHEME}://${HOST}:${PORT}` format:
+* **Scheme**: Protocol of the URI. Default: `http`.
+* **Host**: Hostname or ipv4/ipv6 IP address. Default: localhost.
+* **Port**: Port number. Default: `10101`.
+
+All parts of the URI are optional, but at least one of them must be specified. The following are equivalent:
+
+* `http://localhost:10101`
+* `http://localhost`
+* `http://:10101`
+* `localhost:10101`
+* `localhost`
+* `:10101`
+
+A Pilosa URI is represented by the `github.com/pilosa/pilosa/v2/net URI` struct. Below are a few ways to create `URI` objects:
+
+```go
+import pnet "github.com/pilosa/pilosa/v2/net"
+
+// create the default URI: http://localhost:10101
+uri1 := pnet.DefaultURI()
+
+// create a URI from string address
+uri2, err := pnet.NewURIFromAddress("index1.pilosa.com:20202");
+
+// create a URI with the given host and port
+uri3, err := pnet.NewURIFromHostPort("index1.pilosa.com", 20202);
+```
+
+## Pilosa Client
+
+In order to interact with a Pilosa server, an instance of `client.Client` should be created. The client is thread-safe and uses a pool of connections to the server, so we recommend creating a single instance of the client and sharing it when necessary.
+
+If the Pilosa server is running at the default address (`http://localhost:10101`) you can create the client with default options using:
+
+```go
+import "github.com/pilosa/pilosa/v2/client"
+
+cli := client.DefaultClient()
+```
+
+To use a custom server address, use the `NewClient` function:
+
+```go
+uri, err := pnet.NewURIFromAddress("http://index1.pilosa.com:15000")
+if err != nil {
+    // Act on the error
+}
+cli, err := client.NewClient(uri)
+```
+
+Equivalently:
+```go
+cli, err := client.NewClient("http://index1.pilosa.com:15000")
+```
+
+If you are running a cluster of Pilosa servers, you can create a `Cluster` struct that keeps addresses of those servers:
+
+```go
+uri1, err := pnet.NewURIFromAddress(":10101")
+uri2, err := pnet.NewURIFromAddress(":10110")
+uri3, err := pnet.NewURIFromAddress(":10111")
+cluster := client.NewClusterWithHost(uri1, uri2, uri3)
+
+// Create a client with the cluster
+cli, err := client.NewClient(cluster)
+```
+
+That is equivalent to:
+```go
+cli, err := client.NewClient([]string{":10101", ":10110", ":10111"})
+
+```
+
+It is possible to customize the behaviour of the underlying HTTP client by passing `ClientOption` structs to the `NewClient` function:
+
+```go
+cli, err := client.NewClient(cluster,
+    client.OptClientConnectTimeout(1000),  // if can't connect in  a second, close the connection
+    client.OptClientSocketTimeout(10000),  // if no response received in 10 seconds, close the connection
+    client.OptClientPoolSizePerRoute(3),  // number of connections in the pool per host
+    client.OptClientTotalPoolSize(10))   // number of total connections in the pool
+```
+
+Once you create a client, you can create indexes, fields or start sending queries.
+
+Here is how you would create a index and field:
+
+```go
+// materialize repository index definition and stargazer field definition initialized before
+err := cli.SyncSchema(schema)
+```
+
+You can send queries to a Pilosa server using the `Query` function of the `Client` struct:
+
+```go
+response, err := cli.Query(field.Row(5));
+```
+
+`Query` accepts zero or more options:
+
+```go
+response, err := cli.Query(field.Row(5), pilosa.ColumnAttrs(true), pilosa.ExcludeColumns(true))
+```
+
+## Server Response
+
+When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned.
+
+A `QueryResponse` struct may contain zero or more results of `QueryResult` type. You can access all results using the `Results` function of `QueryResponse` (which returns a list of `QueryResult` objects), or you can use the `Result` method (which returns either the first result or `nil` if there are no results):
+
+```go
+response, err := cli.Query(field.Row(5))
+if err != nil {
+    // Act on the error
+}
+
+// check that there's a result and act on it
+result := response.Result()
+if result != nil {
+    // Act on the result
+}
+
+// iterate over all results
+for _, result := range response.Results() {
+    // Act on the result
+}
+```
+
+Similarly, a `QueryResponse` struct may include a number of column attributes if `ColumnAttrs` query option was set to `true`:
+
+```go
+var column *pilosa.ColumnItem
+
+// iterate over all columns
+for _, column = range response.ColumnAttrs() {
+    // Act on the column item
+}
+```
+
+`QueryResult` objects contain:
+
+* `Row()` function to retrieve a row result,
+* `CountItems()` function to retrieve column count per row ID entries returned from `TopN` queries,
+* `Count()` function to retrieve the number of rows per the given row ID returned from `Count` queries.
+* `Value()` function to retrieve the result of `Min`, `Max` or `Sum` queries.
+* `Changed()` function returns whether a `Set` or `Clear` query changed a column.
+
+```go
+row := result.Row()
+columns := row.Columns
+attributes := row.Attributes
+
+countItems := result.CountItems()
+
+count := result.Count()
+
+value := result.Value()
+
+changed := result.Changed()
+```
+
+## SSL/TLS
+
+Make sure the Pilosa server runs on a TLS address. [How To Set Up a Secure Cluster](https://www.pilosa.com/docs/latest/tutorials/#how-to-set-up-a-secure-cluster) tutorial explains how to do that.
+
+In order to enable TLS support on the client side, the scheme of the address should be explicitly specified as `https`, e.g.: `https://01.pilosa.local:10501`
+
+This client library uses the `net/http` module of Go standard library. You can pass a [tls.Config](https://golang.org/pkg/crypto/tls/#Config) struct in a `pilosa.TLSConfig` option to the client. If the Pilosa server is using a certificate from a recognized authority, you can use the defaults.
+
+If you are using a self signed certificate, just pass `pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})` to `pilosa.NewClient` function:
+```go
+client, _ := pilosa.NewClient("https://01.pilosa.local:10501", pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true}))
+```

client/docs/tracing.md

@@ -0,0 +1,111 @@
+# Tracing
+
+Pilosa client supports distributed tracing via the [OpenTracing](https://opentracing.io/) API.
+
+In order to use a tracer with Go-Pilosa, you should:
+1. Create the tracer,
+2. Pass the `OptClientOption(tracer)` to `NewClient`.
+
+In this document, we will be using the [Jaeger](https://www.jaegertracing.io) tracer, but OpenTracing has support for [other tracing systems](https://opentracing.io/docs/supported-tracers/).
+
+## Running the Pilosa Server
+
+Let's run a temporary Pilosa container:
+
+    $ docker run -it --rm -p 10101:10101 pilosa/pilosa:v1.2.0
+
+Check that you can access Pilosa:
+
+    $ curl localhost:10101
+    Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.
+
+## Running the Jaeger Server
+
+Let's run a Jaeger Server container:
+
+    $ docker run -it --rm -p 5775:5775/udp -p 16686:16686 jaegertracing/all-in-one:latest
+    ...<title>Jaeger UI</title>...
+
+## Writing the Sample Code
+
+The sample code depdends on the Jaeger Go client, so let's install it first:
+
+    $ go get -u github.com/uber/jaeger-client-go/
+
+Save the following sample code as `gopilosa-tracing.go`:
+```go
+package main
+
+import (
+	"log"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/client"
+	"github.com/uber/jaeger-client-go"
+	"github.com/uber/jaeger-client-go/config"
+)
+
+func main() {
+	// Create the tracer.
+	cfg := config.Configuration{
+		Sampler: &config.SamplerConfig{
+			Type:  "const",
+			Param: 1,
+		},
+		Reporter: &config.ReporterConfig{
+			LogSpans:            true,
+			BufferFlushInterval: 1 * time.Second,
+			// Jaeger Server address
+			LocalAgentHostPort:  "127.0.0.1:5775",
+		},
+	}
+	tracer, closer, err := cfg.New(
+		"go_pilosa_test",
+		config.Logger(jaeger.StdLogger),
+	)
+
+	// Don't forget to close the tracer.
+	defer closer.Close()
+
+	// Create the client, and pass the tracer.
+	cli, err := client.NewClient(":10101", pilosa.OptClientTracer(tracer))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Read the schema from the server.
+	// This should create a trace on the Jaeger server.
+	schema, err := cli.Schema()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Create and sync the sample schema.
+	// This should create a trace on the Jaeger server.
+	myIndex := schema.Index("my-index")
+	myField := myIndex.Field("my-field")
+	err = cli.SyncSchema(schema)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Run a query on Pilosa.
+	// This should create a trace on the Jaeger server.
+	_, err = cli.Query(myField.Set(1, 1000))
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+```
+
+## Checking the Tracing Data
+
+Run the sample code:
+
+    $ go run gopilosa-tracing.go
+
+
+* Open http://localhost:16686 in your web browser to visit Jaeger UI.
+* Click on the *Search* tab and select `go_pilosa_test` in the *Service* dropdown on the right.
+* Click on *Find Traces* button at the bottom left.
+* You should see a couple of traces, such as: `Client.Query`, `Client.CreateField`, `Client.Schema`, etc.

client/egpool/egpool.go

@@ -0,0 +1,123 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package egpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+type Group struct {
+	PoolSize int
+
+	jobs chan func() error
+
+	sema     chan struct{}
+	errMu    sync.Mutex
+	firstErr error
+	errs     []error
+}
+
+func (eg *Group) Go(f func() error) {
+	if eg.PoolSize <= 0 {
+		eg.PoolSize = 1
+	}
+
+	if eg.jobs == nil {
+		eg.jobs = make(chan func() error)
+		eg.sema = make(chan struct{}, eg.PoolSize)
+	}
+
+	// Start the job in an idle worker if possible.
+	select {
+	case eg.jobs <- f:
+		return
+	default:
+	}
+
+	// Start a new worker if necessary.
+	select {
+	case eg.jobs <- f:
+		// A worker finished its previous job and took this one over.
+		return
+	case eg.sema <- struct{}{}:
+		// Start a new worker.
+		go eg.processJobs()
+		eg.jobs <- f
+	}
+}
+
+func (eg *Group) err(err error) {
+	eg.errMu.Lock()
+	defer eg.errMu.Unlock()
+
+	if eg.firstErr == nil {
+		eg.firstErr = err
+	}
+	eg.errs = append(eg.errs, err)
+}
+
+type ErrPanic struct {
+	Value interface{}
+}
+
+func (p ErrPanic) Error() string {
+	return fmt.Sprintf("panic: %v", p.Value)
+}
+
+var ErrGoexit = errors.New("runtime.Goexit used in job function")
+
+func (eg *Group) processJobs() {
+	// Notify pool of shutdown.
+	defer func() { <-eg.sema }()
+
+	// Handle panic and Goexit.
+	var finished bool
+	defer func() {
+		if !finished {
+			if p := recover(); p != nil {
+				eg.err(ErrPanic{p})
+			} else {
+				eg.err(ErrGoexit)
+			}
+		}
+	}()
+
+	// Run jobs from queue.
+	for jobFn := range eg.jobs {
+		err := jobFn()
+		if err != nil {
+			eg.err(err)
+		}
+	}
+
+	finished = true
+}
+
+func (eg *Group) Wait() error {
+	if eg.jobs == nil {
+		return nil
+	}
+	close(eg.jobs)
+	for i := 0; i < eg.PoolSize; i++ {
+		eg.sema <- struct{}{}
+	}
+	return eg.firstErr
+}
+
+func (eg *Group) Errors() []error {
+	return eg.errs
+}

client/egpool/egpool_test.go

@@ -0,0 +1,50 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package egpool_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/client/egpool"
+)
+
+func TestEGPool(t *testing.T) {
+	eg := egpool.Group{}
+
+	a := make([]int, 10)
+
+	for i := 0; i < 10; i++ {
+		i := i
+		eg.Go(func() error {
+			a[i] = i
+			if i == 7 {
+				return errors.New("blah")
+			}
+			return nil
+		})
+	}
+
+	err := eg.Wait()
+	if err == nil || err.Error() != "blah" {
+		t.Errorf("expected err blah, got: %v", err)
+	}
+
+	for i := 0; i < 10; i++ {
+		if a[i] != i {
+			t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i])
+		}
+	}
+}

client/error.go

@@ -0,0 +1,38 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import "github.com/pkg/errors"
+
+// Predefined Pilosa errors.
+var (
+	ErrEmptyCluster                = errors.New("No usable addresses in the cluster")
+	ErrIndexExists                 = errors.New("Index exists")
+	ErrFieldExists                 = errors.New("Field exists")
+	ErrInvalidIndexName            = errors.New("Invalid index name")
+	ErrInvalidFieldName            = errors.New("Invalid field name")
+	ErrInvalidLabel                = errors.New("Invalid label")
+	ErrInvalidKey                  = errors.New("Invalid key")
+	ErrTriedMaxHosts               = errors.New("Tried max hosts, still failing")
+	ErrAddrURIClusterExpected      = errors.New("Addresses, URIs or a cluster is expected")
+	ErrInvalidQueryOption          = errors.New("Invalid query option")
+	ErrInvalidIndexOption          = errors.New("Invalid index option")
+	ErrInvalidFieldOption          = errors.New("Invalid field option")
+	ErrNoFragmentNodes             = errors.New("No fragment nodes")
+	ErrNoShard                     = errors.New("Index has no shards")
+	ErrUnknownType                 = errors.New("Unknown type")
+	ErrSingleServerAddressRequired = errors.New("OptClientManualServerAddress requires a single URI or address")
+	ErrPreconditionFailed          = errors.New("Precondition failed")
+)

client/logimport.go

@@ -0,0 +1,45 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"encoding/gob"
+	"io"
+)
+
+type importLog struct {
+	Index     string
+	Path      string
+	Shard     uint64
+	IsRoaring bool
+	Timestamp int64 // Unix Nanoseconds
+	Data      []byte
+}
+
+type encoder interface {
+	Encode(thing interface{}) error
+}
+
+func newImportLogEncoder(w io.Writer) encoder {
+	return gob.NewEncoder(w)
+}
+
+type decoder interface {
+	Decode(thing interface{}) error
+}
+
+func newImportLogDecoder(r io.Reader) decoder {
+	return gob.NewDecoder(r)
+}

client/logimport_test.go

@@ -0,0 +1,155 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	tests := []importLog{
+		{
+			Index: "go-testindex",
+			Path:  "/index/go-testindex/field/importfield-batchsize/import?clear=false",
+			Shard: 0,
+			Data:  make([]byte, 3918),
+		},
+		{
+			Index: "go-testindex",
+			Path:  "/index/go-testindex/field/importfield-batchsize/import?clear=false",
+			Shard: 0,
+			Data:  make([]byte, 3918),
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 9,
+			Data:  []byte("something"),
+		},
+		{
+			Index: "",
+			Path:  "",
+			Shard: 0,
+			Data:  nil,
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 10,
+			Data:  []byte("blahaslkdjfeoiwujf"),
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 10,
+			Data:  make([]byte, 10000),
+		},
+		{
+			Index: "zoop",
+			Path:  "blah",
+			Shard: 8923734,
+			Data:  []byte("blahaslkdjfeoiwujf"),
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			nl := importLog{
+				Index: test.Index,
+				Path:  test.Path,
+				Shard: test.Shard,
+				Data:  make([]byte, len(test.Data)),
+			}
+			copy(nl.Data, test.Data)
+			buf := &bytes.Buffer{}
+			enc := newImportLogEncoder(buf)
+			err := enc.Encode(nl)
+			if err != nil {
+				t.Fatalf("writing to buf: %v", err)
+			}
+			dec := newImportLogDecoder(buf)
+			l2 := &importLog{}
+			err = dec.Decode(l2)
+			if err != nil {
+				t.Fatalf("reading from buf: %v", err)
+			}
+			if l2.Index != test.Index {
+				t.Errorf("indexes not equal:\n%s\n%s", test.Index, l2.Index)
+			}
+			if l2.Path != test.Path {
+				t.Errorf("paths not equal:\n%s\n%s", test.Path, l2.Path)
+			}
+			if l2.Shard != test.Shard {
+				t.Errorf("shards not equal exp: %d got %d", test.Shard, l2.Shard)
+			}
+			if !reflect.DeepEqual(test.Data, l2.Data) {
+				t.Errorf("data not equal \n%v\n%v", test.Data, l2.Data)
+			}
+
+		})
+	}
+	buf, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("getting temp file: %v", err)
+	}
+	enc := newImportLogEncoder(buf)
+	for _, test := range tests {
+		a := &test
+		err := enc.Encode(a)
+		if err != nil {
+			t.Errorf("encoding to buf: %v", err)
+		}
+	}
+
+	name := buf.Name()
+	err = buf.Close()
+	if err != nil {
+		t.Fatalf("closing temp file: %v", err)
+	}
+
+	buf, err = os.Open(name)
+	if err != nil {
+		t.Fatalf("reopening: %v", err)
+	}
+
+	dec := newImportLogDecoder(buf)
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			l := &importLog{}
+			err := dec.Decode(l)
+			// err := l.ReadFrom(buf)
+			if err != nil {
+				t.Errorf("reading from buf: %v", err)
+			}
+			if l.Index != test.Index {
+				t.Errorf("indexes not equal:\n%s\n%s", test.Index, l.Index)
+			}
+			if l.Path != test.Path {
+				t.Errorf("paths not equal:\n%s\n%s", test.Path, l.Path)
+			}
+			if l.Shard != test.Shard {
+				t.Errorf("shards not equal exp: %d got %d", test.Shard, l.Shard)
+			}
+			if !reflect.DeepEqual(test.Data, l.Data) {
+				t.Errorf("data not equal \n%v\n%v", test.Data, l.Data)
+			}
+		})
+	}
+}

client/metrics.go

@@ -0,0 +1,29 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+const (
+	// MetricBatchImportDurationSeconds records the full time of the
+	// RecordBatch.Import call. This includes starting and finishing a
+	// transaction, doing key translation, building fragments locally,
+	// importing all data, and resetting internal structures.
+	MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
+
+	// MetricBatchFlushDurationSeconds records the full time for
+	// RecordBatch.Flush (if splitBatchMode is in use). This includes
+	// starting and finishing a transaction, importing all data, and
+	// resetting internal structures.
+	MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
+)

client/record.go

@@ -0,0 +1,75 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+// Record is a Column or a FieldValue.
+type Record interface {
+	Shard(shardWidth uint64) uint64
+	Less(other Record) bool
+}
+
+// RecordIterator is an iterator for a record.
+type RecordIterator interface {
+	NextRecord() (Record, error)
+}
+
+// Column defines a single Pilosa column.
+type Column struct {
+	RowID     uint64
+	ColumnID  uint64
+	RowKey    string
+	ColumnKey string
+	Timestamp int64
+}
+
+// Shard returns the shard for this column.
+func (b Column) Shard(shardWidth uint64) uint64 {
+	return b.ColumnID / shardWidth
+}
+
+// Less returns true if this column sorts before the given Record.
+func (b Column) Less(other Record) bool {
+	if ob, ok := other.(Column); ok {
+		if b.RowID == ob.RowID {
+			return b.ColumnID < ob.ColumnID
+		}
+		return b.RowID < ob.RowID
+	}
+	return false
+}
+
+// FieldValue represents the value for a column within a
+// range-encoded field.
+type FieldValue struct {
+	ColumnID  uint64
+	ColumnKey string
+	Value     int64
+}
+
+// Shard returns the shard for this field value.
+func (v FieldValue) Shard(shardWidth uint64) uint64 {
+	return v.ColumnID / shardWidth
+}
+
+// Less returns true if this field value sorts before the given Record.
+func (v FieldValue) Less(other Record) bool {
+	if ov, ok := other.(FieldValue); ok {
+		return v.ColumnID < ov.ColumnID
+	}
+	return false
+}

client/record_test.go

@@ -0,0 +1,83 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client_test
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/client"
+)
+
+func TestColumnShard(t *testing.T) {
+	a := client.Column{RowID: 15, ColumnID: 55, Timestamp: 100101}
+	target := uint64(0)
+	if a.Shard(100) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(100))
+	}
+	target = 5
+	if a.Shard(10) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(10))
+	}
+}
+
+func TestColumnLess(t *testing.T) {
+	a := client.Column{RowID: 10, ColumnID: 200}
+	a2 := client.Column{RowID: 10, ColumnID: 1000}
+	b := client.Column{RowID: 200, ColumnID: 10}
+	c := client.FieldValue{ColumnID: 1}
+	if !a.Less(a2) {
+		t.Fatalf("%v should be less than %v", a, a2)
+	}
+	if !a.Less(b) {
+		t.Fatalf("%v should be less than %v", a, b)
+	}
+	if b.Less(a) {
+		t.Fatalf("%v should not be less than %v", b, a)
+	}
+	if c.Less(a) {
+		t.Fatalf("%v should not be less than %v", c, a)
+	}
+}
+
+func TestFieldValueShard(t *testing.T) {
+	a := client.FieldValue{ColumnID: 55, Value: 125}
+	target := uint64(0)
+	if a.Shard(100) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(100))
+	}
+	target = 5
+	if a.Shard(10) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(10))
+	}
+
+}
+
+func TestFieldValueLess(t *testing.T) {
+	a := client.FieldValue{ColumnID: 55, Value: 125}
+	b := client.FieldValue{ColumnID: 100, Value: 125}
+	c := client.Column{ColumnID: 1, RowID: 2}
+	if !a.Less(b) {
+		t.Fatalf("%v should be less than %v", a, b)
+	}
+	if b.Less(a) {
+		t.Fatalf("%v should not be less than %v", b, a)
+	}
+	if c.Less(a) {
+		t.Fatalf("%v should not be less than %v", c, a)
+	}
+}

client/response.go

@@ -0,0 +1,595 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/pilosa/pilosa/v2/pb"
+)
+
+// QueryResponse types.
+const (
+	QueryResultTypeNil uint32 = iota
+	QueryResultTypeRow
+	QueryResultTypePairs
+	QueryResultTypePairsField
+	QueryResultTypeValCount
+	QueryResultTypeUint64
+	QueryResultTypeBool
+	QueryResultTypeRowIDs // this is not used by the client
+	QueryResultTypeGroupCounts
+	QueryResultTypeRowIdentifiers
+	QueryResultTypePair
+	QueryResultTypePairField
+	QueryResultTypeSignedRow
+)
+
+// QueryResponse represents the response from a Pilosa query.
+type QueryResponse struct {
+	ResultList   []QueryResult `json:"results,omitempty"`
+	ColumnList   []ColumnItem  `json:"columns,omitempty"`
+	ErrorMessage string        `json:"error-message,omitempty"`
+	Success      bool          `json:"success,omitempty"`
+}
+
+func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, error) {
+	if response.Err != "" {
+		return &QueryResponse{
+			ErrorMessage: response.Err,
+			Success:      false,
+		}, nil
+	}
+	results := make([]QueryResult, 0, len(response.Results))
+	for _, r := range response.Results {
+		result, err := newQueryResultFromInternal(r)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, result)
+	}
+	columns := make([]ColumnItem, 0, len(response.ColumnAttrSets))
+	for _, p := range response.ColumnAttrSets {
+		columnItem, err := newColumnItemFromInternal(p)
+		if err != nil {
+			return nil, err
+		}
+		columns = append(columns, columnItem)
+	}
+
+	return &QueryResponse{
+		ResultList: results,
+		ColumnList: columns,
+		Success:    true,
+	}, nil
+}
+
+// Results returns all results in the response.
+func (qr *QueryResponse) Results() []QueryResult {
+	return qr.ResultList
+}
+
+// Result returns the first result or nil.
+func (qr *QueryResponse) Result() QueryResult {
+	if len(qr.ResultList) == 0 {
+		return nil
+	}
+	return qr.ResultList[0]
+}
+
+// Columns returns all column attributes in the response.
+// *DEPRECATED*
+func (qr *QueryResponse) Columns() []ColumnItem {
+	return qr.ColumnList
+}
+
+// Column returns the attributes for first column.
+// *DEPRECATED*
+func (qr *QueryResponse) Column() ColumnItem {
+	if len(qr.ColumnList) == 0 {
+		return ColumnItem{}
+	}
+	return qr.ColumnList[0]
+}
+
+// ColumnAttrs returns all column attributes in the response.
+func (qr *QueryResponse) ColumnAttrs() []ColumnItem {
+	return qr.ColumnList
+}
+
+// QueryResult represents one of the results in the response.
+type QueryResult interface {
+	Type() uint32
+	Row() RowResult
+	CountItems() []CountResultItem
+	CountItem() CountResultItem
+	Count() int64
+	Value() int64
+	Changed() bool
+	GroupCounts() []GroupCount
+	RowIdentifiers() RowIdentifiersResult
+}
+
+func newQueryResultFromInternal(result *pb.QueryResult) (QueryResult, error) {
+	switch result.Type {
+	case QueryResultTypeNil:
+		return NilResult{}, nil
+	case QueryResultTypeRow:
+		return newRowResultFromInternal(result.Row)
+	case QueryResultTypePairs:
+		return countItemsFromInternal(result.Pairs), nil
+	case QueryResultTypePairsField:
+		return countItemsFromInternal(result.PairsField.Pairs), nil
+	case QueryResultTypeValCount:
+		return &ValCountResult{
+			Val: result.ValCount.Val,
+			Cnt: result.ValCount.Count,
+		}, nil
+	case QueryResultTypeUint64:
+		return IntResult(result.N), nil
+	case QueryResultTypeBool:
+		return BoolResult(result.Changed), nil
+	case QueryResultTypeRowIdentifiers:
+		return &RowIdentifiersResult{
+			IDs:  result.RowIdentifiers.Rows,
+			Keys: result.RowIdentifiers.Keys,
+		}, nil
+	case QueryResultTypeGroupCounts:
+		return groupCountsFromInternal(result.GroupCounts), nil
+	case QueryResultTypePair:
+		return CountItem{CountResultItem: countItemFromInternal(result.Pairs[0])}, nil
+	case QueryResultTypePairField:
+		return CountItem{CountResultItem: countItemFromInternal(result.PairField.Pair)}, nil
+	}
+
+	return nil, ErrUnknownType
+}
+
+// CountResultItem represents a result from TopN call.
+type CountResultItem struct {
+	ID    uint64 `json:"id"`
+	Key   string `json:"key,omitempty"`
+	Count uint64 `json:"count"`
+}
+
+func (c *CountResultItem) String() string {
+	if c.Key != "" {
+		return fmt.Sprintf("%s:%d", c.Key, c.Count)
+	}
+	return fmt.Sprintf("%d:%d", c.ID, c.Count)
+}
+
+type CountItem struct {
+	CountResultItem
+}
+
+// Type is the type of this result.
+func (CountItem) Type() uint32 { return QueryResultTypePairField }
+
+// Row returns a RowResult.
+func (CountItem) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (t CountItem) CountItems() []CountResultItem { return []CountResultItem{t.CountResultItem} }
+
+// CountItem returns a CountResultItem
+func (t CountItem) CountItem() CountResultItem { return t.CountResultItem }
+
+// Count returns the result of a Count call.
+func (CountItem) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (CountItem) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (CountItem) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (CountItem) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (CountItem) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+func countItemFromInternal(item *pb.Pair) CountResultItem {
+	return CountResultItem{ID: item.ID, Key: item.Key, Count: item.Count}
+}
+
+func countItemsFromInternal(items []*pb.Pair) TopNResult {
+	result := make([]CountResultItem, 0, len(items))
+	for _, v := range items {
+		result = append(result, countItemFromInternal(v))
+	}
+	return TopNResult(result)
+}
+
+// TopNResult is returned from TopN call.
+type TopNResult []CountResultItem
+
+// Type is the type of this result.
+func (TopNResult) Type() uint32 { return QueryResultTypePairsField }
+
+// Row returns a RowResult.
+func (TopNResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (t TopNResult) CountItems() []CountResultItem { return t }
+
+// CountItem returns a CountResultItem
+func (t TopNResult) CountItem() CountResultItem {
+	if len(t) >= 1 {
+		return t[0]
+	}
+	return CountResultItem{}
+}
+
+// Count returns the result of a Count call.
+func (TopNResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (TopNResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (TopNResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (TopNResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls.
+type RowResult struct {
+	Attributes map[string]interface{} `json:"attrs"`
+	Columns    []uint64               `json:"columns"`
+	Keys       []string               `json:"keys"`
+}
+
+func newRowResultFromInternal(row *pb.Row) (*RowResult, error) {
+	attrs, err := convertInternalAttrsToMap(row.Attrs)
+	if err != nil {
+		return nil, err
+	}
+	result := &RowResult{
+		Attributes: attrs,
+		Columns:    row.Columns,
+		Keys:       row.Keys,
+	}
+	return result, nil
+}
+
+// Type is the type of this result.
+func (RowResult) Type() uint32 { return QueryResultTypeRow }
+
+// Row returns a RowResult.
+func (b RowResult) Row() RowResult { return b }
+
+// CountItems returns a CountResultItem slice.
+func (RowResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (RowResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (RowResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (RowResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (RowResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (RowResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (RowResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// MarshalJSON serializes this row result.
+func (b RowResult) MarshalJSON() ([]byte, error) {
+	columns := b.Columns
+	if columns == nil {
+		columns = []uint64{}
+	}
+	keys := b.Keys
+	if keys == nil {
+		keys = []string{}
+	}
+	return json.Marshal(struct {
+		Attributes map[string]interface{} `json:"attrs"`
+		Columns    []uint64               `json:"columns"`
+		Keys       []string               `json:"keys"`
+	}{
+		Attributes: b.Attributes,
+		Columns:    columns,
+		Keys:       keys,
+	})
+}
+
+// ValCountResult is returned from Min, Max and Sum calls.
+type ValCountResult struct {
+	Val int64 `json:"val"`
+	Cnt int64 `json:"count"`
+}
+
+// Type is the type of this result.
+func (ValCountResult) Type() uint32 { return QueryResultTypeValCount }
+
+// Row returns a RowResult.
+func (ValCountResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (ValCountResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (ValCountResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (c ValCountResult) Count() int64 { return c.Cnt }
+
+// Value returns the result of a Min, Max or Sum call.
+func (c ValCountResult) Value() int64 { return c.Val }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (ValCountResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (ValCountResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (ValCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// IntResult is returned from Count call.
+type IntResult int64
+
+// Type is the type of this result.
+func (IntResult) Type() uint32 { return QueryResultTypeUint64 }
+
+// Row returns a RowResult.
+func (IntResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (IntResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (IntResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (i IntResult) Count() int64 { return int64(i) }
+
+// Value returns the result of a Min, Max or Sum call.
+func (IntResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (IntResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (IntResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (IntResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// BoolResult is returned from Set and Clear calls.
+type BoolResult bool
+
+// Type is the type of this result.
+func (BoolResult) Type() uint32 { return QueryResultTypeBool }
+
+// Row returns a RowResult.
+func (BoolResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (BoolResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (BoolResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (BoolResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (BoolResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (b BoolResult) Changed() bool { return bool(b) }
+
+// GroupCounts returns the result of a GroupBy call.
+func (BoolResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// NilResult is returned from calls which don't return a value, such as SetRowAttrs.
+type NilResult struct{}
+
+// Type is the type of this result.
+func (NilResult) Type() uint32 { return QueryResultTypeNil }
+
+// Row returns a RowResult.
+func (NilResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (NilResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (NilResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (NilResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (NilResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (NilResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (NilResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (NilResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// FieldRow represents a Group in a GroupBy call result.
+type FieldRow struct {
+	FieldName string `json:"field"`
+	RowID     uint64 `json:"rowID"`
+	RowKey    string `json:"rowKey"`
+	Value     *int64 `json:"value,omitempty"`
+}
+
+// GroupCount contains groups and their count in a GroupBy call result.
+type GroupCount struct {
+	Groups []FieldRow `json:"groups"`
+	Count  int64      `json:"count"`
+	Agg    int64      `json:"agg"`
+}
+
+// GroupCountResult is returned from GroupBy call.
+type GroupCountResult []GroupCount
+
+// Type is the type of this result.
+func (GroupCountResult) Type() uint32 { return QueryResultTypeGroupCounts }
+
+// Row returns a RowResult.
+func (GroupCountResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (GroupCountResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (GroupCountResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (GroupCountResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (GroupCountResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (GroupCountResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (r GroupCountResult) GroupCounts() []GroupCount { return r }
+
+// RowIdentifiers returns the result of a Rows call.
+func (GroupCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// RowIdentifiersResult is returned from a Rows call.
+type RowIdentifiersResult struct {
+	IDs  []uint64 `json:"ids"`
+	Keys []string `json:"keys,omitempty"`
+}
+
+// Type is the type of this result.
+func (RowIdentifiersResult) Type() uint32 { return QueryResultTypeRowIdentifiers }
+
+// Row returns a RowResult.
+func (RowIdentifiersResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (RowIdentifiersResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (RowIdentifiersResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (RowIdentifiersResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (RowIdentifiersResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (RowIdentifiersResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (RowIdentifiersResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (r RowIdentifiersResult) RowIdentifiers() RowIdentifiersResult { return r }
+
+func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult {
+	result := make([]GroupCount, 0, len(items.Groups))
+	for _, g := range items.Groups {
+		groups := make([]FieldRow, 0, len(g.Group))
+		for _, f := range g.Group {
+			fr := FieldRow{
+				FieldName: f.Field,
+				RowID:     f.RowID,
+				RowKey:    f.RowKey,
+			}
+			if f.Value != nil {
+				fr.Value = &f.Value.Value
+			}
+			groups = append(groups, fr)
+		}
+		result = append(result, GroupCount{
+			Groups: groups,
+			Count:  int64(g.Count),
+			Agg:    int64(g.Agg),
+		})
+	}
+	return GroupCountResult(result)
+}
+
+const (
+	stringType = 1
+	intType    = 2
+	boolType   = 3
+	floatType  = 4
+)
+
+func convertInternalAttrsToMap(attrs []*pb.Attr) (attrsMap map[string]interface{}, err error) {
+	attrsMap = make(map[string]interface{}, len(attrs))
+	for _, attr := range attrs {
+		switch attr.Type {
+		case stringType:
+			attrsMap[attr.Key] = attr.StringValue
+		case intType:
+			attrsMap[attr.Key] = attr.IntValue
+		case boolType:
+			attrsMap[attr.Key] = attr.BoolValue
+		case floatType:
+			attrsMap[attr.Key] = attr.FloatValue
+		default:
+			return nil, errors.New("Unknown attribute type")
+		}
+	}
+
+	return attrsMap, nil
+}
+
+// ColumnItem represents data about a column.
+// Column data is only returned if QueryOptions.Columns was set to true.
+type ColumnItem struct {
+	ID         uint64                 `json:"id,omitempty"`
+	Key        string                 `json:"key,omitempty"`
+	Attributes map[string]interface{} `json:"attributes,omitempty"`
+}
+
+func newColumnItemFromInternal(column *pb.ColumnAttrSet) (ColumnItem, error) {
+	attrs, err := convertInternalAttrsToMap(column.Attrs)
+	if err != nil {
+		return ColumnItem{}, err
+	}
+	return ColumnItem{
+		ID:         column.ID,
+		Key:        column.Key,
+		Attributes: attrs,
+	}, nil
+}

client/response_test.go

@@ -0,0 +1,348 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/pb"
+)
+
+func TestNewRowResultFromInternal(t *testing.T) {
+	targetAttrs := map[string]interface{}{
+		"name":       "some string",
+		"age":        int64(95),
+		"registered": true,
+		"height":     1.83,
+	}
+	targetColumns := []uint64{5, 10}
+	attrs := []*pb.Attr{
+		{Key: "name", StringValue: "some string", Type: 1},
+		{Key: "age", IntValue: 95, Type: 2},
+		{Key: "registered", BoolValue: true, Type: 3},
+		{Key: "height", FloatValue: 1.83, Type: 4},
+	}
+	row := &pb.Row{
+		Attrs:   attrs,
+		Columns: []uint64{5, 10},
+	}
+	result, err := newRowResultFromInternal(row)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	// assertMapEquals(t, targetAttrs, result.Attributes)
+	if !reflect.DeepEqual(targetAttrs, result.Attributes) {
+		t.Fatal()
+	}
+	if !reflect.DeepEqual(targetColumns, result.Columns) {
+		t.Fatal()
+	}
+}
+
+func TestNewQueryResponseFromInternal(t *testing.T) {
+	targetAttrs := map[string]interface{}{
+		"name":       "some string",
+		"age":        int64(95),
+		"registered": true,
+		"height":     1.83,
+	}
+	targetColumns := []uint64{5, 10}
+	targetCountItems := []CountResultItem{
+		{ID: 10, Count: 100},
+	}
+	attrs := []*pb.Attr{
+		{Key: "name", StringValue: "some string", Type: 1},
+		{Key: "age", IntValue: 95, Type: 2},
+		{Key: "registered", BoolValue: true, Type: 3},
+		{Key: "height", FloatValue: 1.83, Type: 4},
+	}
+	row := &pb.Row{
+		Attrs:   attrs,
+		Columns: []uint64{5, 10},
+	}
+	pairs := []*pb.Pair{
+		{ID: 10, Count: 100},
+	}
+	response := &pb.QueryResponse{
+		Results: []*pb.QueryResult{
+			{Type: QueryResultTypeRow, Row: row},
+			{Type: QueryResultTypePairs, Pairs: pairs},
+		},
+		Err: "",
+	}
+	qr, err := newQueryResponseFromInternal(response)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	if qr.ErrorMessage != "" {
+		t.Fatalf("ErrorMessage should be empty")
+	}
+	if !qr.Success {
+		t.Fatalf("IsSuccess should be true")
+	}
+
+	results := qr.Results()
+	if len(results) != 2 {
+		t.Fatalf("Number of results should be 2")
+	}
+	if results[0] != qr.Result() {
+		t.Fatalf("Result() should return the first result")
+	}
+	if !reflect.DeepEqual(targetAttrs, results[0].Row().Attributes) {
+		t.Fatalf("The row result should contain the attributes")
+	}
+	if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) {
+		t.Fatalf("The row result should contain the columns")
+	}
+	if !reflect.DeepEqual(targetCountItems, results[1].CountItems()) {
+		t.Fatalf("The response should include count items")
+	}
+}
+
+func TestNewQueryResponseWithErrorFromInternal(t *testing.T) {
+	response := &pb.QueryResponse{
+		Err: "some error",
+	}
+	qr, err := newQueryResponseFromInternal(response)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	if qr.ErrorMessage != "some error" {
+		t.Fatalf("The response should include the error message")
+	}
+	if qr.Success {
+		t.Fatalf("IsSuccess should be false")
+	}
+	if qr.Result() != nil {
+		t.Fatalf("If there are no results, Result should return nil")
+	}
+}
+
+func TestNewQueryResponseFromInternalFailure(t *testing.T) {
+	attrs := []*pb.Attr{
+		{Key: "name", StringValue: "some string", Type: 99},
+	}
+	row := &pb.Row{
+		Attrs: attrs,
+	}
+	response := &pb.QueryResponse{
+		Results: []*pb.QueryResult{{Type: QueryResultTypeRow, Row: row}},
+	}
+	qr, err := newQueryResponseFromInternal(response)
+	if qr != nil && err == nil {
+		t.Fatalf("Should have failed")
+	}
+	response = &pb.QueryResponse{
+		ColumnAttrSets: []*pb.ColumnAttrSet{{ID: 1, Attrs: attrs}},
+	}
+	qr, err = newQueryResponseFromInternal(response)
+	if qr != nil && err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestCountResultItemToString(t *testing.T) {
+	tests := []struct {
+		item     *CountResultItem
+		expected string
+	}{
+		{item: &CountResultItem{ID: 100, Count: 50}, expected: "100:50"},
+		{item: &CountResultItem{Key: "blah", Count: 50}, expected: "blah:50"},
+		{item: &CountResultItem{Key: "blah", ID: 22, Count: 50}, expected: "blah:50"},
+		{item: &CountResultItem{Key: "blah", ID: 22}, expected: "blah:0"},
+		{item: &CountResultItem{}, expected: "0:0"},
+	}
+
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d: ", i), func(t *testing.T) {
+			if tst.expected != tst.item.String() {
+				t.Fatalf("%s != %s", tst.expected, tst.item.String())
+			}
+		})
+	}
+}
+
+func TestMarshalResults(t *testing.T) {
+	attrs := []*pb.Attr{
+		{Key: "name", StringValue: "some string", Type: 1},
+		{Key: "age", IntValue: 95, Type: 2},
+		{Key: "registered", BoolValue: true, Type: 3},
+		{Key: "height", FloatValue: 1.83, Type: 4},
+	}
+	row := &pb.Row{
+		Attrs:   attrs,
+		Columns: []uint64{5, 10},
+	}
+	pairs := []*pb.Pair{
+		{ID: 10, Count: 100},
+	}
+	pbufResults := []*pb.QueryResult{
+		{Type: QueryResultTypeRow, Row: row},
+		{Type: QueryResultTypePairs, Pairs: pairs},
+	}
+	resultJSONStrings := make([]string, len(pbufResults))
+	for i, pr := range pbufResults {
+		r, err := newQueryResultFromInternal(pr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		b, err := json.Marshal(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resultJSONStrings[i] = string(b)
+	}
+	targetJSON := []string{
+		`{"attrs":{"age":95,"height":1.83,"name":"some string","registered":true},"columns":[5,10],"keys":[]}`,
+		`[{"id":10,"count":100}]`,
+	}
+	for i := range targetJSON {
+		if sortedString(targetJSON[i]) != sortedString(resultJSONStrings[i]) {
+			t.Fatalf("%v != %v ", targetJSON[i], resultJSONStrings[i])
+		}
+	}
+
+}
+
+func TestUnknownQueryResultType(t *testing.T) {
+	result := &pb.QueryResult{
+		Type: 999,
+	}
+	_, err := newQueryResultFromInternal(result)
+	if err != ErrUnknownType {
+		t.Fatalf("Should have failed with ErrUnknownType")
+	}
+}
+
+func TestTopNResult(t *testing.T) {
+	result := TopNResult{
+		CountResultItem{ID: 100, Count: 10},
+	}
+	expectResult(t, result, QueryResultTypePairsField, RowResult{}, []CountResultItem{{100, "", 10}}, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestRowResult(t *testing.T) {
+	result := RowResult{
+		Columns: []uint64{1, 2, 3},
+	}
+	targetBmp := RowResult{
+		Columns: []uint64{1, 2, 3},
+	}
+	expectResult(t, result, QueryResultTypeRow, targetBmp, nil, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestRowResultNilColumns(t *testing.T) {
+	result := RowResult{
+		Columns: nil,
+	}
+	_, err := result.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSumCountResult(t *testing.T) {
+	result := ValCountResult{
+		Val: 100,
+		Cnt: 50,
+	}
+	expectResult(t, result, QueryResultTypeValCount, RowResult{}, nil, 100, 50, false, nil, RowIdentifiersResult{})
+}
+
+func TestIntResult(t *testing.T) {
+	result := IntResult(11)
+	expectResult(t, result, QueryResultTypeUint64, RowResult{}, nil, 0, 11, false, nil, RowIdentifiersResult{})
+}
+
+func TestBoolResult(t *testing.T) {
+	result := BoolResult(true)
+	expectResult(t, result, QueryResultTypeBool, RowResult{}, nil, 0, 0, true, nil, RowIdentifiersResult{})
+}
+
+func TestNilResult(t *testing.T) {
+	result := NilResult{}
+	expectResult(t, result, QueryResultTypeNil, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestGroupCountResult(t *testing.T) {
+	result := GroupCountResult{
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
+	}
+	expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
+	}, RowIdentifiersResult{})
+}
+
+func TestGroupCountWithValueResult(t *testing.T) {
+	var a, b int64 = -1, 1
+
+	result := GroupCountResult{
+		{Groups: []FieldRow{{FieldName: "f1", Value: &a}}, Count: 1},
+		{Groups: []FieldRow{{FieldName: "f1", Value: &b}}, Count: 1},
+	}
+
+	var aa, bb int64 = -1, 1
+	expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
+		{Groups: []FieldRow{{FieldName: "f1", Value: &aa}}, Count: 1},
+		{Groups: []FieldRow{{FieldName: "f1", Value: &bb}}, Count: 1},
+	}, RowIdentifiersResult{})
+}
+
+func TestRowIdentifiersResult(t *testing.T) {
+	result := RowIdentifiersResult{
+		IDs: []uint64{1, 2, 3, 4},
+	}
+	expectResult(t, result, QueryResultTypeRowIdentifiers, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{
+		IDs: []uint64{1, 2, 3, 4},
+	})
+}
+
+func expectResult(t *testing.T, r QueryResult, resultType uint32, bmp RowResult,
+	countItems []CountResultItem, sum int64, count int64, changed bool,
+	groupCounts []GroupCount, rowIdentifiers RowIdentifiersResult) {
+	if resultType != r.Type() {
+		log.Fatalf("Result type: %d != %d", resultType, r.Type())
+	}
+	if !reflect.DeepEqual(bmp, r.Row()) {
+		log.Fatalf("Row: %v != %v", bmp, r.Row())
+	}
+	if !reflect.DeepEqual(countItems, r.CountItems()) {
+		log.Fatalf("Count items: %v != %v", countItems, r.CountItems())
+	}
+	if count != r.Count() {
+		log.Fatalf("Count: %d != %d", count, r.Count())
+	}
+	if sum != r.Value() {
+		log.Fatalf("Sum: %d != %d", sum, r.Value())
+	}
+	if changed != r.Changed() {
+		log.Fatalf("Changed: %v != %v", changed, r.Changed())
+	}
+	if !reflect.DeepEqual(groupCounts, r.GroupCounts()) {
+		log.Fatalf("Group counts: %v != %v", groupCounts, r.GroupCounts())
+	}
+	if !reflect.DeepEqual(rowIdentifiers, r.RowIdentifiers()) {
+		log.Fatalf("Row identifiers: %v != %v", rowIdentifiers, r.RowIdentifiers())
+	}
+}

client/shardnodes.go

@@ -0,0 +1,66 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"sync"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+)
+
+type shardNodes struct {
+	data map[string]map[uint64][]*pnet.URI
+	mu   *sync.RWMutex
+}
+
+func newShardNodes() shardNodes {
+	return shardNodes{
+		data: make(map[string]map[uint64][]*pnet.URI),
+		mu:   &sync.RWMutex{},
+	}
+}
+
+func (s shardNodes) Get(index string, shard uint64) ([]*pnet.URI, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if idx, ok := s.data[index]; ok {
+		if uris, ok := idx[shard]; ok {
+			return uris, true
+		}
+	}
+	return nil, false
+}
+
+func (s shardNodes) Put(index string, shard uint64, uris []*pnet.URI) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.data[index]
+	if !ok {
+		idx = make(map[uint64][]*pnet.URI)
+	}
+	idx[shard] = uris
+	s.data[index] = idx
+}
+
+func (s shardNodes) Invalidate() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for k := range s.data {
+		delete(s.data, k)
+	}
+}

client/tracer.go

@@ -0,0 +1,89 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/log"
+)
+
+type NoopTracer struct{}
+
+type NoopSpan struct{}
+
+func (s NoopSpan) Finish() {
+	// pass
+}
+func (s NoopSpan) FinishWithOptions(opts opentracing.FinishOptions) {
+	// pass
+}
+
+func (s NoopSpan) Context() opentracing.SpanContext {
+	return nil
+}
+func (s NoopSpan) SetOperationName(operationName string) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) SetTag(key string, value interface{}) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) LogFields(fields ...log.Field) {
+	// pass
+}
+
+func (s NoopSpan) LogKV(alternatingKeyValues ...interface{}) {
+	// pass
+}
+
+func (s NoopSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) BaggageItem(restrictedKey string) string {
+	return ""
+}
+
+func (s NoopSpan) Tracer() opentracing.Tracer {
+	return nil
+}
+
+func (s NoopSpan) LogEvent(event string) {
+	// pass
+}
+
+func (s NoopSpan) LogEventWithPayload(event string, payload interface{}) {
+	// pass
+}
+
+func (s NoopSpan) Log(data opentracing.LogData) {
+	// pass
+}
+
+func (t NoopTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
+	return NoopSpan{}
+}
+
+func (t NoopTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
+	return nil
+}
+
+func (t NoopTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
+	return nil, nil
+}

client/validate.go

@@ -0,0 +1,54 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"regexp"
+)
+
+const (
+	maxLabel = 64
+	maxKey   = 64
+)
+
+var labelRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_-]*$")
+var keyRegex = regexp.MustCompile("^[A-Za-z0-9_{}+/=.~%:-]*$")
+
+// ValidLabel returns true if the given label is valid, otherwise false.
+func ValidLabel(label string) bool {
+	return len(label) <= maxLabel && labelRegex.Match([]byte(label))
+}
+
+// ValidKey returns true if the given key is valid, otherwise false.
+func ValidKey(key string) bool {
+	return len(key) <= maxKey && keyRegex.Match([]byte(key))
+}
+
+func validateLabel(label string) error {
+	if ValidLabel(label) {
+		return nil
+	}
+	return ErrInvalidLabel
+}
+
+func validateKey(key string) error {
+	if ValidKey(key) {
+		return nil
+	}
+	return ErrInvalidKey
+}

client/validate_test.go

@@ -0,0 +1,73 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import "testing"
+
+func TestValidateLabel(t *testing.T) {
+	labels := []string{
+		"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, label := range labels {
+		if validateLabel(label) != nil {
+			t.Fatalf("Should be valid label: %s", label)
+		}
+	}
+}
+
+func TestValidateLabelInvalid(t *testing.T) {
+	labels := []string{
+		"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, label := range labels {
+		if validateLabel(label) == nil {
+			t.Fatalf("Should be invalid label: %s", label)
+		}
+	}
+}
+
+func TestValidateKey(t *testing.T) {
+	keys := []string{
+		"", "1", "ab", "ab1", "b-c", "d_e", "pilosa.com",
+		"bbf8d41c-7dba-40c4-94dc-94677b43bcf3",                 // UUID
+		"{bbf8d41c-7dba-40c4-94dc-94677b43bcf3}",               // Windows GUID
+		"https%3A//www.pilosa.com/about/%23contact",            // escaped URL
+		"aHR0cHM6Ly93d3cucGlsb3NhLmNvbS9hYm91dC8jY29udGFjdA==", // base64
+		"urn:isbn:1234567",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, key := range keys {
+		if validateKey(key) != nil {
+			t.Fatalf("Should be valid key: %s", key)
+		}
+	}
+}
+
+func TestValidateKeyInvalid(t *testing.T) {
+	keys := []string{
+		"\"", "'", "slice\\dice", "valid?no", "yüce", "*xyz", "with space", "<script>",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, key := range keys {
+		if validateKey(key) == nil {
+			t.Fatalf("Should be invalid key: %s", key)
+		}
+	}
+}

client/version.go

@@ -0,0 +1,21 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+// Version is the client version.
+const Version = "v1.3.0"

cluster_internal_test.go

@@ -15,42 +15,81 @@
 package pilosa
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"os"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/gorilla/mux"
+	pnet "github.com/pilosa/pilosa/v2/net"
-	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
-	"github.com/pkg/errors"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pilosa/pilosa/v2/topology"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
 )
 
+// GlobalPortMap avoids many races and port conflicts when setting
+// up ports for test clusters. Used for tests only.
+var globalPortMap *GlobalPortMapper
+
+func init() {
+	globalPortMap = NewGlobalPortMapper(300)
+}
+
+// GlobalPortMapper maintains a pool of available ports by
+// holding them open until GetPort() is called.
+type GlobalPortMapper struct {
+	availPorts map[int]net.Listener
+}
+
+// reserve n ports
+func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) {
+
+	pm = &GlobalPortMapper{
+		availPorts: make(map[int]net.Listener),
+	}
+	for i := 0; i < n; i++ {
+		lsn, _ := net.Listen("tcp", ":0")
+		r := lsn.Addr()
+		port := r.(*net.TCPAddr).Port
+		pm.availPorts[port] = lsn
+	}
+	return
+}
+
+func (pm *GlobalPortMapper) GetPort() (port int, err error) {
+	for port, lsn := range pm.availPorts {
+		lsn.Close()
+		return port, nil
+	}
+	return -1, fmt.Errorf("no more ports available")
+}
+
+func (pm *GlobalPortMapper) MustGetPort() int {
+	port, err := pm.GetPort()
+	if err != nil {
+		panic(err)
+	}
+	return port
+}
+
 // Ensure that fragCombos creates the correct fragment mapping.
 func TestFragCombos(t *testing.T) {
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
 
 	c := newCluster()
 	c.addNodeBasicSorted(node0)
@@ -90,13 +129,34 @@ func TestFragCombos(t *testing.T) {
 	}
 }
 
-// newIndexWithTempPath returns a new instance of Index.
+// newHolderWithTempPath returns a new instance of Holder.
-func newIndexWithTempPath(name string) *Index {
+func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	cfg := mustHolderConfig()
+	cfg.StorageConfig.Backend = backend
+	h := NewHolder(path, cfg)
+	PanicOn(h.Open())
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
+	return h
+}
+
+// newIndexWithTempPath returns a new instance of Index.
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
+	if err != nil {
+		panic(err)
+	}
+	h := NewHolder(path, nil)
+	PanicOn(h.Open())
+	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -105,28 +165,27 @@ func newIndexWithTempPath(name string) *Index {
 
 // Ensure that fragSources creates the correct fragment mapping.
 func TestFragSources(t *testing.T) {
-
+	uri0, err := pnet.NewURIFromAddress("host0")
-	uri0, err := NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri3, err := NewURIFromAddress("host3")
+	uri3, err := pnet.NewURIFromAddress("host3")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
-	node3 := &Node{ID: "node3", URI: *uri3}
+	node3 := &topology.Node{ID: "node3", URI: *uri3}
 
 	c1 := newCluster()
 	c1.ReplicaN = 1
@@ -157,27 +216,53 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 101, nil)
+
+	// Obtain transaction.
+	var shard uint64
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 1
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*2+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 2
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*3+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 3
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	PanicOn(tx.Commit())
 
 	tests := []struct {
 		from     *cluster
@@ -194,8 +279,8 @@ func TestFragSources(t *testing.T) {
 				"node0": {},
 				"node1": {},
 				"node2": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -206,11 +291,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(1)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -221,11 +306,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(3)},
 				},
 				"node2": {},
 			},
@@ -274,37 +359,37 @@ func TestFragSources(t *testing.T) {
 // Ensure that fragSources creates the correct fragment mapping.
 func TestResizeJob(t *testing.T) {
 
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
 
 	tests := []struct {
-		existingNodes []*Node
+		existingNodes []*topology.Node
-		node          *Node
+		node          *topology.Node
 		action        string
 		expectedIDs   map[string]bool
 	}{
 		{
-			existingNodes: []*Node{node0, node1},
+			existingNodes: []*topology.Node{node0, node1},
 			node:          node2,
 			action:        resizeJobActionAdd,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
 		},
 		{
-			existingNodes: []*Node{node0, node1, node2},
+			existingNodes: []*topology.Node{node0, node1, node2},
 			node:          node2,
 			action:        resizeJobActionRemove,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false},
@@ -325,22 +410,27 @@ func TestResizeJob(t *testing.T) {
 // Ensure the cluster can fairly distribute partitions across the nodes.
 func TestCluster_Owners(t *testing.T) {
 	c := cluster{
-		nodes: []*Node{
+		noder: topology.NewLocalNoder([]*topology.Node{
 			{URI: NewTestURIFromHostPort("serverA", 1000)},
 			{URI: NewTestURIFromHostPort("serverB", 1000)},
 			{URI: NewTestURIFromHostPort("serverC", 1000)},
-		},
+		}),
 		Hasher:   NewTestModHasher(),
 		ReplicaN: 2,
 	}
 
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
 	// Verify nodes are distributed.
-	if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) {
+	if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*topology.Node{cNodes[0], cNodes[1]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 
 	// Verify nodes go around the ring.
-	if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) {
+	if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*topology.Node{cNodes[2], cNodes[0]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 }
@@ -351,7 +441,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := topology.ShardToShardPartition(index, shard, partitionN)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -381,7 +471,7 @@ func TestHasher(t *testing.T) {
 		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
 	} {
 		for i, v := range tt.bucket {
-			hasher := &jmphasher{}
+			hasher := &topology.Jmphasher{}
 			if got := hasher.Hash(tt.key, i+1); got != v {
 				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
 			}
@@ -391,9 +481,14 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
-	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
+	shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
 
 	if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
 		t.Fatalf("unexpected shars for node's index: %v", shards)
@@ -401,20 +496,22 @@ func TestCluster_ContainsShards(t *testing.T) {
 }
 
 func TestCluster_Nodes(t *testing.T) {
-	uri0 := NewTestURIFromHostPort("node0", 0)
+	const urisCount = 4
-	uri1 := NewTestURIFromHostPort("node1", 0)
+	var uris []pnet.URI
-	uri2 := NewTestURIFromHostPort("node2", 0)
+	arbitraryPorts := []int{17384, 17385, 17386, 17387}
-	uri3 := NewTestURIFromHostPort("node3", 0)
+	for i := 0; i < urisCount; i++ {
+		uris = append(uris, NewTestURIFromHostPort(fmt.Sprintf("node%d", i), uint16(arbitraryPorts[i])))
+	}
 
-	node0 := &Node{ID: "node0", URI: uri0}
+	node0 := &topology.Node{ID: "node0", URI: uris[0]}
-	node1 := &Node{ID: "node1", URI: uri1}
+	node1 := &topology.Node{ID: "node1", URI: uris[1]}
-	node2 := &Node{ID: "node2", URI: uri2}
+	node2 := &topology.Node{ID: "node2", URI: uris[2]}
-	node3 := &Node{ID: "node3", URI: uri3}
+	node3 := &topology.Node{ID: "node3", URI: uris[3]}
 
-	nodes := []*Node{node0, node1, node2}
+	nodes := []*topology.Node{node0, node1, node2}
 
 	t.Run("NodeIDs", func(t *testing.T) {
-		actual := Nodes(nodes).IDs()
+		actual := topology.Nodes(nodes).IDs()
 		expected := []string{node0.ID, node1.ID, node2.ID}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
@@ -422,24 +519,24 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Filter", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
+		actual := topology.Nodes(topology.Nodes(nodes).Filter(nodes[1])).URIs()
-		expected := []URI{uri0, uri2}
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("FilterURI", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
+		actual := topology.Nodes(topology.Nodes(nodes).FilterURI(uris[1])).URIs()
-		expected := []URI{uri0, uri2}
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("Contains", func(t *testing.T) {
-		actualTrue := Nodes(nodes).Contains(node1)
+		actualTrue := topology.Nodes(nodes).Contains(node1)
-		actualFalse := Nodes(nodes).Contains(node3)
+		actualFalse := topology.Nodes(nodes).Contains(node3)
 		if !reflect.DeepEqual(actualTrue, true) {
 			t.Errorf("expected: %v, but got: %v", true, actualTrue)
 		}
@@ -449,9 +546,9 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Clone", func(t *testing.T) {
-		clone := Nodes(nodes).Clone()
+		clone := topology.Nodes(nodes).Clone()
-		actual := Nodes(clone).URIs()
+		actual := topology.Nodes(clone).URIs()
-		expected := []URI{uri0, uri1, uri2}
+		expected := []pnet.URI{uris[0], uris[1], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
@@ -459,9 +556,9 @@ func TestCluster_Nodes(t *testing.T) {
 }
 
 func TestCluster_PreviousNode(t *testing.T) {
-	node0 := &Node{ID: "node0"}
+	node0 := &topology.Node{ID: "node0"}
-	node1 := &Node{ID: "node1"}
+	node1 := &topology.Node{ID: "node1"}
-	node2 := &Node{ID: "node2"}
+	node2 := &topology.Node{ID: "node2"}
 
 	t.Run("OneNode", func(t *testing.T) {
 		c := newCluster()
@@ -512,332 +609,6 @@ func TestCluster_PreviousNode(t *testing.T) {
 	})
 }
 
-// NEXT: move this test to internal and unexport IsCoordinator
-func TestCluster_Coordinator(t *testing.T) {
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-
-	c1 := *newCluster()
-	c1.Node = node1
-	c1.Coordinator = node1.ID
-	c2 := *newCluster()
-	c2.Node = node2
-	c2.Coordinator = node1.ID
-
-	t.Run("IsCoordinator", func(t *testing.T) {
-		if !c1.isCoordinator() {
-			t.Errorf("!IsCoordinator error: %v", c1.Node)
-		} else if c2.isCoordinator() {
-			t.Errorf("IsCoordinator error: %v", c2.Node)
-		}
-	})
-}
-
-func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
-
-	uri0 := NewTestURIFromHostPort("host0", 0)
-	uri1 := NewTestURIFromHostPort("host1", 0)
-	uri2 := NewTestURIFromHostPort("host2", 0)
-	invalid := NewTestURIFromHostPort("invalid", 0)
-
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
-
-	t.Run("AddNode", func(t *testing.T) {
-		err := c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// add the same host.
-		err = c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = c1.addNode(node2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		actual := c1.nodeIDs()
-		expected := []string{node0.ID, node1.ID, node2.ID}
-
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("ContainsID", func(t *testing.T) {
-		if !c1.Topology.ContainsID(node1.ID) {
-			t.Errorf("!ContainsHost error: %v", node1.ID)
-		} else if c1.Topology.ContainsID(nodeinvalid.ID) {
-			t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
-		}
-	})
-}
-
-// Ensure that general cluster functionality works as expected.
-func TestCluster_ResizeStates(t *testing.T) {
-
-	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		node := tc.Clusters[0]
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"some-other-host"},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		expected := "coordinator node0 is not in topology: [some-other-host]"
-		err := tc.Open()
-		if err == nil || errors.Cause(err).Error() != expected {
-			t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node0 := tc.Clusters[0]
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"node0", "node2"},
-		}
-		if err := tc.WriteTopology(node0.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		// Ensure that node is in state STARTING before the other node joins.
-		if node0.State() != ClusterStateStarting {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
-		}
-
-		// Expect an error by adding a node not in the topology.
-		expectedError := "host is not in topology: node1"
-		if err := tc.addNode(); err == nil || err.Error() != expectedError {
-			t.Errorf("did not receive expected error: %s", expectedError)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node2 := tc.Clusters[2]
-
-		// Ensure that node comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node2.State() != ClusterStateNormal {
-			t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Add Bit Data to node0.
-		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-
-		// Before starting the resize, get the CheckSum to use for
-		// comparison later.
-		node0Field := node0.holder.Field("i", "f")
-		node0View := node0Field.view("standard")
-		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
-
-		// addNode needs to block until the resize process has completed.
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes come up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Bits
-		// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
-		node1Field := node1.holder.Field("i", "f")
-		node1View := node1Field.view("standard")
-		node1Fragment := node1View.Fragment(1)
-
-		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
-			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-}
-
 func TestAE(t *testing.T) {
 	t.Run("AbortDoesn'tBlockUninitialized", func(t *testing.T) {
 		c := newCluster()
@@ -846,6 +617,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -857,11 +629,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@@ -893,97 +667,4 @@ func TestAE(t *testing.T) {
 			t.Fatalf("abort should not have blocked this long")
 		}
 	})
-
-}
-
-// Ensures that coordinator can be changed.
-func TestCluster_UpdateCoordinator(t *testing.T) {
-	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
-
-		oldNode := c.nodes[0]
-		newNode := c.nodes[1]
-
-		// Update coordinator to the same value.
-		if c.updateCoordinator(oldNode) {
-			t.Errorf("did not expect coordinator to change")
-		} else if c.Coordinator != oldNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
-		}
-
-		// Update coordinator to a new value.
-		if !c.updateCoordinator(newNode) {
-			t.Errorf("expected coordinator to change")
-		} else if c.Coordinator != newNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
-		}
-	})
-}
-
-func TestCluster_confirmNodeDownUp(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be up")
-	}
-
-}
-func TestCluster_confirmNodeDownTimeout(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
-}
-
-func TestCluster_confirmNodeDownDown(t *testing.T) {
-	uri := URI{}
-	uri.Scheme = "http"
-	uri.Host = "DoesntMatter"
-	uri.Port = 6666
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
 }

cmd/badloader/badloader.go

@@ -0,0 +1,152 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+
+	//"fmt"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	pnet "github.com/pilosa/pilosa/v2/net"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+
+	"os"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *http.InternalClient) error {
+	t0 := time.Now()
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				PanicOn("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				PanicOn(err)
+			}
+			return nil
+		}
+		n++
+		if n%500 == 0 {
+			VV("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		//vv("parts = '%#v'", parts)
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			PanicOn(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+// badloader reproduce a union in place issue for us. slurp is
+// the new "good" loader, and should always be preferred now
+// when not trying to repro that bug. pulled from 85fa67e8
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	PanicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	PanicOn(UploadTar(tarSrcPath, c))
+	VV("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pnet.URI
+
+func init() {
+	var err error
+	globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
+	PanicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
+	return globURI
+}

cmd/convert.go

@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
 			return inspector.Run(context.Background())
 		},
 	}
+	flags := inspectCmd.Flags()
+	flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+	flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+	flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+	flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+	flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+	flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
 	return inspectCmd
 }

cmd/export.go

@@ -50,7 +50,7 @@ The file does not contain any headers.
 	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
 	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
-	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
 
 	return exportCmd
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
@@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
-	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
 
 	return importCmd
 }

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
+					Max:       pql.NewDecimal(100, 0),
-					Min:       -10,
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/pilosa-bench/main.go

@@ -0,0 +1,358 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"expvar"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	phttp "github.com/pilosa/pilosa/v2/http"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	requestCountVar          = expvar.NewInt("request_count")
+	requestCurrentLatencyVar = expvar.NewFloat("request_current_latency") // seconds
+	requestAvgLatencyVar     = expvar.NewFloat("request_avg_latency")     // seconds
+	requestTotalLatencyVar   = expvar.NewFloat("request_total_latency")   // seconds
+	requestPerSecVar         = expvar.NewFloat("request_per_sec")
+)
+
+func main() {
+	if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) (err error) {
+	fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
+	hostport := fs.String("hostport", "localhost:10101", "")
+	typ := fs.String("type", "row", "query type (row)")
+	n := fs.Int("n", 1000, "number of queries")
+	rate := fs.Int("rate", 1, "number of queries per second")
+	verbose := fs.Bool("v", false, "verbose logging")
+	from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
+	to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse from/to time.
+	var opt queryOptions
+	if *from != "" {
+		if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
+			return fmt.Errorf("cannot parse -from time")
+		}
+	}
+	if *to != "" {
+		if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
+			return fmt.Errorf("cannot parse -to time")
+		}
+	}
+
+	if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
+		return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
+	}
+
+	// Clear time prefix on log.
+	log.SetFlags(0)
+	if !*verbose {
+		log.SetOutput(ioutil.Discard)
+	}
+
+	// Setup PRNG to have consistent values for the same set of data.
+	rand.Seed(0)
+
+	// Setup connection to pilosa.
+	client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
+	if err != nil {
+		return err
+	}
+
+	// Set up HTTP endpoint to provide /debug endpoints.
+	fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
+	go func() { _ = http.ListenAndServe(":7070", nil) }()
+
+	// Run separate goroutine to calculate the current req/sec & latency.
+	go monitor()
+
+	// Load all id/keys for each field.
+	log.Printf("loading field identifiers")
+	fieldIDMap, err := loadFields(ctx, client)
+	if err != nil {
+		return fmt.Errorf("cannot load field identifiers: %w", err)
+	} else if len(fieldIDMap) == 0 {
+		return fmt.Errorf("no field identifiers available, please verify data exists")
+	}
+
+	// Generate list of sorted keys.
+	fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
+	for k, f := range fieldIDMap {
+		switch *typ {
+		case "row-bsi":
+			if f.info.Options.Type != "int" {
+				continue
+			}
+		case "row-range", "topk":
+			if f.info.Options.Type != "time" {
+				continue
+			}
+		default:
+			if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
+				continue
+			}
+		}
+
+		fieldKeys = append(fieldKeys, k)
+	}
+	sort.Slice(fieldKeys, func(i, j int) bool {
+		return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
+	})
+
+	// Ensure we have appropriate fields for our query type.
+	if len(fieldKeys) == 0 {
+		return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
+	}
+
+	log.Printf("issuing %d queries at %d query/sec", *n, *rate)
+
+	// Repeatedly issue queries based on available row data.
+	ticker := time.NewTicker(time.Second / time.Duration(*rate))
+	var g errgroup.Group
+	for i := 0; i < *n; i++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		key := fieldKeys[rand.Intn(len(fieldKeys))]
+		q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
+		if err != nil {
+			return fmt.Errorf("cannot generate query: %w", err)
+		}
+
+		log.Printf("[query] %s", q)
+
+		g.Go(func() error {
+			t := time.Now()
+			_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
+			if err != nil {
+				return err
+			}
+			elapsed := time.Since(t).Seconds()
+			requestCountVar.Add(1)
+			requestTotalLatencyVar.Add(elapsed)
+			requestAvgLatencyVar.Set(requestTotalLatencyVar.Value() / float64(requestCountVar.Value()))
+			return nil
+		})
+	}
+
+	return g.Wait()
+}
+
+// monitor runs in a separate goroutine and updates metrics.
+func monitor() {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	var lastTime time.Time
+	var lastN int64
+	var lastLatency float64
+	for range ticker.C {
+		now, n := time.Now(), requestCountVar.Value()
+		latency := requestTotalLatencyVar.Value()
+
+		if !lastTime.IsZero() {
+			elapsed := lastTime.Sub(now).Seconds()
+			if n > 0 {
+				requestCurrentLatencyVar.Set((lastLatency - latency) / float64(n))
+			}
+			requestPerSecVar.Set(float64(lastN-n) / elapsed)
+		}
+		lastTime, lastN, lastLatency = now, n, latency
+	}
+}
+
+func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
+	switch typ {
+	case "row":
+		return generateRowQuery(index, field, identifiers), nil
+	case "row-bsi":
+		return generateRowBSIQuery(index, field), nil
+	case "row-range":
+		return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
+	case "count":
+		return generateCountQuery(index, field, identifiers), nil
+	case "intersect":
+		return generateIntersectQuery(index, field, identifiers), nil
+	case "union":
+		return generateUnionQuery(index, field, identifiers), nil
+	case "difference":
+		return generateDifferenceQuery(index, field, identifiers), nil
+	case "xor":
+		return generateXorQuery(index, field, identifiers), nil
+	case "groupby":
+		return generateGroupByQuery(index, field), nil
+	case "topk":
+		return generateTopKQuery(index, field, opt.from, opt.to), nil
+	default:
+		return "", fmt.Errorf("invalid query type: %q", typ)
+	}
+}
+
+func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
+	}
+	return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
+}
+
+func generateRowBSIQuery(index, field string) string {
+	return fmt.Sprintf("Row(%s > 0)", field)
+}
+
+func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+	}
+	return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	a := make([]string, rand.Intn(9)+1)
+	for i := range a {
+		a[i] = generateRowQuery(index, field, identifiers)
+	}
+	return strings.Join(a, ", ")
+}
+
+func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
+}
+
+func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateGroupByQuery(index, field string) string {
+	return fmt.Sprintf("GroupBy(Rows(%s))", field)
+}
+
+func generateTopKQuery(index, field string, from, to time.Time) string {
+	return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+// loadFields returns a mapping of index/field names to field info & identifiers.
+func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
+	indexes, err := client.Schema(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[fieldKey]*fieldInfo)
+	for _, ii := range indexes {
+		for _, f := range ii.Fields {
+			log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
+
+			switch f.Options.Type {
+			case "set", "mutex", "time":
+				identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
+				if err != nil {
+					return nil, fmt.Errorf("fetch fields: %w", err)
+				} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
+					m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
+				}
+
+			case "int":
+				m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
+			}
+		}
+	}
+	return m, nil
+}
+
+// fetchFieldIDs returns a list of field IDs or keys.
+func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
+	resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
+	if err != nil {
+		return nil, err
+	}
+
+	switch result := resp.Results[0].(type) {
+	case *pilosa.RowIdentifiers:
+		return result, nil
+	case pilosa.RowIdentifiers:
+		return &result, nil
+	default:
+		return nil, fmt.Errorf("unexpected result type: %T", result)
+	}
+}
+
+func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
+	return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
+}
+
+func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
+	return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
+}
+
+type fieldKey struct {
+	index string
+	field string
+}
+
+type fieldInfo struct {
+	info        *pilosa.FieldInfo
+	identifiers *pilosa.RowIdentifiers
+}
+
+func compareFieldKeys(x, y fieldKey) int {
+	if cmp := strings.Compare(x.index, y.index); cmp != 0 {
+		return cmp
+	}
+	return strings.Compare(x.field, y.field)
+}
+
+type queryOptions struct {
+	from, to time.Time
+}

cmd/random-query/main.go

@@ -0,0 +1,557 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math"
+	"math/rand"
+	nethttp "net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+// RandomQueryConfig
+type RandomQueryConfig struct {
+
+	// user facing flags
+	HostPort    string    // -hostport
+	TreeDepth   int       // -d
+	QueryCount  int       // -n
+	Verbose     bool      // -v
+	VeryVerbose bool      // -V
+	TimeFromArg string    // --time.from
+	TimeToArg   string    // --time.to
+	TimeFrom    time.Time // parsed time
+	TimeTo      time.Time // parsed time
+	TimeRange   int64     // hours between parsed times
+
+	IndexMap map[string]*Features
+
+	API  *pilosa.API
+	Info []*pilosa.IndexInfo
+
+	BitmapFunc []string
+
+	Rnd *rand.Rand
+}
+
+type API interface {
+
+	// InternalClient
+	Schema(ctx context.Context) ([]*pilosa.IndexInfo, error)
+	Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error)
+
+	// API for contrast; just a little different:
+	//Schema(ctx context.Context) []*IndexInfo
+	//Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error)
+}
+
+// have to wrap because the ugly little differences between InternalClient and API
+type wrapper struct {
+	api *pilosa.API
+}
+
+func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	return w.api.Schema(ctx, false)
+}
+
+func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	r, err := w.api.Query(ctx, queryRequest)
+	return &r, err
+}
+
+func wrapApiToInternalClient(api *pilosa.API) *wrapper {
+	return &wrapper{api: api}
+}
+
+// These times are copied from the "kitchen sink" data generator to serve as defaults.
+var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
+var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
+
+// call DefineFlags before myflags.Parse()
+func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
+	fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
+	fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
+	fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
+	fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results")
+	fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+	fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *RandomQueryConfig) ValidateConfig() error {
+	if c.TreeDepth < 1 {
+		return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth)
+	}
+	if c.QueryCount < 0 {
+		return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
+	}
+	var err error
+	c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg)
+	if err != nil {
+		return fmt.Errorf("-time.from value couldn't be parsed: %w", err)
+	}
+	c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg)
+	if err != nil {
+		return fmt.Errorf("-time.to value couldn't be parsed: %w", err)
+	}
+	c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours())
+	if c.TimeRange < 1 {
+		return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
+			c.TimeToArg, c.TimeFromArg)
+	}
+	return nil
+}
+
+var ProgramName = "random-query"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
+	cfg := NewRandomQueryConfig()
+	cfg.DefineFlags(myflags)
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+
+	err = cfg.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *RandomQueryConfig) Run() (err error) {
+	remoteClient := nethttp.DefaultClient
+	cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	totalQ := 0
+	loops := 0
+	t0 := time.Now()
+
+	report := func() {
+		dur := time.Since(t0)
+		if dur > 0 {
+			qps := 1e9 * float64(totalQ) / float64(dur)
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: %0.02f", totalQ, dur, qps)
+		} else {
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: N/A", totalQ, dur)
+		}
+	}
+	defer report()
+
+NewSetup:
+	err = cfg.Setup(cli)
+	if err != nil {
+		return err
+	}
+
+	if len(cfg.IndexMap) == 0 {
+		return fmt.Errorf("no rows to query")
+	}
+
+	var indexes []string
+	for index := range cfg.IndexMap {
+		indexes = append(indexes, index)
+	}
+
+	for j := 0; ; j++ {
+		if cfg.QueryCount > 0 {
+			if j >= cfg.QueryCount {
+				break
+			}
+		} else {
+			// else keep doing queries forever...
+			if loops > 0 && loops%500 == 0 {
+				// ...but account for any new data arrived by getting
+				// the schema and rows again every so often.
+				loops++
+				goto NewSetup
+			}
+		}
+		if totalQ > 0 && totalQ%100 == 0 {
+			report()
+		}
+
+		index := indexes[cfg.Rnd.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		PanicOn(err)
+
+		if cfg.Verbose {
+			fmt.Printf("pql = '%v'\n", pql)
+		}
+
+		// Query node0.
+		res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
+			return err
+		}
+		if cfg.VeryVerbose {
+			fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+		}
+		totalQ++
+		loops++
+
+	}
+
+	return nil
+}
+
+type Features struct {
+	Slc           []IndexFieldRow
+	Ranges        []IndexFieldRange
+	Distinctables []IndexFieldRange
+	SlcWeight     int
+	RangeWeight   int
+}
+
+// Pick either a feature entry or a random query on a range, weighted
+// by number of features and approximate weight of ranges
+func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight)
+	if r < f.SlcWeight {
+		return f.Slc[r].Query(cfg)
+	}
+	r = cfg.Rnd.Intn(len(f.Ranges))
+	return f.Ranges[r].Query(cfg)
+}
+
+func NewRandomQueryConfig() *RandomQueryConfig {
+	return &RandomQueryConfig{
+		IndexMap: make(map[string]*Features),
+	}
+}
+
+type IndexFieldRow struct {
+	Index    string
+	Field    string
+	RowID    uint64
+	RowKey   string
+	IsRowKey bool
+	HasTime  bool
+	IsInt    bool
+}
+
+func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
+	fromTo := ""
+	// 5% of queries on a time field will use the standard view
+	// anyway.
+	if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
+		startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
+		endHours := cfg.Rnd.Int63n(cfg.TimeRange-startHours) + 1 + startHours
+		startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
+		endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
+		fromTo = fmt.Sprintf(", from=%s, to=%s",
+			startTime.Format(pilosaTimeFmt),
+			endTime.Format(pilosaTimeFmt))
+	}
+	if fea.IsRowKey {
+		return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)}
+	}
+	return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
+}
+
+type IndexFieldRange struct {
+	Index           string
+	Field           string
+	Min, Max, Scale int64
+	ScaleDiv        float64
+	Range           uint64
+}
+
+// We want to pick one of (1) a single-operation filter, (2) a
+// between-filter of some kind.
+// So, that's one of <=, >=, ==, !=, >, <, or
+// one of [<, <], [<, <=], [<=, <=], [<=, <].
+var binaryOps = []string{
+	"<=", ">=", "==", "!=", "<", ">",
+}
+
+func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Int63n(10)
+	// this is unevenly weighted, but there's no Uint64N, and
+	// Int63n can't represent the whole range.
+	v1 := cfg.Rnd.Uint64() % i.Range
+	v2 := cfg.Rnd.Uint64() % i.Range
+	if v1 > v2 {
+		v1, v2 = v2, v1
+	}
+	v1 = v1 + uint64(i.Min)
+	v2 = v2 + uint64(i.Min)
+	var v1s, v2s string
+	if i.Scale != 0 {
+		v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1))/i.ScaleDiv)
+		v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2))/i.ScaleDiv)
+	} else {
+		v1s = strconv.FormatInt(int64(v1), 10)
+		v2s = strconv.FormatInt(int64(v2), 10)
+	}
+	if r < 4 {
+		lte := "<="
+		op1 := lte[:1+(r&1)]
+		op2 := lte[:1+((r>>1)&1)]
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)",
+			v1s, op1, i.Field, op2, v2s)}
+	} else {
+		if cfg.Rnd.Int63n(2) == 1 {
+			v1s = v2s
+		}
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r-4], v1s)}
+	}
+}
+
+// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
+// and spits back a PQL query
+//
+func (cfg *RandomQueryConfig) Setup(api API) (err error) {
+	ctx := context.Background()
+	cfg.Info, err = api.Schema(ctx)
+	if err != nil {
+		return err
+	}
+	foundIntField := false
+	for i, ii := range cfg.Info {
+		_ = i
+		for k, fld := range ii.Fields {
+			_ = k
+			switch fld.Options.Type {
+			case "set", "mutex", "time":
+				pql := fmt.Sprintf("Rows(%v)", fld.Name)
+
+				res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
+				PanicOn(err)
+				if cfg.VeryVerbose {
+					fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+				}
+				// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
+				// e.g.
+				// success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"}
+				// success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}'
+
+				switch x := res.Results[0].(type) {
+				case *pilosa.RowIdentifiers:
+					// internalClient gets this
+					cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time")
+				case pilosa.RowIdentifiers:
+					// test gets this
+					cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time")
+				}
+			case "int":
+				foundIntField = true
+				fallthrough // I bet you thought you'd never see this used
+			case "decimal":
+				cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
+			default:
+				AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
+			}
+		}
+	}
+
+	cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
+	if foundIntField {
+		cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
+	}
+	seed := int64(42)
+	cfg.Rnd = rand.New(rand.NewSource(seed))
+
+	return nil
+}
+
+func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) {
+	for _, rowID := range x.Rows {
+		cfg.AddFeature(index, field, rowID, "", false, hasTime)
+	}
+	for _, rowKey := range x.Keys {
+		cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
+	}
+}
+
+const maxEffectiveRange = 1000000
+
+func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) {
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	if min.Scale != scale || max.Scale != scale {
+		PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
+			min.Scale, max.Scale, scale))
+	}
+
+	effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
+	// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
+	// wraps to 0. Anything closer together will be fine. We accept the loss
+	// of accuracy in the range from not representing quite the full value
+	// in that edge case.
+	if effectiveRange == 0 {
+		effectiveRange--
+	}
+
+	// we assume that the Value of the field is already scaled, I guess?
+	newRange := IndexFieldRange{
+		Index:    index,
+		Field:    field,
+		Min:      min.Value,
+		Max:      max.Value,
+		Scale:    scale,
+		ScaleDiv: math.Pow(10, float64(scale)),
+		Range:    effectiveRange,
+	}
+	f.Ranges = append(f.Ranges, newRange)
+	if !decimal {
+		f.Distinctables = append(f.Distinctables, newRange)
+	}
+	// We want to add more values for larger int fields, but the
+	// default KitchenSink field has a range of 1<<64 which would make
+	// it completely dominate weights, so...
+	if effectiveRange > maxEffectiveRange {
+		effectiveRange = maxEffectiveRange
+	}
+	f.RangeWeight += int(effectiveRange)
+}
+
+func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
+
+	tree := cfg.GenTree(index, cfg.TreeDepth)
+	pql = tree.ToPQL()
+
+	// avoid using too much bandwidth, just count the final bitmap.
+	pql = fmt.Sprintf("Count(%v)", pql)
+	return
+}
+
+type Tree struct {
+	Chd  []*Tree
+	S    string
+	Args []string // Extra args to pass after children, such as a field for Distinct.
+}
+
+func (tr *Tree) StringIndent(ind int) (s string) {
+	spc := strings.Repeat("    ", ind)
+	spc1 := strings.Repeat("    ", ind+1)
+	var chds []string
+	leaf := true
+	if len(tr.Chd) == 0 {
+		// leaf
+	} else {
+		leaf = false
+		for _, chd := range tr.Chd {
+			chds = append(chds, chd.StringIndent(ind+1))
+		}
+	}
+	if leaf {
+		s += fmt.Sprintf("%v %v\n", spc1, tr.S)
+	} else {
+		for i, c := range chds {
+			if i == 0 {
+				s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c)
+			} else {
+				s += fmt.Sprintf("%v", c)
+			}
+		}
+	}
+	return
+}
+
+const pilosaTimeFmt = "2006-01-02T15:04"
+
+func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
+	features := cfg.IndexMap[index]
+	if depth == 0 {
+		return features.RandomQuery(cfg)
+	}
+
+	r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
+	f := cfg.BitmapFunc[r]
+	tr = &Tree{S: f}
+	numChild := 2
+	switch f {
+	case "Union", "Intersect", "Xor":
+		numChild = cfg.Rnd.Intn(8) + 2
+	case "Not":
+		numChild = 1
+	case "Difference":
+		numChild = 2
+	case "Distinct":
+		numChild = 1
+		r = cfg.Rnd.Intn(len(features.Distinctables))
+		tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field))
+	}
+	for i := 0; i < numChild; i++ {
+		tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
+	}
+	return
+}
+
+func (tr *Tree) ToPQL() (s string) {
+
+	if len(tr.Chd) == 0 {
+		// leaf
+		return tr.S
+	}
+
+	var chds []string
+	for _, c := range tr.Chd {
+		chds = append(chds, c.ToPQL())
+	}
+	// If we had no extra args, this does nothing.
+	chds = append(chds, tr.Args...)
+	all := strings.Join(chds, ", ")
+	return fmt.Sprintf("%v(%v)", tr.S, all)
+}
+
+func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) {
+
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	f.Slc = append(f.Slc, IndexFieldRow{
+		Index:    index,
+		Field:    field,
+		RowID:    rowID,
+		RowKey:   rowKey,
+		IsRowKey: isRowKey,
+		HasTime:  hasTime,
+	})
+	f.SlcWeight++
+}

cmd/random-query/main_test.go

@@ -0,0 +1,166 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"math/rand"
+	"strconv"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+func Test_RandomQuery(t *testing.T) {
+
+	cfg := NewRandomQueryConfig()
+
+	nNodes := 1
+	nReplicas := 1
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	defer c.Close()
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+	_ = dirs
+
+	ctx := context.Background()
+
+	indexes := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(indexes))
+	field := make([]*pilosa.Field, len(indexes))
+
+	var err error
+
+	for i := range indexes {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	timestamp := int64(0)
+
+	for i := range indexes {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, uint64(j))
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the primary and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexes[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+		//vv("rowIDs = '%#v'", rowIDs)
+		//vv("colKeys = '%#v'", colKeys)
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+		//qcx.Reset()
+	}
+	// end of setup.
+
+	PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
+
+	for j := 0; j < 4; j++ {
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		PanicOn(err)
+
+		//vv("pql = '%v'", pql)
+
+		// Query node0.
+		res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = res
+		//vv("success on pql = '%v'; res='%v'", pql, res.Results[0])
+	}
+}

cmd/rbf.go

@@ -0,0 +1,160 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newRBFCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "rbf",
+		Short: "Inspect RBF data files.",
+		Long: `
+Provides a set of commands for inspecting RBF data files.
+`,
+	}
+	cmd.AddCommand(newRBFCheckCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFDumpCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPagesCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPageCommand(stdin, stdout, stderr))
+	return cmd
+}
+
+func newRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFCheckCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "check [flags] PATH",
+		Short: "Run consistency check on RBF data.",
+		Long: `
+Executes a consistency check on an RBF data directory.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFDumpCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "dump [flags] PATH PGNO [PGNO...]",
+		Short: "Prints RBF raw page data",
+		Long: `
+Dumps the raw hex data for one or more RBF pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPagesCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "pages [flags] PATH",
+		Short: "Prints metadata for the list of all pages",
+		Long: `
+Prints a line for every page in the database with its type/status.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&c.WithTree, "with-tree", false, "Display b-tree name for each row")
+	return cmd
+}
+
+func newRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPageCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "page [flags] PATH PGNO [PGNO...]",
+		Short: "Prints data for a page(s)",
+		Long: `
+Prints the header & cell data for one or more pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}

cmd/root.go

@@ -26,23 +26,18 @@ import (
 )
 
 func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	productName := "Pilosa " + pilosa.Version
-	if pilosa.EnterpriseEnabled {
-		productName = "Pilosa Enterprise " + pilosa.Version
-	}
 	rc := &cobra.Command{
 		Use: "pilosa",
+		// TODO: These short/long descriptions could use some updating.
 		Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
-		// TODO - is documentation actually there?
 		Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
+tools for administering Pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
 at https://www.pilosa.com/docs/.
 
-` + productName + `
+` + pilosa.VersionInfo() + "\n",
-Build Time: ` + pilosa.BuildTime + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
 			err := setAllConfig(v, cmd.Flags(), "PILOSA")
@@ -74,6 +69,7 @@ Build Time: ` + pilosa.BuildTime + "\n",
 	rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr))
 	rc.AddCommand(newImportCommand(stdin, stdout, stderr))
 	rc.AddCommand(newInspectCommand(stdin, stdout, stderr))
+	rc.AddCommand(newRBFCommand(stdin, stdout, stderr))
 	rc.AddCommand(newServeCmd(stdin, stdout, stderr))
 	rc.AddCommand(newHolderCmd(stdin, stdout, stderr))
 

cmd/root_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/spf13/cobra"
 )
 
@@ -55,6 +56,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +68,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()
@@ -132,7 +136,7 @@ func executeDry(t *testing.T, tests []commandTest) {
 // a temp config file with the cfgFileContent string as its content.
 func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
 	// make config file
-	cfgFile, err := ioutil.TempFile("", "")
+	cfgFile, err := testhook.TempFile(t, "cmdconf")
 	failErr(t, err, "making temp file")
 	_, err = cfgFile.WriteString(ct.cfgFileContent)
 	failErr(t, err, "writing config to temp file")
@@ -145,6 +149,10 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
 		err = os.Setenv(name, val)
 		failErr(t, err, fmt.Sprintf("setting environment variable '%s' to '%s'", name, val))
 	}
+	// address common case where system might have postgres bind
+	// setting that an existing running Pilosa is using (causing test
+	// failures due to port conflict)
+	os.Setenv("PILOSA_POSTGRES_BIND", "")
 
 	// make command and set args
 	rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard)
@@ -173,20 +181,16 @@ func TestRootCommand(t *testing.T) {
 }
 
 func TestRootCommand_Config(t *testing.T) {
-	file, err := ioutil.TempFile("", "test.conf")
+	file, err := testhook.TempFile(t, "test.conf")
 	if err != nil {
-		panic(err)
+		t.Fatalf("creating config file: %v", err)
 	}
 	config := `data-dir = "/tmp/pil5_0"
 bind = "127.0.0.1:10101"
 
 [cluster]
   replicas = 2
-  partitions = 128
+  partitions = 128`
-  hosts = [
-    "127.0.0.1:10101",
-    "127.0.0.1:10111",
-  ]`
 	if _, err := file.Write([]byte(config)); err != nil {
 		t.Fatalf("writing config file: %v", err)
 	}

cmd/server_test.go

@@ -15,13 +15,14 @@
 package cmd_test
 
 import (
-	"io/ioutil"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/cmd"
 	_ "github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/toml"
 	"github.com/pkg/errors"
 )
@@ -34,17 +35,24 @@ func TestServerHelp(t *testing.T) {
 	}
 }
 
+// I have no idea why the linter in ci is complaining about this being unused.
+func nextPort() string { //nolint:unused
+	return fmt.Sprintf(`"localhost:%d"`, 0)
+}
+
 func TestServerConfig(t *testing.T) {
-	actualDataDir, err := ioutil.TempDir("", "")
+	t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
+	actualDataDir, err := testhook.TempDir(t, "")
 	failErr(t, err, "making data dir")
-	logFile, err := ioutil.TempFile("", "")
+	logFile, err := testhook.TempFile(t, "")
 	failErr(t, err, "making log file")
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
+				"PILOSA_LONG_QUERY_TIME":         "1m30s",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
 				"PILOSA_MAX_WRITES_PER_REQUEST":  "2000",
 				"PILOSA_PROFILE_BLOCK_RATE":      "9123",
@@ -52,15 +60,13 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
-	bind = "localhost:0"
+	bind = ` + nextPort() + `
+	bind-grpc = ` + nextPort() + `
 	max-writes-per-request = 3000
+	long-query-time = "1m10s"
 
 	[cluster]
-		disabled = true
 		replicas = 2
-		hosts = [
-			"localhost:19444",
-		]
 		long-query-time = "1m10s"
 	[profile]
 		block-rate = 100
@@ -71,7 +77,7 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.DataDir, actualDataDir)
 				v.Check(cmd.Server.Config.Bind, "localhost:42454")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
@@ -95,20 +101,15 @@ func TestServerConfig(t *testing.T) {
 				"PILOSA_PROFILE_MUTEX_FRACTION": "444",
 			},
 			cfgFileContent: `
-	bind = "localhost:0"
+	bind = ` + nextPort() + `
+	bind-grpc = ` + nextPort() + `
 	data-dir = "` + actualDataDir + `"
-	[cluster]
-		disabled = true
-		hosts = [
-			"localhost:19444",
-		]
 	[profile]
 		block-rate = 100
 		mutex-fraction = 10
 	`,
 			validation: func() error {
 				v := validator{}
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
 				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
 				v.Check(cmd.Server.Config.Profile.BlockRate, 4832)
@@ -122,11 +123,8 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
-	[cluster]
-		hosts = [
-			"localhost:19444",
-		]
 	[anti-entropy]
 		interval = "11m0s"
 	[metric]
@@ -139,7 +137,6 @@ func TestServerConfig(t *testing.T) {
 	`,
 			validation: func() error {
 				v := validator{}
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
 				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
 				v.Check(cmd.Server.Config.LogPath, logFile.Name())
 				v.Check(cmd.Server.Config.Metric.Service, "statsd")
@@ -163,6 +160,7 @@ func TestServerConfig(t *testing.T) {
 
 	// run server tests
 	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
 			com := test.setupCommand(t)
 			executed := make(chan struct{})
 			var execErr error
@@ -186,5 +184,90 @@ func TestServerConfig(t *testing.T) {
 				t.Fatalf("Failed test %d due to: %v", i, err)
 			}
 			test.reset()
+		})
+	}
+}
+func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
+	t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
+	actualDataDir, err := testhook.TempDir(t, "")
+	failErr(t, err, "making data dir")
+
+	tests := []commandTest{
+		// TEST 0
+		{
+			args: []string{"server", "--long-query-time", "1m10s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+             	data-dir = "` + actualDataDir + `"
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*70))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*70))
+				return v.Error()
+			},
+		},
+		// TEST 1
+		{
+			args: []string{"server", "--cluster.long-query-time", "1m20s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*80))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*80))
+				return v.Error()
+			},
+		},
+		// TEST 2: Use old value if both are provided because it is the simplest implementation
+		{
+			args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*50))
+				v.Check(toml.Duration(cmd.Server.Config.Cluster.LongQueryTime), toml.Duration(time.Second*90))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*90))
+				return v.Error()
+			},
+		},
+	}
+
+	// run server tests
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			com := test.setupCommand(t)
+			executed := make(chan struct{})
+			var execErr error
+			go func() {
+				execErr = com.Execute()
+				close(executed)
+			}()
+			select {
+			case <-cmd.Server.Started:
+			case <-executed:
+			}
+			if execErr != nil {
+				t.Fatalf("executing server command: %v", execErr)
+			}
+			err := cmd.Server.Close()
+			failErr(t, err, "closing pilosa server command")
+			<-executed
+			failErr(t, execErr, "executing command")
+
+			if err := test.validation(); err != nil {
+				t.Fatalf("Failed test %d due to: %v", i, err)
+			}
+			test.reset()
+		})
 	}
 }

cmd/slurp/slurp.go

@@ -0,0 +1,263 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	pnet "github.com/pilosa/pilosa/v2/net"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+// slurp: slurp is a load-tester for importing bulk data.
+//        It allows us to measure write performance.
+
+type stateMachine struct {
+	viewData  map[string][]byte
+	lastIndex string
+	lastField string
+	lastShard uint64
+	state     string
+	client    *http.InternalClient
+	start     time.Time
+
+	profile string
+	host    string
+}
+
+func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
+	parts := strings.Split(h.Name, "/")
+	switch parts[0] {
+	case "roaring":
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		PanicOn(err)
+		if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return err
+		}
+		if _, already := r.viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		r.viewData[view] = roaringData
+		r.lastIndex = index
+		r.lastField = field
+		r.lastShard = shard
+	case "bolt":
+		if r.state == "roaring" {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+			VV("Finished import %v", time.Since(r.start))
+
+			if r.profile != "" {
+				stopProfile(r.host, r.profile)
+			}
+		}
+		//
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+
+		switch v := parts[len(parts)-1]; v {
+		case "keys":
+			index := parts[1]
+			fieldName := parts[2]
+			if fieldName == "_keys" {
+				//skip index keys are not not real fields so will have no need for field keys
+				return nil
+
+			}
+
+			byteData, err := ioutil.ReadAll(tr)
+			PanicOn(err)
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
+			if err != nil {
+				return err
+			}
+		default:
+			VV("%v", h.Name)
+			index := parts[1]
+			partition, err := strconv.ParseUint(v, 10, 64)
+			if err != nil {
+				return err
+			}
+			byteData, err := ioutil.ReadAll(tr)
+			PanicOn(err)
+
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	r.state = parts[0]
+	return nil
+}
+func (r *stateMachine) Upload() error {
+	if len(r.viewData) > 0 {
+		request := &pilosa.ImportRoaringRequest{
+			Views: r.viewData,
+		}
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+		err := r.client.ImportRoaring(context.Background(), uri, r.lastIndex, r.lastField, r.lastShard, false, request)
+		if err != nil {
+			return err
+		}
+		r.viewData = make(map[string][]byte)
+	}
+	return nil
+}
+
+func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error {
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	runner := &stateMachine{
+		viewData: make(map[string][]byte),
+		start:    time.Now(),
+		profile:  profile,
+		host:     host,
+	}
+	runner.client = client
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			_ = runner.Upload()
+			break
+		}
+		if err != nil {
+			PanicOn(err)
+		}
+		err = runner.NewHeader(header, tarReader)
+		PanicOn(err)
+	}
+	return nil
+}
+
+func main() {
+	var host string
+	var profile string
+	var tarSrcPath string
+	flag.StringVar(&host, "host", "127.0.0.1:10101", "host to import into")
+	flag.StringVar(&profile, "profile", "", "profile and save a cpu profile of the import to this file")
+	flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import")
+	flag.Parse()
+
+	uri, err := pnet.NewURIFromAddress(host)
+	PanicOn(err)
+	globURI = uri
+
+	h := &gohttp.Client{}
+	if profile != "" {
+		startProfile(host)
+	}
+	c, err := http.NewInternalClient(host, h)
+	PanicOn(err)
+
+	t0 := time.Now()
+	println("uploading", tarSrcPath)
+	PanicOn(UploadTar(tarSrcPath, c, profile, host))
+	VV("total elapsed '%v'", time.Since(t0))
+}
+
+func startProfile(host string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/start",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+}
+
+func stopProfile(host, outfile string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/stop",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+
+	fd, err := os.Create(outfile)
+	PanicOn(err)
+	defer fd.Close()
+	_, err = io.Copy(fd, resp.Body)
+	PanicOn(err)
+
+}
+
+var globURI *pnet.URI
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
+	return globURI
+}

const_amd64.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,11 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-// +build enterprise
 
-package server
+// +build amd64
 
-import (
+package pilosa
-	_ "github.com/pilosa/pilosa/v2/enterprise"
+
-)
+const TxInitialMmapSize = 4 << 30 // 4GB

const_other.go

@@ -0,0 +1,21 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !amd64
+
+package pilosa
+
+// this is a stubbed out file to let 386/arm build.
+
+const TxInitialMmapSize = 1 << 30 // 1GB

ctl/check_test.go

@@ -16,20 +16,22 @@ package ctl
 
 import (
 	"bytes"
-	"encoding/hex"
 	"io"
-	"io/ioutil"
-	"math/rand"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
 	"context"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestCheckCommand_RunCacheFile(t *testing.T) {
-	cacheFile := TempFileName("test", ".cache")
+	fi, err := testhook.TempFile(t, "test*.cache")
+	if err != nil {
+		t.Fatalf("creating test file: %v", err)
+	}
+	cacheFile := fi.Name()
 
 	rder := []byte{}
 	stdin := bytes.NewReader(rder)
@@ -37,7 +39,7 @@ func TestCheckCommand_RunCacheFile(t *testing.T) {
 	cm := NewCheckCommand(stdin, w, w)
 	cm.Paths = []string{cacheFile}
 
-	err := cm.Run(context.Background())
+	err = cm.Run(context.Background())
 	w.Close()
 	var buf bytes.Buffer
 	if _, err := io.Copy(&buf, r); err != nil {
@@ -50,7 +52,11 @@ func TestCheckCommand_RunCacheFile(t *testing.T) {
 }
 
 func TestCheckCommand_RunSnapshot(t *testing.T) {
-	snapshotFile := TempFileName("test", ".snapshotting")
+	fi, err := testhook.TempFile(t, "test*.snapshotting")
+	if err != nil {
+		t.Fatalf("creating test file: %v", err)
+	}
+	snapshotFile := fi.Name()
 
 	rder := []byte{}
 	stdin := bytes.NewReader(rder)
@@ -58,7 +64,7 @@ func TestCheckCommand_RunSnapshot(t *testing.T) {
 	cm := NewCheckCommand(stdin, w, w)
 	cm.Paths = []string{snapshotFile}
 
-	err := cm.Run(context.Background())
+	err = cm.Run(context.Background())
 	w.Close()
 	var buf bytes.Buffer
 	if _, err := io.Copy(&buf, r); err != nil {
@@ -71,10 +77,11 @@ func TestCheckCommand_RunSnapshot(t *testing.T) {
 }
 
 func TestCheckCommand_Run(t *testing.T) {
-	file, err := ioutil.TempFile("", "")
+	file, err := testhook.TempFile(t, "run-command")
 	if err != nil {
 		t.Fatal(err)
 	}
+	fname := file.Name()
 	if _, err := file.Write([]byte("1234,1223")); err != nil {
 		t.Fatalf("writing to temp file: %v", err)
 	}
@@ -84,7 +91,7 @@ func TestCheckCommand_Run(t *testing.T) {
 	stdin := bytes.NewReader(rder)
 	r, w, _ := os.Pipe()
 	cm := NewCheckCommand(stdin, w, w)
-	cm.Paths = []string{file.Name()}
+	cm.Paths = []string{fname}
 
 	err = cm.Run(context.Background())
 	w.Close()
@@ -99,10 +106,3 @@ func TestCheckCommand_Run(t *testing.T) {
 	}
 	//	Todo: need correct roaring file for happy path
 }
-
-// TempFileName generates a temporary filename with extension
-func TempFileName(prefix, suffix string) string {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix)
-}

ctl/common.go

@@ -31,12 +31,12 @@ type CommandWithTLSSupport interface {
 }
 
 // SetTLSConfig creates common TLS flags
-func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
+func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
-	flags.StringVarP(certificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
+	flags.StringVarP(certificatePath, prefix+"tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
-	flags.StringVarP(certificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
+	flags.StringVarP(certificateKeyPath, prefix+"tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
-	flags.StringVarP(caCertPath, "tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
+	flags.StringVarP(caCertPath, prefix+"tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
-	flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
+	flags.BoolVarP(skipVerify, prefix+"tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
-	flags.BoolVarP(enableClientVerification, "tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
+	flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
 }
 
 // commandClient returns a pilosa.InternalHTTPClient for the command

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -257,7 +261,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, useColumnKeys, useRowK
 func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowKeys bool, bits []pilosa.Bit) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
-	// If keys are used, all bits are sent to the primary translate store (i.e. coordinator).
+	// If keys are used, all bits are sent to the primary translate store.
 	if useColumnKeys || useRowKeys {
 		logger.Printf("importing keys: n=%d", len(bits))
 		if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
 			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
-		}
 
-		// Parse FieldValue.
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
+			}
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
 			value, err := strconv.ParseInt(record[1], 10, 64)
 			if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
 			}
-		val.Value = value
+			req.Values = append(req.Values, value)
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
 		}
-			a = a[:0]
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,9 +25,11 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestImportCommand_Validation(t *testing.T) {
@@ -57,7 +59,7 @@ func TestImportCommand_Basic(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import.csv")
+		file, err := testhook.TempFile(t, "import.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -72,7 +74,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -89,7 +91,7 @@ func TestImportCommand_Basic(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import.csv")
+		file, err := testhook.TempFile(t, "import.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -101,7 +103,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -122,7 +124,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import-value.csv")
+		file, err := testhook.TempFile(t, "import-value.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -134,7 +136,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -161,7 +163,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import-value.csv")
+		file, err := testhook.TempFile(t, "import-value.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -176,7 +178,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -206,7 +208,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -218,7 +220,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -246,7 +248,7 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,10 +270,10 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 	}
 	ctx := context.Background()
 
-	c := test.MustRunCluster(t, 2)
+	c := test.MustRunCluster(t, 3)
 	defer c.Close()
-	cmd0 := c[0]
+	cmd0 := c.GetNode(0)
-	cmd1 := c[1]
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -299,20 +301,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
+		if err := test.RetryUntil(2*time.Second, func() error {
 			qry := "Count(Row(f=foo0))"
 			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
 			if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
+				return fmt.Errorf("Querying data for validation: %s", err)
 			}
 
 			// Read body and unmarshal response.
 			exp := `{"results":[100]}` + "\n"
 			if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
+				return fmt.Errorf("reading: %s", err)
 			} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.
@@ -320,7 +328,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -332,7 +340,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -358,7 +366,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -366,7 +374,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 	cm.Host = cmd.API.Node().URI.HostPort()
 	cm.Index = "i"
 	cm.Field = "f"
-	file, err := ioutil.TempFile("", "import.csv")
+	file, err := testhook.TempFile(t, "import.csv")
 	if err != nil {
 		t.Fatalf("creating tempfile: %v", err)
 	}
@@ -380,7 +388,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid row id on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatalf("creating tempfile: %v", err)
 	}
@@ -394,7 +402,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid column id on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -408,7 +416,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid timestamp on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -446,12 +454,12 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-value.csv")
+	file, err := testhook.TempFile(t, "import-value.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -483,7 +491,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	}
 
 	file.Close()
-	file, err = ioutil.TempFile("", "import-value2.csv")
+	file, err = testhook.TempFile(t, "import-value2.csv")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -498,7 +506,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	}
 
 	file.Close()
-	file, err = ioutil.TempFile("", "import-value3.csv")
+	file, err = testhook.TempFile(t, "import-value3.csv")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -522,7 +530,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -540,7 +548,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 	cm.Field = "f"
 
 	t.Run("Valid", func(t *testing.T) {
-		file, err := ioutil.TempFile("", "import-bool.csv")
+		file, err := testhook.TempFile(t, "import-bool.csv")
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -558,7 +566,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	// Ensure that invalid bool values return an error.
 	t.Run("Invalid", func(t *testing.T) {
-		file, err := ioutil.TempFile("", "import-invalid-bool.csv")
+		file, err := testhook.TempFile(t, "import-invalid-bool.csv")
 		if err != nil {
 			t.Fatal(err)
 		}

ctl/inspect.go

@@ -16,15 +16,23 @@ package ctl
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
+	"hash/fnv"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
-	"unsafe"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pb"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -33,6 +41,12 @@ import (
 type InspectCommand struct {
 	// Path to data file
 	Path string
+	// don't list details of objects
+	Quiet bool
+	// list only this many objects
+	Max int
+	// Filters:
+	InspectOpts pilosa.InspectRequest
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
+type pointerContext struct {
+	from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(cmd.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+		if cmd.Max != 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(cmd.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+		if cmd.Max > 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
 // Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
 	if err != nil {
@@ -58,7 +183,172 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "statting file")
 	}
+	if fi.IsDir() {
+		total := 0
+		infos, err := f.Readdir(0)
+		if err != nil {
+			return err
+		}
+		if len(infos) == 0 {
+			return errors.New("directory contains no files")
+		}
 
+		names := make([]string, len(infos))
+		nameToInfo := make(map[string]os.FileInfo, len(infos))
+		// find numeric-only names; we'll operate on
+		// either those, or the whole holder if we find
+		// a .topology file.
+		n := 0
+		for _, fi := range infos {
+			name := fi.Name()
+			if name == ".topology" {
+				return cmd.InspectHolder(ctx, cmd.Path)
+			}
+			if _, err := strconv.Atoi(name); err == nil {
+				names[n] = name
+				nameToInfo[name] = fi
+				n++
+			}
+		}
+		if n == 0 {
+			return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+		}
+		names = names[:n]
+		fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+		for _, name := range names {
+			f2, err := os.Open(filepath.Join(cmd.Path, name))
+			if err != nil {
+				return fmt.Errorf("opening %q: %v", name, err)
+			}
+			fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+			err = cmd.InspectFile(f2, nameToInfo[name])
+			total++
+			f2.Close()
+			if err != nil {
+				return fmt.Errorf("inspecting %q: %v", name, err)
+			}
+		}
+		return nil
+	}
+	return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology pb.Topology, myID string, err error) {
+	buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, errors.Wrap(err, "reading file")
+	}
+	if err := proto.Unmarshal(buf, &topology); err != nil {
+		return topology, myID, errors.Wrap(err, "unmarshalling")
+	}
+	sort.Slice(topology.NodeIDs,
+		func(i, j int) bool {
+			return topology.NodeIDs[i] < topology.NodeIDs[j]
+		})
+	buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, nil
+	}
+	myID = strings.TrimSpace(string(buf))
+	return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+	var shardMap map[uint64]int
+	var ok bool
+	if shardMap, ok = partitions[index]; !ok {
+		shardMap = make(map[uint64]int)
+		partitions[index] = shardMap
+	}
+	if partition, ok = shardMap[shard]; !ok {
+		var buf [8]byte
+		binary.BigEndian.PutUint64(buf[:], shard)
+
+		// Hash the bytes and mod by partition count.
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write(buf[:])
+		partition = int(h.Sum64() % uint64(partitionN))
+		shardMap[shard] = partition
+	}
+	return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+	parts := strings.Split(path, "/")
+	shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+	holder := pilosa.NewHolder(path, nil)
+	holder.Opts.Inspect = true
+	holder.Opts.ReadOnly = true
+	err := holder.Open()
+	if err != nil {
+		return fmt.Errorf("%s: holder open: %v", path, err)
+	}
+	holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+	if err != nil {
+		return fmt.Errorf("%s: inspect: %v", path, err)
+	}
+	myPartition := 0
+	topology, myID, err := loadTopology(path)
+	if err == nil {
+		fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+		if len(topology.NodeIDs) > 1 {
+			fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+		}
+		found := false
+		for i := range topology.NodeIDs {
+			if topology.NodeIDs[i] == myID {
+				found = true
+				myPartition = i
+				break
+			}
+		}
+		if !found {
+			fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+		}
+	} else {
+		fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+	}
+	for _, name := range holderInfo.FragmentNames {
+		partition, err := findPartitionPath(name, len(topology.NodeIDs))
+		if err != nil {
+			fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+		} else {
+			if partition == myPartition {
+				fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+			} else {
+				fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+			}
+		}
+		details := holderInfo.FragmentInfo[name]
+		cmd.DisplayInfo(details.BitmapInfo)
+		if details.BlockChecksums != nil {
+			fmt.Fprintf(cmd.Stdout, "  Checksums [%d total]:\n", len(details.BlockChecksums))
+			for _, block := range details.BlockChecksums {
+				fmt.Fprintf(cmd.Stdout, "   %8d: %x\n", block.ID, block.Checksum)
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
 	// Memory map the file.
 	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
@@ -72,39 +362,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	}()
 	// Attach the mmap file to the bitmap.
 	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
+	fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
-	bm := roaring.NewBitmap()
+	var info roaring.BitmapInfo
-	if err := bm.UnmarshalBinary(data); err != nil {
+	_, _, err = roaring.InspectBinary(data, true, &info)
-		return errors.Wrap(err, "unmarshalling")
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	cmd.DisplayInfo(info)
+	if err != nil {
+		return errors.Wrap(err, "inspecting")
 	}
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	return nil
+}
 
-	// Retrieve stats.
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
-	t = time.Now()
+	pC := pointerContext{
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
+		from: info.From,
-	info := bm.Info()
+		to:   info.To,
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	}
 
 	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
+	fmt.Fprintf(cmd.Stdout, "  Bitmap Info:\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Bits: %d\n", info.BitCount)
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintf(cmd.Stdout, "    Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Operations: %d (%d bits)\n", info.Ops, info.OpN)
 	fmt.Fprintln(cmd.Stdout, "")
 
 	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
+	if !cmd.Quiet {
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+		if info.ContainerCount > 0 {
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
+			cmd.PrintContainers(info, pC)
-	for _, ci := range info.Containers {
+		}
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
+		if info.Ops > 0 {
-			ci.Key,
+			cmd.PrintOps(info)
-			ci.Type,
+		}
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
 	}
-	tw.Flush()
-
-	return nil
 }

ctl/inspect_test.go

@@ -18,10 +18,11 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestInspectCommand_Run(t *testing.T) {
@@ -30,7 +31,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	r, w, _ := os.Pipe()
 
 	cm := NewInspectCommand(stdin, w, w)
-	file, err := ioutil.TempFile("", "inspectTest")
+	file, err := testhook.TempFile(t, "inspectTest")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -41,7 +42,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	file.Close()
 	cm.Path = file.Name()
 	err = cm.Run(context.Background())
-	expectedError := "unmarshalling: "
+	expectedError := "inspecting: "
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Fatalf("expected error '%s', got '%v'", expectedError, err)
 	}
@@ -52,7 +53,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	if err != nil {
 		t.Fatalf("copying data: %v", err)
 	}
-	if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+	if !strings.Contains(buf.String(), "inspecting bitmap...") {
 		t.Fatalf("Inspect doesn't work: %s", err)
 	}
 

ctl/rbf_check.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFCheckCommand represents a command for running a consistency check on RBF.
+type RBFCheckCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFCheckCommand returns a new instance of RBFCheckCommand.
+func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckCommand {
+	return &RBFCheckCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Run check on the database.
+	if err := db.Check(); err != nil {
+		return err
+	}
+
+	// If successful, print a success message.
+	fmt.Fprintln(cmd.Stdout, "ok")
+
+	return nil
+}

ctl/rbf_dump.go

@@ -0,0 +1,111 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFDumpCommand represents a command for dumping raw data for an RBF page.
+type RBFDumpCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFDumpCommand returns a new instance of RBFDumpCommand.
+func NewRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFDumpCommand {
+	return &RBFDumpCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFDumpCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch each page & dump.
+	for _, pgno := range cmd.Pgnos {
+		buf, err := tx.PageData(pgno)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(cmd.Stdout, "## PAGE %d\n", pgno)
+		fmt.Fprintln(cmd.Stdout, compressedHexDump(buf))
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func compressedHexDump(b []byte) string {
+	const prefixN = len("00000000")
+
+	var output []string
+	var prev string
+	var ellipsis bool
+
+	lines := strings.Split(strings.TrimSpace(hex.Dump(b)), "\n")
+	for i, line := range lines {
+		// Add line to output if it is not repeating or the last line.
+		if i == 0 || i == len(lines)-1 || trimPrefixN(line, prefixN) != trimPrefixN(prev, prefixN) {
+			output = append(output, line)
+			prev, ellipsis = line, false
+			continue
+		}
+
+		// Add an ellipsis for the first duplicate line.
+		if !ellipsis {
+			output = append(output, "...")
+			ellipsis = true
+			continue
+		}
+	}
+
+	return strings.Join(output, "\n")
+}
+
+// trimPrefixN trims n bytes from the beginning of a string.
+func trimPrefixN(s string, n int) string {
+	if len(s) < n {
+		return ""
+	}
+	return s[n:]
+}

ctl/rbf_page.go

@@ -0,0 +1,140 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFPageCommand represents a command for printing data for a single RBF page.
+type RBFPageCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPageCommand returns a new instance of RBFPageCommand.
+func NewRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPageCommand {
+	return &RBFPageCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPageCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch the page.
+	pages, err := tx.Pages(cmd.Pgnos)
+	if err != nil {
+		return err
+	}
+
+	for _, page := range pages {
+		switch page := page.(type) {
+		case *rbf.MetaPage:
+			cmd.printMetaPage(page)
+		case *rbf.RootRecordPage:
+			cmd.printRootRecordPage(page)
+		case *rbf.LeafPage:
+			cmd.printLeafPage(page)
+		case *rbf.BranchPage:
+			cmd.printBranchPage(page)
+		case *rbf.BitmapPage:
+			cmd.printBitmapPage(page)
+		case *rbf.FreePage:
+			cmd.printFreePage(page)
+		default:
+			return fmt.Errorf("unexpected page type %T", page)
+		}
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func (cmd *RBFPageCommand) printMetaPage(page *rbf.MetaPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: meta\n")
+	fmt.Fprintf(cmd.Stdout, "PageN: %d\n", page.PageN)
+	fmt.Fprintf(cmd.Stdout, "WALID: %d\n", page.WALID)
+	fmt.Fprintf(cmd.Stdout, "Root Record Pgno: %d\n", page.RootRecordPageNo)
+	fmt.Fprintf(cmd.Stdout, "Freelist Pgno: %d\n", page.FreelistPageNo)
+}
+
+func (cmd *RBFPageCommand) printRootRecordPage(page *rbf.RootRecordPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: root record\n")
+	fmt.Fprintf(cmd.Stdout, "Next: %d\n", page.Next)
+	fmt.Fprintf(cmd.Stdout, "Records: n=%d\n", len(page.Records))
+	for i, rec := range page.Records {
+		fmt.Fprintf(cmd.Stdout, "[%d]: name=%q pgno=%d\n", i, rec.Name, rec.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printLeafPage(page *rbf.LeafPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: leaf\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		if cell.Type == rbf.ContainerTypeBitmapPtr {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s pgno=%d\n", i, cell.Key, cell.Type, cell.Pgno)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s values=%v\n", i, cell.Key, cell.Type, cell.Values)
+		}
+	}
+}
+
+func (cmd *RBFPageCommand) printBranchPage(page *rbf.BranchPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: branch\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		fmt.Fprintf(cmd.Stdout, "[%d]: key=%d flags=%d pgno=%d\n", i, cell.Key, cell.Flags, cell.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printBitmapPage(page *rbf.BitmapPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: bitmap\n")
+	fmt.Fprintf(cmd.Stdout, "Values: %v\n", page.Values)
+}
+
+func (cmd *RBFPageCommand) printFreePage(page *rbf.FreePage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: free\n")
+}

ctl/rbf_pages.go

@@ -0,0 +1,149 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/txkey"
+)
+
+// RBFPagesCommand represents a command for printing a list of RBF page metadata.
+type RBFPagesCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Print the b-tree key with each row.
+	WithTree bool
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPagesCommand returns a new instance of RBFPagesCommand.
+func NewRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPagesCommand {
+	return &RBFPagesCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Iterate over each page and grab info.
+	infos, err := tx.PageInfos()
+	if err != nil {
+		return err
+	}
+
+	// Write header.
+	fmt.Fprint(cmd.Stdout, "ID       ")
+	fmt.Fprint(cmd.Stdout, "TYPE       ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "TREE                           ")
+	}
+	fmt.Fprintln(cmd.Stdout, "EXTRA")
+
+	fmt.Fprint(cmd.Stdout, "======== ")
+	fmt.Fprint(cmd.Stdout, "========== ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "============================== ")
+	}
+	fmt.Fprintln(cmd.Stdout, "====================")
+
+	// Print one line for each page.
+	for pgno, info := range infos {
+		switch info := info.(type) {
+		case *rbf.MetaPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "meta")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo)
+
+		case *rbf.RootRecordPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "rootrec")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "next=%d\n", info.Next)
+
+		case *rbf.LeafPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "leaf")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BranchPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "branch")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BitmapPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "bitmap")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		case *rbf.FreePageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "free")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		default:
+			panic(fmt.Sprintf("unexpected page info type %T", info))
+		}
+	}
+
+	return nil
+}
+
+func prefixToString(s string) (ret string) {
+	defer func() {
+		if err := recover(); err != nil {
+			ret = s
+		}
+	}()
+	return txkey.PrefixToString([]byte(s))
+}

ctl/server.go

@@ -15,72 +15,99 @@
 package ctl
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/storage"
 	"github.com/spf13/cobra"
 )
 
 // BuildServerFlags attaches a set of flags to the command for a server instance.
 func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
+	flags.StringVar(&srv.Config.Name, "name", srv.Config.Name, "Name of the node in the cluster.")
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
-	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
+	flags.StringVar(&srv.Config.AdvertiseGRPC, "advertise-grpc", srv.Config.AdvertiseGRPC, "Address to advertise externally for gRPC.")
+	flags.IntVar(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
 	flags.Uint64Var(&srv.Config.MaxMapCount, "max-map-count", srv.Config.MaxMapCount, "Limits the maximum number of active mmaps. Pilosa will fall back to reading files once this is exhausted. Set below your system's vm.max_map_count.")
 	flags.Uint64Var(&srv.Config.MaxFileCount, "max-file-count", srv.Config.MaxFileCount, "Soft limit on the maximum number of fragment files Pilosa keeps open simultaneously.")
+	flags.DurationVar((*time.Duration)(&srv.Config.LongQueryTime), "long-query-time", time.Duration(srv.Config.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
+	flags.IntVar(&srv.Config.QueryHistoryLength, "query-history-length", srv.Config.QueryHistoryLength, "Number of queries to remember in history.")
 
 	// TLS
-	SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
+	SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
 
 	// Handler
-	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).")
+	flags.StringSliceVar(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
 
 	// Cluster
-	flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
+	flags.IntVar(&srv.Config.Cluster.ReplicaN, "cluster.replicas", 1, "Number of hosts each piece of data should be stored on.")
-	flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
+	flags.DurationVar((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", time.Duration(srv.Config.Cluster.LongQueryTime), "RENAMED TO 'long-query-time': Duration that will trigger log and stat messages for slow queries.") // negative duration indicates invalid value because 0 is meaningful
-	flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
+	flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.")
-	flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
 
 	// Translation
-	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
+	flags.StringVar(&srv.Config.Translation.PrimaryURL, "translation.primary-url", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
-	flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
+	flags.IntVar(&srv.Config.Translation.MapSize, "translation.map-size", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
 
-	// Gossip
+	// Etcd
-	flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
+	// Etcd.Name used Config.Name for its value.
-	flags.StringVarP(&srv.Config.Gossip.AdvertiseHost, "gossip.advertise-host", "", srv.Config.Gossip.AdvertiseHost, "Host on which memberlist should advertise.")
+	// Etcd.Dir defaults to a directory under the pilosa data directory.
-	flags.StringVarP(&srv.Config.Gossip.AdvertisePort, "gossip.advertise-port", "", srv.Config.Gossip.AdvertisePort, "Port on which memberlist should advertise.")
+	// Etcd.ClusterName uses Cluster.Name for its value
+	flags.StringVar(&srv.Config.Etcd.LClientURL, "etcd.listen-client-address", srv.Config.Etcd.LClientURL, "Listen client address.")
+	flags.StringVar(&srv.Config.Etcd.AClientURL, "etcd.advertise-client-address", srv.Config.Etcd.AClientURL, "Advertise client address. If not provided, uses the listen client address.")
+	flags.StringVar(&srv.Config.Etcd.LPeerURL, "etcd.listen-peer-address", srv.Config.Etcd.LPeerURL, "Listen peer address.")
+	flags.StringVar(&srv.Config.Etcd.APeerURL, "etcd.advertise-peer-address", srv.Config.Etcd.APeerURL, "Advertise peer address. If not provided, uses the listen peer address.")
+	flags.StringVar(&srv.Config.Etcd.ClusterURL, "etcd.cluster-url", srv.Config.Etcd.ClusterURL, "Cluster URL to join.")
+	flags.StringVar(&srv.Config.Etcd.InitCluster, "etcd.initial-cluster", srv.Config.Etcd.InitCluster, "Initial cluster name1=apurl1,name2=apurl2")
 
-	flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
+	// External postgres database for ExternalLookup
-	flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
+	flags.StringVar(&srv.Config.LookupDBDSN, "lookup-db-dsn", "", "external (postgres) database DSN to use for ExternalLookup calls")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
-	flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
-	flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
 
 	// AntiEntropy
-	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
+	flags.DurationVar((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
+	flags.StringVar(&srv.Config.Metric.Service, "metric.service", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
-	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
+	flags.StringVar(&srv.Config.Metric.Host, "metric.host", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
+	flags.DurationVar((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
-	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
+	flags.BoolVar((&srv.Config.Metric.Diagnostics), "metric.diagnostics", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
-	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
+	flags.StringVar(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
+	flags.StringVar(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
-	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
+	flags.Float64Var(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling
 	flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
 	flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
+
+	// Storage
+	// Note: the default for --storage.backend must be kept "" empty string.
+	// Otherwise we cannot detect and honor the PILOSA_STORAGE_BACKEND env var
+	// over-ride.
+	// TODO: the comment above was carried over from the PILOSA_TXSRC flag, but
+	// we should confirm that this still applies.
+	flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
+	flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk")
+
+	// RowcacheOn
+	flags.BoolVar((&srv.Config.RowcacheOn), "rowcache-on", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)")
+
+	// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.
+	srv.Config.RBFConfig.DefineFlags(flags)
+
+	// Postgres endpoint
+	flags.StringVar(&srv.Config.Postgres.Bind, "postgres.bind", srv.Config.Postgres.Bind, "Address to which to bind a postgres endpoint (leave blank to disable)")
+	SetTLSConfig(flags, "postgres.", &srv.Config.Postgres.TLS.CertificatePath, &srv.Config.Postgres.TLS.CertificateKeyPath, &srv.Config.Postgres.TLS.CACertPath, &srv.Config.Postgres.TLS.SkipVerify, &srv.Config.Postgres.TLS.EnableClientVerification)
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.StartupTimeout), "postgres.startup-timeout", time.Duration(srv.Config.Postgres.StartupTimeout), "Timeout for postgres connection startup. (set 0 to disable)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.ReadTimeout), "postgres.read-timeout", time.Duration(srv.Config.Postgres.ReadTimeout), "Timeout for reads on a postgres connection. (set 0 to disable; does not include connection idling)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
+	flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
+	flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
 }

dbshard_internal_test.go

@@ -0,0 +1,370 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/shardwidth"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	"github.com/pilosa/pilosa/v2/testhook"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+// Shard per db evaluation
+func TestShardPerDB_SetBit(t *testing.T) {
+	f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
+	_ = idx
+	defer f.Clean(t)
+
+	// Set bits on the fragment.
+	if _, err := f.setBit(tx, 120, 1); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 120, 6); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 121, 0); err != nil {
+		t.Fatal(err)
+	}
+	// should have two containers set in the fragment.
+
+	// Verify counts on rows.
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count: %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count: %d", n)
+	}
+
+	// commit the change, and verify it is still there
+	PanicOn(tx.Commit())
+
+	// Close and reopen the fragment & verify the data.
+	err := f.Reopen() // roaring data not being flushed? red on roaring
+	if err != nil {
+		t.Fatal(err)
+	}
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	defer tx.Rollback()
+
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	}
+}
+
+// test that we find all *local* shards
+func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
+	tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly")
+	PanicOn(err)
+	defer os.RemoveAll(tmpdir)
+
+	v2s := NewFieldView2Shards()
+	stdShardSet := newShardSet()
+	for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+		stdShardSet.add(shard)
+	}
+	for _, field := range []string{"f", "_exists"} {
+		v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+	}
+
+	for _, src := range []string{"roaring", "bolt", "rbf"} {
+		cfg := mustHolderConfig()
+		cfg.StorageConfig.Backend = src
+		holder := NewHolder(tmpdir, cfg)
+
+		index := "rick"
+		idx := makeSampleRoaringDir(t, tmpdir, index, src, 1, holder, v2s)
+		if idx == nil {
+			idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
+			PanicOn(err)
+		}
+		estd := "rick/fields/_exists/views/standard"
+		std := "rick/fields/f/views/standard"
+
+		shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
+		PanicOn(err)
+
+		for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+			if !shards[shard] {
+				panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+			}
+		}
+		if src == "roaring" {
+			// check estd too
+			shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
+			PanicOn(err)
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				if !shards[shard] {
+					panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+				}
+			}
+
+			// check GetSortedFieldViewList() and roaringGetFieldView2Shards()
+			vs, err := roaringGetFieldView2Shards(idx)
+			PanicOn(err)
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				PanicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+
+				for _, fv := range fvs {
+					if !vs.has(fv.Field, fv.View, shard) {
+						panic(fmt.Sprintf("vs did not contain fv='%#v' for shard %v", fv, shard))
+					}
+				}
+				tx.Rollback()
+			}
+		} else {
+			// non-roaring: rbf, bolt
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				PanicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+				tx.Rollback()
+			}
+		}
+		holder.Close()
+	}
+}
+
+// data for Test_DBPerShard_GetShardsForIndex
+//
+var sampleRoaringDirList = map[string]string{"roaring": `
+rick/fields/f/views/standard/fragments/215.cache
+rick/fields/f/views/standard/fragments/221.cache
+rick/fields/f/views/standard/fragments/223.cache
+rick/fields/f/views/standard/fragments/93.cache
+rick/fields/f/views/standard/fragments/217.cache
+rick/fields/f/views/standard/fragments/219.cache
+rick/fields/f/views/standard/fragments/217
+rick/fields/f/views/standard/fragments/219
+rick/fields/f/views/standard/fragments/215
+rick/fields/f/views/standard/fragments/221
+rick/fields/f/views/standard/fragments/223
+rick/fields/f/views/standard/fragments/93
+rick/fields/_exists/views/standard/fragments/221
+rick/fields/_exists/views/standard/fragments/215
+rick/fields/_exists/views/standard/fragments/217
+rick/fields/_exists/views/standard/fragments/93
+rick/fields/_exists/views/standard/fragments/219
+rick/fields/_exists/views/standard/fragments/223
+`,
+	"bolt": `
+rick/backends/backend-boltdb/shard.0093-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0215-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0217-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0219-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0221-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0223-bolt/bolt.db
+`,
+	"rbf": `
+rick/backends/backend-rbf/shard.0093-rbf
+rick/backends/backend-rbf/shard.0215-rbf
+rick/backends/backend-rbf/shard.0217-rbf
+rick/backends/backend-rbf/shard.0219-rbf
+rick/backends/backend-rbf/shard.0221-rbf
+rick/backends/backend-rbf/shard.0223-rbf
+`,
+}
+
+func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
+	shards := []uint64{0, 93, 215, 217, 219, 221, 223}
+	fns := strings.Split(sampleRoaringDirList[backend], "\n")
+	firstDone := false
+
+	for i, fn := range fns {
+		// This check is here because in sampleRoaringDirList, the first entry
+		// of each map value is a line feed, so the strings.Split() above
+		// results in a blank entry for the first item. This means that the
+		// slice of shards above has an initial entry "0" which is not used.
+		if fn == "" {
+			continue
+		}
+		var shard uint64
+		switch backend {
+		case "bolt", "rbf":
+			shard = shards[i]
+
+			idx = helperCreateDBShard(h, index, shard)
+
+			// first time only, we'll actually make all the shards at this point because
+			// view2shards has them all anyway.
+			if !firstDone {
+				firstDone = true
+				makeTxTestDBWithViewsShards(h, idx, view2shards)
+			}
+			continue
+		case "roaring":
+		default:
+			t.Fatalf("invalid backend: %s", backend)
+		}
+
+		path := root + sep + filepath.Dir(fn)
+		PanicOn(os.MkdirAll(path, 0755))
+		fd, err := os.Create(root + sep + fn)
+		PanicOn(err)
+		if minBytes > 0 {
+			_, err := fd.Write(make([]byte, minBytes))
+			PanicOn(err)
+		}
+		fd.Close()
+	}
+	return
+}
+
+func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	PanicOn(err)
+	// TODO: It's not clear that this is actually doing anything.
+	dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
+	PanicOn(err)
+	_ = dbs
+	return idx
+}
+
+// keep the ocd linter happy
+var _ = makeBolttestDB
+var _ = makeRBFtestDB
+
+func makeBolttestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+	w, _ := mustOpenEmptyBoltWrapper(path)
+	BoltMustSetBitvalue(w, "index", "field", "view", shard, i)
+	w.Close()
+}
+
+func makeRBFtestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+
+	db := rbf.NewDB(path, nil)
+	err := db.Open()
+	PanicOn(err)
+	defer db.Close()
+
+	tx, err := db.Begin(true)
+	PanicOn(err)
+
+	err = tx.CreateBitmap("x")
+	PanicOn(err)
+
+	_, err = tx.Add("x", i)
+	PanicOn(err)
+
+	err = tx.Commit()
+	PanicOn(err)
+}
+
+func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shards) {
+
+	// TODO(jea): need date time quantum views!!
+	batched := false
+	for field, viewmap := range exp.m {
+		for view, shset := range viewmap {
+
+			ss := shset.CloneMaybe()
+			for shard := range ss {
+
+				// simply write 1 bit to each shard to force its creation.
+				bits := []uint64{(shard << shardwidth.Exponent) + 1}
+				tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+				changeCount, err := tx.Add(idx.name, field, view, shard, batched, bits...)
+				PanicOn(err)
+				if changeCount != len(bits) {
+					panic(fmt.Sprintf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount))
+				}
+
+				PanicOn(tx.Commit())
+			}
+		}
+	}
+
+}
+
+// test that rbf can give us a map[view]*shardSet
+func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
+	tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetFieldView2Shards_map_from_RBF")
+	PanicOn(err)
+	defer os.RemoveAll(tmpdir)
+
+	cfg := mustHolderConfig()
+	cfg.StorageConfig.Backend = "rbf"
+	holder := NewHolder(tmpdir, cfg)
+	defer holder.Close()
+
+	index := "rick"
+	field := "f"
+
+	cim := &CreateIndexMessage{
+		Index:     index,
+		CreatedAt: 0,
+		Meta:      IndexOptions{},
+	}
+
+	idx, err := holder.createIndex(cim, false)
+	PanicOn(err)
+
+	exp := NewFieldView2Shards()
+
+	stdShardSet := newShardSet()
+	stdShardSet.add(12)
+	stdShardSet.add(15)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+
+	hrShardSet := newShardSet()
+	hrShardSet.add(7)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard_2019092416"}, hrShardSet)
+
+	makeTxTestDBWithViewsShards(holder, idx, exp)
+
+	// setup is done
+	view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx)
+	PanicOn(err)
+
+	// compare against setup
+	if !view2shard.equals(exp) {
+		panic(fmt.Sprintf("expected '%v' but got view2shard '%v'", exp, view2shard))
+	}
+}

dbshard_test.go

@@ -0,0 +1,112 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+	. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
+)
+
+func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+
+	t.Run("RowIDColumnKey", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "rick"
+		fieldName := "f"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+
+		rowID := uint64(1)
+		timestamp := int64(0)
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		for i := 1; i <= 10; i++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the primary and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
+		}
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+
+		//select {}
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("unexpected column keys: %#v", keys)
+		}
+	})
+
+}

debugstats/stats.go

@@ -0,0 +1,140 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debugstats
+
+import (
+	"fmt"
+	"math"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+type CallStats struct {
+	// protect elap
+	mu sync.Mutex
+
+	// track how much time each call took.
+	elap map[string]*elapsed
+}
+
+type elapsed struct {
+	dur []float64
+}
+
+func NewCallStats() *CallStats {
+	w := &CallStats{}
+	w.Reset()
+	return w
+}
+
+func (w *CallStats) Reset() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	w.elap = make(map[string]*elapsed)
+}
+
+type LineSorter struct {
+	Line string
+	Tot  float64
+}
+
+type SortByTot []*LineSorter
+
+func (p SortByTot) Len() int {
+	return len(p)
+}
+func (p SortByTot) Less(i, j int) bool {
+	return p[i].Tot < p[j].Tot
+}
+func (p SortByTot) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (c *CallStats) Report(title string) (r string) {
+	r = fmt.Sprintf("CallStats: (%v)\n", title)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var lines []*LineSorter
+	for id, elap := range c.elap {
+		slc := elap.dur
+		n := len(slc)
+		if n == 0 {
+			continue
+		}
+		mean, sd, totaltm := computeMeanSd(slc)
+		if n == 1 {
+			sd = 0
+			mean = slc[0]
+			totaltm = slc[0]
+		}
+		line := fmt.Sprintf("  %20v  N=%8v   avg/op: %12v   sd: %12v  total: %12v\n", id, n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm))
+		lines = append(lines, &LineSorter{Line: line, Tot: totaltm})
+	}
+	sort.Sort(SortByTot(lines))
+	for i := range lines {
+		r += lines[i].Line
+	}
+
+	if false {
+		var m1 runtime.MemStats
+		runtime.ReadMemStats(&m1)
+		r += fmt.Sprintf("\n m1.TotalAlloc = %v\n", m1.TotalAlloc)
+	}
+
+	return
+}
+
+var NaN = math.NaN()
+
+func computeMeanSd(slc []float64) (mean, sd, tot float64) {
+	if len(slc) < 2 {
+		return NaN, NaN, NaN
+	}
+	for _, v := range slc {
+		tot += v
+	}
+	n := float64(len(slc))
+	mean = tot / n
+
+	variance := 0.0
+	for _, v := range slc {
+		tmp := (v - mean)
+		variance += tmp * tmp
+	}
+	variance = variance / n // biased, but we don't care b/c we can have very small n
+	sd = math.Sqrt(variance)
+	if sd < 1e-8 {
+		// sd is super close to zero, NaN out the z-score rather than +/- Inf
+		sd = NaN
+	}
+	return
+}
+
+func (c *CallStats) Add(k string, dur time.Duration) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.elap[k]
+	if !ok {
+		e = &elapsed{}
+		c.elap[k] = e
+	}
+	e.dur = append(e.dur, float64(dur))
+}