Merge pull request #1529 from travisturner/cache-bitdepth

[Core-370] Cache BitDepth on bsiGroup during index.Open

.circleci/config.yml

@@ -1,195 +1,270 @@
-version: 2
-defaults: &defaults
-  working_directory: /go/src/github.com/pilosa/pilosa
-  docker:
-    - image: circleci/golang:1.13
-  environment:
-    GO111MODULE: "on"
-fast-checkout: &fast-checkout
-  attach_workspace:
-    at: .
-jobs:
-  setup:
-    <<: *defaults
+version: 2.1
+
+executors:
+  golang:
+    parameters:
+      version:
+        type: string
+        default: "1.15.8"
+      resource_class:
+        type: string
+        default: medium
+    docker:
+      - image: circleci/golang:<< parameters.version >>
+    resource_class: << parameters.resource_class >>
+    working_directory: /go/src/github.com/pilosa/pilosa
+
+commands:
+  add-github-auth:
+    steps:
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
+      - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
+  restore-mod-cache:
     steps:
-      - checkout
       - restore_cache:
-          keys:
-            - mod-cache-{{ checksum "go.sum" }}
-      - run: "go mod download"
+          key: mod-cache-{{ checksum "go.sum" }}
+  save-mod-cache:
+    steps:
       - save_cache:
           key: mod-cache-{{ checksum "go.sum" }}
           paths:
             - /go/pkg/mod/
-      - persist_to_workspace:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
+  checkout-plus:
     steps:
-      - *fast-checkout
+      - add-github-auth
+      - checkout
+      - restore-mod-cache
+
+jobs:
+  setup:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod download
+      - save-mod-cache
+  check-license-headers:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
+      - checkout-plus
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
       - run: make golangci-lint
-  test-build-arm:
-    <<: *defaults
+  go-mod-tidy:
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - run: go mod tidy
+      - run: git diff --exit-code -- go.mod go.sum
+  check-changelog-label:
+    executor:
+      name: golang
+    steps:
+      - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
+      - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
+  test-build-arm:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
-    <<: *defaults
+  test:
+    parameters:
+      resource_class:
+        type: string
+        default: medium
+      golang_version:
+        type: string
+        default: "1.15.8"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
+      resource_class: << parameters.resource_class >>
     steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
+      - checkout-plus
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - setup_remote_docker
       - run: make clustertests-build
-  prerelease:
-    <<: *base-test
-    steps:
-      - *fast-checkout
-      - run: make prerelease
-      - store_artifacts:
-          path: build
-      - persist_to_workspace:
-          root: .
-          paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
-      - run: make release
+      - checkout-plus
+      - attach_workspace:
+          at: .
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker-release
       - store_artifacts:
           path: build
       - persist_to_workspace:
           root: .
           paths: build
-  prerelease-upload:
-    docker:
-      - image: circleci/python:2.7-jessie
+  publish_release:
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
-      - run: sudo pip install awscli
-      - run: make prerelease-upload
-  dockerhub-upload:
-    <<: *defaults
+      - attach_workspace:
+          at: .
+      - run: go get github.com/tcnksm/ghr
+      - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
+  docker-build:
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
-      - setup_remote_docker
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker GO_VERSION=1.15.8
+      - run: docker run pilosa:$(git describe --tags) help
+  dockerhub-upload-unstable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
-      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
+      - run: docker run pilosa:$(git describe --tags) help
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker:
+          version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+      - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+      - run: make docker
+      - run: docker run pilosa:$(git describe --tags) help
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
-  version: 2
-  test:
+  build:
     jobs:
-      - setup
+      - setup:
+          context: molecula
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
+          context: molecula
           requires:
             - setup
       - check-license-headers:
+          context: molecula
+          requires:
+            - setup
+      - go-mod-tidy:
+          context: molecula
+          requires:
+            - setup
+      - check-changelog-label:
+          context: molecula
           requires:
             - setup
       - test-build-arm:
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          resource_class: large
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-race:
+          matrix:
+            parameters:
+              golang_version: ["1.14.15", "1.15.8"]
+      - test:
+          name: << matrix.test_make_target >>
+          resource_class: xlarge
+          context: molecula
           requires:
             - setup
-      - test-golang-1.13-386:
-          requires:
-            - setup
-      - test-golang-1.13:
-          requires:
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
+          matrix:
+            parameters:
+              test_make_target: ["test-race", "test-txstore-rbf_bolt"]
+      - test:
+          name: test-shardwidth-22
+          context: molecula
+          shard_width: "22"
+          resource_class: large
           requires:
             - setup
       - cluster-tests:
+          context: molecula
           requires:
             - setup
-      - prerelease:
+      - docker-build:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
       - release:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              only: master
+      - publish_release:
+          context: molecula
+          requires:
+            - release
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - prerelease
-      - dockerhub-upload:
+            - setup
+          filters:
+            branches:
+              only: master
+      - dockerhub-upload-stable:
+          context: molecula
           requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+            - setup
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

.dockerignore

@@ -0,0 +1,5 @@
+lattice/.git
+lattice/node_modules
+lattice/build
+statik/statik.go
+build

.gitignore

@@ -4,3 +4,9 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*
+pilosa
+*.dot

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lattice"]
+	path = lattice
+	url = git@github.com:molecula/lattice.git

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,16 +1,46 @@
-FROM golang:1.13.0 as builder
+ARG GO_VERSION=latest
 
-COPY . pilosa
+#######################
+### Lattice builder ###
+#######################
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+FROM moleculacorp/nodejs:latest as lattice-builder
+WORKDIR /lattice
 
-FROM alpine:3.9.4
+COPY lattice/package.json ./
+COPY lattice/yarn.lock ./
+RUN yarn install
 
-LABEL maintainer "dev@pilosa.com"
+COPY lattice ./
+RUN yarn build
+
+######################
+### Pilosa builder ###
+######################
+
+FROM golang:${GO_VERSION} as pilosa-builder
+ARG MAKE_FLAGS
+WORKDIR /pilosa
+
+RUN go get github.com/rakyll/statik
+
+COPY . ./
+COPY --from=lattice-builder /lattice/build /lattice
+RUN /go/bin/statik -src=/lattice -dest=/pilosa
+
+RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
+
+#####################
+### Pilosa runner ###
+#####################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=builder /go/bin/pilosa /pilosa
+COPY --from=pilosa-builder /pilosa/build/pilosa /
 
 COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
@@ -18,5 +48,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
+    && make install FLAGS="-a -mod=vendor"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,23 +1,38 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race  test-txstore-rbf lattice
 
 CLONE_URL=github.com/pilosa/pilosa
+MOD_VERSION=v2
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
-BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
+VARIANT = Molecula
+GO=go
+GOOS=$(shell $(GO) env GOOS)
+GOARCH=$(shell $(GO) env GOARCH)
+VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
-GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
-RELEASE ?= 0
-RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
-BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)"
+GO_VERSION=1.15.8
+DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+TEST_TAGS = roaringparanoia
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+UNAME := $(shell uname -s)
+ifeq ($(UNAME), Darwin)
+    IS_MACOS:=1
+else
+    IS_MACOS:=0
+endif
+
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export CGO_ENABLED=0
 
 # Run tests and compile Pilosa
 default: test build
@@ -28,14 +43,48 @@ clean:
 
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
-	go mod vendor
+	$(GO) mod vendor
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v
+
+# Run test suite with race flag
+test-race:
+	CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v
+
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
+tour:
+	./tournament.sh
 
 bench:
-	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+	$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 
 # Run test suite with coverage enabled
 cover:
@@ -44,17 +93,16 @@ cover:
 
 # Run test suite with coverage enabled and view coverage results in browser
 cover-viz: cover
-	go tool cover -html=build/coverage.out
+	$(GO) tool cover -html=build/coverage.out
 
 # Compile Pilosa
 build:
-	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
 
 # Create a single release build under the build directory
 release-build:
-	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
+	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
+	cp NOTICE README.md LICENSE build/pilosa$(VERSION_ID)
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -64,15 +112,16 @@ ifndef SKIP_CHECK_CLEAN
 	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
 endif
 
-# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
-release: check-clean
+# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
+release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
-	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
+# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
+release-sans-ui: check-clean
+	rm -f statik/statik.go
+	$(MAKE) release-build GOOS=darwin GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=amd64
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
 DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
@@ -81,59 +130,143 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
-	docker-compose -f $(DOCKER_COMPOSE) down
+clustertests-build: vendor
+	docker-compose -f $(DOCKER_COMPOSE) down -v
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
-# Create prerelease builds
-prerelease:
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
-	$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
-
-prerelease-upload:
-	aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
-
 # Install Pilosa
 install:
-	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+
+install-bench:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
+
+# Ensure lattice is cloned and the pinned version is checked out
+lattice:
+	git submodule update --init
+
+# Build the lattice assets
+build-lattice: lattice
+	docker build -t lattice:build ./lattice
+	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
+
+# Upgrade lattice to the latest version
+upgrade-lattice: lattice
+	git submodule update --remote
 
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
-	go generate github.com/pilosa/pilosa/v2/internal
+	$(GO) generate github.com/pilosa/pilosa/v2/internal
+
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	$(GO) generate github.com/pilosa/pilosa/v2/statik
+
+# `go generate` statik assets (lattice UI) in Docker
+generate-statik-docker: build-lattice
+	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
 
 # `go generate` stringers
 generate-stringer:
-	go generate github.com/pilosa/pilosa/v2
+	$(GO) generate github.com/pilosa/pilosa/v2
 
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+generate-proto-grpc: require-protoc require-protoc-gen-go
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
+	# TODO: Modify above commands and remove the below mv if possible.
+	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
+	# I couldn't get it to work during development - Cody
+	cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/
+	rm -rf proto/github.com
+
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
+
+# Create release using Docker
+docker-release:
+	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
+
+# Build a release in Docker
+docker-build: vendor lattice
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --target pilosa-builder \
+	    --tag pilosa:build .
+	docker create --name pilosa-build pilosa:build
+	mkdir -p build/pilosa-$(VERSION_ID)
+	docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID)
+	cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID)
+	docker rm pilosa-build
+	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 
 # Create Docker image from Dockerfile
-docker:
-	docker build -t "pilosa:$(VERSION)" .
+docker-image: vendor lattice
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
+	    --tag pilosa:$(VERSION) .
 	@echo Created docker image: pilosa:$(VERSION)
 
-# Compile Pilosa inside Docker container
-docker-build:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Create docker image (alias)
+docker: docker-image # alias
+
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
+pilosa-keydump:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
+
+# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
+pilosa-chk:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
+
+pilosa-fsck:
+	cd ./cmd/pilosa-fsck && make install && make release
 
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
+
+# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
+# run top tests, not subdirs. print summary red/green after.
+# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
+topt:
+	mv log.topt.roar log.topt.roar.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
+	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
+	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
+
+topt-race:
+	mv log.topt.race log.topt.race.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
+	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
+	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +294,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
 
 ######################
 # Build dependencies #
@@ -175,24 +307,37 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go get -u github.com/rakyll/statik
 
 install-stringer:
-	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
+	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
 
 install-protoc-gen-gofast:
-	GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
+	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc-gen-go:
+	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
 
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 
 install-peg:
-	GO111MODULE=off go get github.com/pointlander/peg
+	GO111MODULE=off $(GO) get github.com/pointlander/peg
 
 install-golangci-lint:
-	GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
+	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 
 install-gometalinter:
-	GO111MODULE=off go get -u github.com/alecthomas/gometalinter
+	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
-	GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
+	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
+
+test-txstore-rbf:
+	PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race
+
+test-txstore-rbf_bolt:
+	PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race
+

NOTICE

@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
 Third-party software licenses
 =============================
 
@@ -115,3 +94,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
 
 1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
 
+Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101).
+
 2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
 
     ```shell
@@ -40,7 +42,7 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
     and verify that it's running:
     
     ```shell
-    curl localhost:10101/nodes
+    curl localhost:10101/status
     ```
 
 3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
@@ -56,6 +58,15 @@ Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) w
 You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
 
 
+## Upgrading UI
+
+Lattice is now a submodule of Pilosa, for the purpose of keeping the frontend and backend components of the UI system synchronized. When making a change to the UI that requires changing both the backend and frontend, follow steps in this order:
+
+1. merge frontend (Lattice) PR
+2. run `make upgrade-lattice`
+3. run `git add lattice`, `git commit -m"Upgrade Lattice"`, `git push`
+4. merge backend (Pilosa) PR
+
 ## Client Libraries
 
 There are supported libraries for the following languages:
@@ -63,13 +74,12 @@ There are supported libraries for the following languages:
 - [Java](https://www.pilosa.com/docs/client-libraries/#java)
 - [Python](https://www.pilosa.com/docs/client-libraries/#python)
 
-## Licenses
+## License
 
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
+Pilosa is licensed under the Apache License, Version 2.0.
+
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
 
 ## Get Support
 

api/client/grpc.go

@@ -0,0 +1,214 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -16,9 +16,11 @@ package pilosa_test
 
 import (
 	"context"
+	"crypto/rand"
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +32,139 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "i"
+		fieldName := "f"
+		attrKey := "k"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, indexName, fieldName)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs0,
+			AttrVals:       attrVals0,
+			Shard:          0,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs1,
+			AttrVals:       attrVals1,
+			Shard:          1,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		m := len(res.ColumnAttrSets)
+		if m != 100 {
+			t.Fatalf("incorrect number of column attrs set; m = %v", m)
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -49,62 +184,79 @@ func TestAPI_Import(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "rick"
-		field := "f"
+		indexName := "rick"
+		fieldName := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
 
 		rowID := uint64(1)
 		timestamp := int64(0)
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
-		for i := 1; i <= 10; i++ {
+		N := 10
+		for i := 1; i <= N; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
-		// Import data with keys to the coordinator (node0) and verify that it gets
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		colKeys = colKeys[:N]
+
+		// Import data with keys to the primary and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowIDs:     rowIDs,
-			ColumnKeys: colKeys,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making lots of other shards? b/c this is not a restriction.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
 		}
 
-		pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
+		qcx := m0.API.Txf().NewQcx()
+
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 			t.Fatal(err)
 		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
-			t.Fatalf("unexpected column keys: %#v", keys)
+			t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
 		}
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 				return err
 			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 				return fmt.Errorf("unexpected column keys: %#v", keys)
@@ -118,7 +270,11 @@ func TestAPI_Import(t *testing.T) {
 	// Relies on the previous test creating an index with TrackExistence and
 	// adding some data.
 	t.Run("SchemaHasNoExists", func(t *testing.T) {
-		schema := m1.API.Schema(context.Background())
+		schema, err := m1.API.Schema(context.Background(), false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		for _, f := range schema[0].Fields {
 			if f.Name == "_exists" {
 				t.Fatalf("found _exists field in schema")
@@ -127,59 +283,6 @@ func TestAPI_Import(t *testing.T) {
 				t.Fatalf("found internal field '%s' in schema output", f.Name)
 			}
 		}
-
-	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
 	})
 }
 
@@ -200,32 +303,34 @@ func TestAPI_ImportValue(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	coord := c.GetPrimary()
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
 		index := "valck"
 		field := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
+		_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		_, err = coord.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
-		// Import data with keys to the coordinator (node0) and verify that it gets
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the primary and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
 			Index:      index,
@@ -233,9 +338,12 @@ func TestAPI_ImportValue(t *testing.T) {
 			ColumnKeys: colKeys,
 			Values:     values,
 		}
-		if err := m0.API.ImportValue(ctx, req); err != nil {
+
+		qcx := coord.API.Txf().NewQcx()
+		if err := coord.API.ImportValue(ctx, qcx, req); err != nil {
 			t.Fatal(err)
 		}
+		panicOn(qcx.Finish())
 
 		pql := fmt.Sprintf("Row(%s>0)", field)
 
@@ -258,6 +366,126 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to node1 and verify that it gets translated and
+		// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+
+		qcx := m1.API.Txf().NewQcx()
+		if err := m1.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		query := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: observerd %+v;  expected '%+v'", ids, colIDs[6:])
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = coord.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = coord.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the node0 and verify that it gets translated
+		// and forwarded to the owner of shard 0 (node1; because of
+		// offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node1.
+		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: observerd %+v;  expected '%+v'", ids, []uint64{1})
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.
@@ -266,3 +494,178 @@ type offsetModHasher struct{}
 func (*offsetModHasher) Hash(key uint64, n int) int {
 	return int(key+1) % n
 }
+
+func (*offsetModHasher) Name() string { return "mod" }
+
+func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	// plan:
+	//  1. set a bit
+	//  2. clear with Import() using the ImportRequest.Clear flag
+	//  3. verifiy the clear is done.
+	//  repeat for ImportValueRequest and ImportValues()
+
+	m0 := c.GetNode(0)
+	m0api := m0.API
+
+	ctx := context.Background()
+	index := "i"
+	fieldAcct0 := "acct0"
+
+	opts := pilosa.OptFieldTypeInt(-1000, 1000)
+
+	_, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = m0api.CreateField(ctx, index, fieldAcct0, opts)
+	if err != nil {
+		t.Fatalf("creating fieldAcct0: %v", err)
+	}
+
+	iraField := "ira" // set field.
+	iraRowID := uint64(3)
+	_, err = m0api.CreateField(ctx, index, iraField)
+	if err != nil {
+		t.Fatalf("creating fieldIRA: %v", err)
+	}
+
+	acctOwnerID := uint64(78) // ColumnID
+	shard := acctOwnerID / ShardWidth
+	acct0bal := int64(500)
+
+	ivr0 := &pilosa.ImportValueRequest{
+		Index:     index,
+		Field:     fieldAcct0,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		Values:    []int64{acct0bal},
+	}
+	ir0 := &pilosa.ImportRequest{
+		Index:     index,
+		Field:     iraField,
+		Shard:     shard,
+		ColumnIDs: []uint64{acctOwnerID},
+		RowIDs:    []uint64{iraRowID},
+	}
+
+	qcx := m0api.Txf().NewQcx()
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bitIsSet := func() bool {
+		query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+		cols := res.Results[0].(*pilosa.Row).Columns()
+		for i := range cols {
+			if cols[i] == acctOwnerID {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !bitIsSet() {
+		panic("IRA bit should have been set")
+	}
+
+	queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
+		query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
+		res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+		panicOn(err)
+
+		if len(res.Results) == 0 {
+			return 0
+		}
+		valCount := res.Results[0].(pilosa.ValCount)
+		return valCount.Val
+	}
+
+	bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+
+	if bal != acct0bal {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
+	}
+
+	// clear the bit
+	qcx = m0api.Txf().NewQcx()
+	ir0.Clear = true
+	if err := m0api.Import(ctx, qcx, ir0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	if bitIsSet() {
+		panic("IRA bit should have been cleared")
+	}
+
+	// clear the BSI
+	qcx = m0api.Txf().NewQcx()
+	ivr0.Clear = true
+	if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
+		t.Fatal(err)
+	}
+	panicOn(qcx.Finish())
+
+	bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
+	if bal != 0 {
+		panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
+	}
+}
+
+func TestAPI_IDAlloc(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	primary := c.GetPrimary().API
+
+	key := pilosa.IDAllocKey{
+		Index: "index",
+		Key:   "key",
+	}
+	var session [32]byte
+	_, err := rand.Read(session[:])
+	if err != nil {
+		t.Fatalf("obtaining random bytes: %v", err)
+	}
+
+	const toReserve = 2
+
+	ids, err := primary.ReserveIDs(key, session, ^uint64(0), toReserve)
+	if err != nil {
+		t.Fatalf("reserving IDs: %v", err)
+	}
+
+	var numIds uint64
+	for _, idr := range ids {
+		numIds += (idr.Last - idr.First) + 1
+	}
+	if numIds != toReserve {
+		t.Errorf("expected %d ids but got %d: %v", toReserve, numIds, ids)
+	}
+
+	err = primary.CommitIDs(key, session, numIds)
+	if err != nil {
+		t.Fatalf("committing IDs: %v", err)
+	}
+
+	err = primary.ResetIDAlloc(key.Index)
+	if err != nil {
+		t.Fatalf("resetting ID alloc: %v", err)
+	}
+}

apimethod_string.go

@@ -19,25 +19,36 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
-	_ = x[apiFieldAttrDiff-12]
-	_ = x[apiImport-13]
-	_ = x[apiImportValue-14]
-	_ = x[apiIndex-15]
-	_ = x[apiIndexAttrDiff-16]
-	_ = x[apiQuery-17]
-	_ = x[apiRecalculateCaches-18]
-	_ = x[apiRemoveNode-19]
-	_ = x[apiResizeAbort-20]
-	_ = x[apiSetCoordinator-21]
-	_ = x[apiShardNodes-22]
-	_ = x[apiViews-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiTranslateData-11]
+	_ = x[apiField-12]
+	_ = x[apiFieldAttrDiff-13]
+	_ = x[apiImport-14]
+	_ = x[apiImportValue-15]
+	_ = x[apiIndex-16]
+	_ = x[apiIndexAttrDiff-17]
+	_ = x[apiQuery-18]
+	_ = x[apiRecalculateCaches-19]
+	_ = x[apiRemoveNode-20]
+	_ = x[apiResizeAbort-21]
+	_ = x[apiSchema-22]
+	_ = x[apiShardNodes-23]
+	_ = x[apiState-24]
+	_ = x[apiViews-25]
+	_ = x[apiApplySchema-26]
+	_ = x[apiStartTransaction-27]
+	_ = x[apiFinishTransaction-28]
+	_ = x[apiTransactions-29]
+	_ = x[apiGetTransaction-30]
+	_ = x[apiActiveQueries-31]
+	_ = x[apiPastQueries-32]
+	_ = x[apiIDReserve-33]
+	_ = x[apiIDCommit-34]
+	_ = x[apiIDReset-35]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 324, 337, 345, 353, 367, 386, 406, 421, 438, 454, 468, 480, 491, 501}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

attr_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"os"
 	"reflect"
 	"runtime"
@@ -24,11 +23,12 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure database can set and retrieve column attributes.
 func TestAttrStore_Attrs(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -57,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) {
 
 // Ensure database returns a non-nil empty map if unset.
 func TestAttrStore_Attrs_Empty(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	if m, err := s.Attrs(100); err != nil {
@@ -69,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
 
 // Ensure database can unset attributes if explicitly set to nil.
 func TestAttrStore_Attrs_Unset(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -89,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
 
 // Ensure attribute block checksums can be returned.
 func TestAttrStore_Blocks(t *testing.T) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(t)
 	defer s.Close()
 
 	// Set attributes.
@@ -135,19 +135,22 @@ type AttrStore struct {
 }
 
 // NewAttrStore returns a new instance of AttrStore.
-func NewAttrStore(string) pilosa.AttrStore {
-	f, err := ioutil.TempFile("", "pilosa-attr-")
+func NewAttrStore(tb testing.TB) pilosa.AttrStore {
+	f, err := testhook.TempFile(tb, "pilosa-attr-")
 	if err != nil {
 		panic(err)
 	}
+	// Note, even though the file is closed, TempFile will still avoid
+	// creating the same name again if we leave it existing. The boltdb
+	// code may already be deleting this, so the TestHook deletion
+	// may not matter but it's more reliable this way.
 	f.Close()
-	os.Remove(f.Name())
 
 	return &AttrStore{boltdb.NewAttrStore(f.Name())}
 }
 
 func BenchmarkAttrStore_Duplicate(b *testing.B) {
-	s := MustOpenAttrStore()
+	s := MustOpenAttrStore(b)
 	defer s.Close()
 
 	// Set attributes.
@@ -186,8 +189,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) {
 }
 
 // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
-func MustOpenAttrStore() pilosa.AttrStore {
-	s := NewAttrStore("")
+func MustOpenAttrStore(tb testing.TB) pilosa.AttrStore {
+	s := NewAttrStore(tb)
 	if err := s.Open(); err != nil {
 		panic(err)
 	}

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

barrier.go

@@ -0,0 +1,219 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"github.com/glycerine/idem"
+)
+
+// Barrier allows us to temporarily halt all readers, so that
+// a writer can commit alone and thus compact the db.
+// The Barrier starts unblocked, alllowing passage to any
+// caller of WaitAtGate().
+type Barrier struct {
+	wait       chan *appointment // send upon entering the waiting room.
+	halt       *idem.Halter
+	blockReqCh chan *blockReq
+	unblockCh  chan *unblock
+}
+
+type blockReq struct {
+	count int
+	done  chan struct{}
+}
+
+func newBlockReq(count int) *blockReq {
+	return &blockReq{
+		count: count,
+		done:  make(chan struct{}),
+	}
+}
+
+type appointment struct {
+	id   int
+	done chan struct{}
+}
+
+func newAppointment(id int) *appointment {
+	return &appointment{
+		id:   id,
+		done: make(chan struct{}),
+	}
+}
+
+// NewBarrier is either open, allowing immediate passage,
+// or blocked, halting all callers at WaitAtGate()
+// until the barrier is opened. By default it is open.
+//
+// Barrier.Close() must be called when the barrier
+// is no longer needed to avoid a goroutine leak.
+func NewBarrier() (b *Barrier) {
+	b = &Barrier{
+		wait:       make(chan *appointment), // waiters indicate they are waiting for the gate by sending here.
+		halt:       idem.NewHalter(),
+		blockReqCh: make(chan *blockReq),
+		unblockCh:  make(chan *unblock),
+	}
+	go func() {
+		defer b.halt.Done.Close()
+
+		var waitlist []*appointment
+		var curBlockReq *blockReq
+
+		for {
+			select {
+			case br := <-b.blockReqCh:
+				if br.count == 0 {
+					close(br.done)
+					continue
+				}
+				if curBlockReq == nil {
+					// good, changing state from open to closed barrier.
+				} else {
+					panic("got 2nd block request atop of first")
+				}
+				curBlockReq = br
+				//vv("barrier: request to block for %v waiters", br.count)
+				if len(waitlist) != 0 {
+					panic("had waiters when we were open, internal/client bug")
+				}
+			case appt := <-b.wait:
+				//vv("barrier.wait sees appt = '%#v' and  curBlockReq = '%#v'", appt, curBlockReq)
+				if curBlockReq == nil {
+					close(appt.done)
+					continue
+				}
+				waitlist = append(waitlist, appt)
+				n := len(waitlist)
+				th := curBlockReq.count
+				if th < 0 {
+					// infinite waiters. we block everybody until we
+					// see an unblock request.
+					continue
+				}
+				if n >= th {
+					close(curBlockReq.done)
+					curBlockReq = nil
+				}
+			case ub := <-b.unblockCh:
+				for _, appt := range waitlist {
+					close(appt.done)
+				}
+				waitlist = nil
+				curBlockReq = nil
+				close(ub.done)
+			case <-b.halt.ReqStop.Chan:
+				return
+			}
+		}
+	}()
+	return
+}
+
+// WaitAtGate will return immediately
+// if the barrier is unblocked. Otherwise
+// it will not return until another
+// goroutine unblocks the barrier.
+func (b *Barrier) WaitAtGate(id int) {
+	appt := newAppointment(id)
+	select {
+	case b.wait <- appt:
+		select {
+		case <-appt.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// Close should be called to stop the
+// barrier's background goroutine when
+// you are done using the barrier.
+func (b *Barrier) Close() {
+	b.halt.ReqStop.Close()
+	<-b.halt.Done.Chan
+}
+
+type unblock struct {
+	done chan struct{}
+}
+
+func newUnblock() *unblock {
+	return &unblock{
+		done: make(chan struct{}),
+	}
+}
+
+// Unblock lets all waiting goroutines resume execution.
+func (b *Barrier) UnblockReaders() {
+	ub := newUnblock()
+	select {
+	case b.unblockCh <- ub:
+		select {
+		case <-ub.done:
+		case <-b.halt.ReqStop.Chan:
+		}
+	case <-b.halt.ReqStop.Chan:
+	}
+}
+
+// BlockUntil is called with a count, the
+// number of waiters required to be present and waiting
+// at the gate before call returns.
+// A count of < 0 will return immediately and raise
+// the barrier to any number of arriving readers.
+// A count of 0 is a no-op.
+//
+// Otherwise we raise the barrier
+// and wait until we have seen count other goroutines waiting
+// on it.
+//
+// We return without releasing the waiters. Call
+// Open when you want them to resume.
+func (b *Barrier) BlockUntil(count int) {
+	if count == 0 {
+		return
+	}
+	req := newBlockReq(count)
+	b.blockReqCh <- req
+	if count > 0 {
+		<-req.done
+	}
+}
+
+// BlockAllReadersNoWait raises the barrier to
+// an infinite number of waiters and returns immediately
+// to the caller.
+func (b *Barrier) BlockAllReadersNoWait() {
+	req := newBlockReq(-1) // -1 means block any number of readers.
+	b.blockReqCh <- req
+	// don't wait. <-req.done
+}

barrier_test.go

@@ -0,0 +1,90 @@
+// home https://github.com/glycerine/lmdb-go
+// Copyright (c) 2020, the lmdb-go authors
+// Copyright (c) 2015, Bryan Matsuo
+// All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+
+//     Redistributions of source code must retain the above copyright notice, this
+//     list of conditions and the following disclaimer.
+
+//     Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+
+//     Neither the name of the author nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without specific
+//     prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pilosa
+
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestBarrierHolds(t *testing.T) {
+	b := NewBarrier()
+	defer b.Close()
+
+	released := int64(0)
+
+	waiter := func(i int) {
+		b.WaitAtGate(i)
+		//vv("goro %v is released", i)
+		atomic.AddInt64(&released, 1)
+	}
+
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+	time.Sleep(time.Second)
+	r := atomic.SwapInt64(&released, 0)
+	if r != 3 {
+		panic("open barrier held back goro")
+	}
+	//vv("good: barrier started open")
+
+	//seenAll := make(chan bool)
+	b.BlockAllReadersNoWait()
+	for i := 0; i < 3; i++ {
+		go waiter(i)
+	}
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic("bad: barrier did not hold back goro")
+	}
+	//vv("good: barrier of 4 did not release on 3")
+	go waiter(4)
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 0 {
+		panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r))
+	}
+
+	b.UnblockReaders()
+
+	time.Sleep(time.Second)
+	r = atomic.SwapInt64(&released, 0)
+	if r != 4 {
+		panic("bad: unblock should have released 4 goro")
+	}
+
+}

bluegreentx_test.go

@@ -0,0 +1,98 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	cryrand "crypto/rand"
+)
+
+var _ = context.Background
+var _ = os.Open
+var _ = strings.Split
+
+func TestMultiReaderB(t *testing.T) {
+	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
+	// member io.Readers, else it should panic. This should hold for
+	// varying sizes of inputs.
+
+	for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
+		src := io.LimitReader(cryrand.Reader, int64(n))
+
+		a := make([]byte, n)
+		nr := 0
+		for nr < n {
+			na, err := src.Read(a)
+			panicOn(err)
+			nr += na
+		}
+		if nr != n {
+			panic("short read")
+		}
+
+		b := make([]byte, n)
+		copy(b, a)
+		if !bytes.Equal(a, b) {
+			panic("test prep failed")
+		}
+
+		m := &MultiReaderB{
+			a: ioutil.NopCloser(bytes.NewBuffer(a)),
+			b: ioutil.NopCloser(bytes.NewBuffer(b)),
+		}
+
+		// should not trigger the internal panic of MultiReadB
+		ncp, err := io.Copy(ioutil.Discard, m)
+		panicOn(err)
+		if ncp != int64(n) {
+			panic("short copy")
+		}
+
+		for victim := 0; victim < n; victim += 7 {
+
+			copy(b, a)
+			if victim%2 == 0 {
+				// corrupt b
+				b[victim] = (b[victim] + 1) % 255
+			} else {
+				// corrupt a
+				a[victim] = (a[victim] + 1) % 255
+			}
+			m = &MultiReaderB{
+				a: ioutil.NopCloser(bytes.NewBuffer(a)),
+				b: ioutil.NopCloser(bytes.NewBuffer(b)),
+			}
+			helperShouldPanicOnCopy(m)
+		}
+	}
+}
+
+func helperShouldPanicOnCopy(m *MultiReaderB) {
+	// differences in bytes read should be noticed
+	defer func() {
+		r := recover()
+		if r == nil {
+			panic("expected panic on byte difference but didn't see it")
+		}
+	}()
+	_, _ = io.Copy(ioutil.Discard, m)
+}

boltdb/attrstore.go

@@ -25,9 +25,9 @@ import (
 
 	"github.com/cespare/xxhash"
 
-	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 // attrBlockSize is the size of attribute blocks for anti-entropy.

boltdb/translate.go

@@ -11,29 +11,51 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
+
+	"runtime/pprof"
 )
 
+var _ = pprof.StartCPUProfile
+
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
+
+	// ErrTranslateKeyNotFound is returned when translating key
+	// and the underlying store returns an empty set
+	ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set")
+
+	bucketKeys = []byte("keys")
+	bucketIDs  = []byte("ids")
+)
+
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+
+	errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found"
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +67,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
-	index string
-	field string
+	index       string
+	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +95,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -74,6 +108,15 @@ func NewTranslateStore(index, field string) *TranslateStore {
 
 // Open opens the translate file.
 func (s *TranslateStore) Open() (err error) {
+
+	// add the path to the problem database if we panic handling it.
+	defer func() {
+		r := recover()
+		if r != nil {
+			panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r))
+		}
+	}()
+
 	if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
 	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
@@ -82,9 +125,9 @@ func (s *TranslateStore) Open() (err error) {
 
 	// Initialize buckets.
 	if err := s.db.Update(func(tx *bolt.Tx) error {
-		if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil {
+		if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
 			return err
-		} else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil {
+		} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
 			return err
 		}
 		return nil
@@ -108,6 +151,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,105 +183,155 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
-func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
-	// Find id by key under read lock.
-	if err := s.db.View(func(tx *bolt.Tx) error {
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
-		return nil
-	}); err != nil {
+// TranslateKey converts a string key to an integer ID.
+// If key does not have an associated id then one is created, unless writable is false,
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
+func (s *TranslateStore) TranslateKey(key string, writable bool) (uint64, error) {
+	ids, err := s.translateKeys([]string{key}, writable)
+	if err != nil {
 		return 0, err
-	} else if id != 0 {
-		return id, nil
 	}
-
-	if s.ReadOnly() {
-		return 0, pilosa.ErrTranslateStoreReadOnly
+	if len(ids) == 0 {
+		return 0, ErrTranslateKeyNotFound
 	}
+	return ids[0], nil
+}
 
-	// Find or create id under write lock.
-	var written bool
-	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
-			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
-			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
-			return err
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
+// If a key does not have an associated id then one is created, unless writable is false,
+// then the function will return the error pilosa.ErrTranslatingKeyNotFound.
+func (s *TranslateStore) TranslateKeys(keys []string, writable bool) ([]uint64, error) {
+	return s.translateKeys(keys, writable)
+}
+
+// FindKeys looks up the ID for each key.
+// Keys are not created if they do not exist.
+// Missing keys are not considered errors, so the length of the result may be less than that of the input.
+func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
+	result := make(map[string]uint64, len(keys))
+	err := s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
+		}
+		for _, key := range keys {
+			id, _ := findIDByKey(bkt, key)
+			if id == 0 {
+				// The key does not exist.
+				continue
+			}
+
+			result[key] = id
 		}
-		written = true
 		return nil
-	}); err != nil {
-		return 0, err
+	})
+	if err != nil {
+		return nil, err
 	}
 
+	return result, nil
+}
+
+// CreateKeys maps all keys to IDs, creating the IDs if they do not exist.
+// If the translator is read-only, this will return an error.
+func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
+	if s.ReadOnly() {
+		return nil, pilosa.ErrTranslateStoreReadOnly
+	}
+
+	written := false
+	result := make(map[string]uint64, len(keys))
+	err := s.db.Update(func(tx *bolt.Tx) error {
+		keyBucket := tx.Bucket(bucketKeys)
+		if keyBucket == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
+		}
+		idBucket := tx.Bucket(bucketIDs)
+		if keyBucket == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
+		}
+		for _, key := range keys {
+			id, boltKey := findIDByKey(keyBucket, key)
+			if id == 0 {
+				// The key does not exist.
+				id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+				if err := keyBucket.Put(boltKey, u64tob(id)); err != nil {
+					return err
+				} else if err := idBucket.Put(u64tob(id), boltKey); err != nil {
+					return err
+				}
+				written = true
+			}
+
+			result[key] = id
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
 	if written {
 		s.notifyWrite()
 	}
 
-	return id, nil
+	return result, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
-func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
-	if len(keys) == 0 {
-		return nil, nil
-	}
+func (s *TranslateStore) translateKeys(keys []string, writable bool) ([]uint64, error) {
+	ids := make([]uint64, 0, len(keys))
 
-	// Allocate slice for ID mapping.
-	ids = make([]uint64, len(keys))
-
-	// Find ids by key under read lock.
-	var found int
-	if err := s.db.View(func(tx *bolt.Tx) error {
-		bkt := tx.Bucket([]byte("keys"))
-		for i, key := range keys {
-			if id := findIDByKey(bkt, key); id != 0 {
-				ids[i] = id
-				found++
+	if s.ReadOnly() || !writable {
+		found := 0
+		if err := s.db.View(func(tx *bolt.Tx) error {
+			bkt := tx.Bucket(bucketKeys)
+			if bkt == nil {
+				return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
 			}
+			for _, key := range keys {
+				if id, _ := findIDByKey(bkt, key); id != 0 {
+					ids = append(ids, id)
+					found++
+				}
+			}
+			return nil
+		}); err != nil {
+			return nil, err
 		}
-		return nil
-	}); err != nil {
-		return nil, err
-	} else if found == len(keys) {
-		return ids, nil
-	}
-
-	if s.ReadOnly() {
-		return ids, pilosa.ErrTranslateStoreReadOnly
+		if found == len(keys) {
+			return ids, nil
+		}
+		if s.ReadOnly() {
+			return ids, pilosa.ErrTranslateStoreReadOnly
+		}
+		if !writable {
+			return nil, pilosa.ErrTranslatingKeyNotFound
+		}
+		return nil, nil
 	}
 
 	// Find or create ids under write lock if any keys were not found.
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		for i, key := range keys {
-			if ids[i] != 0 {
+		bkt := tx.Bucket(bucketKeys)
+		for _, key := range keys {
+			id, boltKey := findIDByKey(bkt, key)
+			if id != 0 {
+				ids = append(ids, id)
 				continue
 			}
-
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
-				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
+			id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+			if err := bkt.Put(boltKey, u64tob(id)); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
+			} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), boltKey); err != nil {
 				return err
 			}
+			ids = append(ids, id)
 			written = true
 		}
 		return nil
 	}); err != nil {
 		return nil, err
 	}
-
 	if written {
 		s.notifyWrite()
 	}
@@ -248,7 +346,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) {
 		return "", err
 	}
 	defer func() { _ = tx.Rollback() }()
-	return findKeyByID(tx.Bucket([]byte("ids")), id), nil
+	return findKeyByID(tx.Bucket(bucketIDs), id), nil
 }
 
 // TranslateIDs converts a list of integer IDs to a list of string keys.
@@ -263,9 +361,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 	}
 	defer func() { _ = tx.Rollback() }()
 
+	bucket := tx.Bucket(bucketIDs)
+
 	keys := make([]string, len(ids))
 	for i, id := range ids {
-		keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id)
+		keys[i] = findKeyByID(bucket, id)
 	}
 	return keys, nil
 }
@@ -273,9 +373,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 // ForceSet writes the id/key pair to the store even if read only. Used by replication.
 func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil {
+		if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
 			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil {
 			return err
 		}
 		return nil
@@ -286,7 +386,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +411,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
-			max = btou64(key)
-		}
+		max = maxID(tx)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +419,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -353,7 +506,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 		var found bool
 		if err := r.store.db.View(func(tx *bolt.Tx) error {
 			// Find ID/key lookup at offset or later.
-			cur := tx.Bucket([]byte("ids")).Cursor()
+			cur := tx.Bucket(bucketIDs).Cursor()
 			key, value := cur.Seek(u64tob(r.offset))
 			if key == nil {
 				return nil
@@ -387,13 +540,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
-	if value := bkt.Get([]byte(key)); value != nil {
-		return btou64(value)
+// emptyKey is a sentinel byte slice which stands for "" as a key.
+var emptyKey = []byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }

boltdb/translate_test.go

@@ -14,112 +14,241 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
-	"io/ioutil"
-	"os"
+	"fmt"
+	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pilosa/pilosa/v2/topology"
 )
 
+//var vv = pilosa.VV
+
 func TestTranslateStore_TranslateKey(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	// Ensure initial key translates to first ID for shard
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
-	if id, err := s.TranslateKey("bar"); err != nil {
+	if id, err := s.TranslateKey("bar", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	if id, err := s.TranslateKey("foo", true); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
 
 func TestTranslateStore_TranslateKeys(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial keys translate to incrementing IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"abc", "abc"}, true)
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids[0]; got != want {
+		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
+	}
+
+	// Ensure initial keys translate to incrementing IDs.
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
+		t.Fatal(err)
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
-	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
+	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}, true); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
 
+func TestTranslateStore_CreateKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.CreateKeys("abc", "abc")
+	if err != nil {
+		t.Fatal(err)
+	} else if _, ok := ids["abc"]; !ok {
+		t.Fatalf(`missing "abc"; got %v`, ids)
+	} else if len(ids) > 1 {
+		t.Fatalf("expected one key, got %d in %v", len(ids), ids)
+	}
+
+	// Ensure different keys translate to different IDs.
+	ids1, err := s.CreateKeys("foo", "bar")
+	if err != nil {
+		t.Fatal(err)
+	} else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar {
+		t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo)
+	}
+
+	// Ensure retranslation returns original IDs.
+	if ids, err := s.CreateKeys("bar", "foo"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(ids, ids1) {
+		t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1)
+	}
+
+	// Ensure retranslating with existing and non-existing keys returns correctly.
+	if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil {
+		t.Fatal(err)
+	} else if got, want := ids["foo"], ids1["foo"]; got != want {
+		t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want)
+	} else if _, ok := ids["baz"]; !ok {
+		t.Fatalf(`missing translation for "baz"; got %v`, ids)
+	} else if got, want := ids["bar"], ids1["bar"]; got != want {
+		t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want)
+	}
+}
+
+func TestTranslateStore_ReadKey(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	id, err := s.TranslateKey("foo", false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+
+	s.SetReadOnly(true)
+	id, err = s.TranslateKey("foo", true)
+	if err == nil {
+		t.Fatalf("got error: %+v, want: 'translate store read only'", err)
+	}
+	if id != 0 {
+		t.Fatalf("TranslateKey()=%d, want %d", id, 0)
+	}
+	s.SetReadOnly(false)
+
+	// Ensure next key autoincrements.
+	if id, err = s.TranslateKey("foo", true); err != nil {
+		t.Fatal(err)
+	}
+	id1, err := s.TranslateKey("foo", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id1 != id {
+		t.Fatalf("TranslateKey()=%d, want %d", id1, id)
+	}
+}
+
+func TestTranslateStore_ReadKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	ids, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != pilosa.ErrTranslatingKeyNotFound {
+		t.Fatal(err)
+	}
+	for _, id := range ids {
+		if id != 0 {
+			t.Fatalf("TranslateKeys()=%d, want %d", id, 0)
+		}
+	}
+
+	// Ensure next key autoincrements.
+	if ids, err = s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, true); err != nil {
+		t.Fatal(err)
+	}
+	ids1, err := s.TranslateKeys([]string{"foo", "bar", "baz", "baz", "bar", "foo"}, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range ids1 {
+		if ids1[i] != ids[i] {
+			t.Fatalf("TranslateKeys()=%d, want %d", ids1[i], ids[i])
+		}
+	}
+}
 func TestTranslateStore_TranslateID(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo", true)
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("", true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -130,13 +259,116 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	}
 }
 
+func TestTranslateStore_FindKeys(t *testing.T) {
+	cases := []struct {
+		name   string
+		data   []string
+		lookup []string
+	}{
+		{
+			name:   "All",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h"},
+		},
+		{
+			name:   "Extra",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h", "65"},
+		},
+		{
+			name:   "None",
+			data:   []string{"a", "b", "c"},
+			lookup: []string{"d", "e"},
+		},
+		{
+			name:   "Empty",
+			lookup: []string{"h"},
+		},
+		{
+			name: "LookupNothing",
+		},
+	}
+
+	for _, c := range cases {
+		c := c
+		t.Run(c.name, func(t *testing.T) {
+			s := MustOpenNewTranslateStore(t)
+			defer MustCloseTranslateStore(s)
+
+			var naiveMap map[string]uint64
+			if c.data != nil {
+				// Load in key data.
+				keys := c.data
+				ids, err := s.TranslateKeys(keys, true)
+				if err != nil {
+					t.Errorf("failed to import keys: %v", err)
+					return
+				}
+				if len(ids) != len(keys) {
+					t.Errorf("mapped %d keys to %d ids", len(keys), len(ids))
+					return
+				}
+				naiveMap = make(map[string]uint64, len(keys))
+				for i, key := range keys {
+					naiveMap[key] = ids[i]
+				}
+			}
+
+			// Compute expected lookup result.
+			result := map[string]uint64{}
+			for _, key := range c.lookup {
+				id, ok := naiveMap[key]
+				if !ok {
+					// The key is expected to be missing.
+					continue
+				}
+
+				result[key] = id
+			}
+
+			// Find the keys.
+			found, err := s.FindKeys(c.lookup...)
+			if err != nil {
+				t.Errorf("failed to find keys: %v", err)
+			} else if !reflect.DeepEqual(result, found) {
+				t.Errorf("expected %v but found %v", result, found)
+			}
+		})
+	}
+}
+
+func TestTranslateStore_MaxID(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	// Generate a bunch of keys.
+	var lastk uint64
+	for i := 0; i < 1026; i++ {
+		k, err := s.TranslateKey(strconv.Itoa(i), true)
+		if err != nil {
+			t.Fatalf("translating %d: %v", i, err)
+		}
+		lastk = k
+	}
+
+	// Verify the max ID.
+	max, err := s.MaxID()
+	if err != nil {
+		t.Fatalf("checking max ID: %v", err)
+	}
+	if max != lastk {
+		t.Fatalf("last key is %d but max is %d", lastk, max)
+	}
+}
+
 func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"}, true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +383,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +392,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz", true)
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -188,7 +421,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure reader will read as soon as a new write comes in using WriteNotify().
 	t.Run("WriteNotify", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -198,20 +431,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo", true)
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -226,7 +464,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on close.
 	t.Run("Close", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -260,7 +498,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on store close.
 	t.Run("StoreClose", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -294,22 +532,87 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 }
 
 // MustNewTranslateStore returns a new TranslateStore with a temporary path.
-func MustNewTranslateStore() *boltdb.TranslateStore {
-	f, err := ioutil.TempFile("", "")
+func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
+	f, err := testhook.TempFile(tb, "translate-store")
 	if err != nil {
 		panic(err)
 	} else if err := f.Close(); err != nil {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore(t)
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}, true); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
-func MustOpenNewTranslateStore() *boltdb.TranslateStore {
-	s := MustNewTranslateStore()
+func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
+	s := MustNewTranslateStore(tb)
 	if err := s.Open(); err != nil {
 		panic(err)
 	}
@@ -320,7 +623,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 	if err := s.Close(); err != nil {
 		panic(err)
-	} else if err := os.Remove(s.Path); err != nil {
-		panic(err)
 	}
 }

broadcast.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"fmt"
 
+	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pkg/errors"
 )
 
@@ -26,11 +27,22 @@ type Serializer interface {
 	Unmarshal([]byte, Message) error
 }
 
+// NopSerializer represents a Serializer that doesn't do anything.
+var NopSerializer Serializer = &nopSerializer{}
+
+type nopSerializer struct{}
+
+// Marshal is a no-op implementation of Serializer Marshal method.
+func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
+
+// Unmarshal is a no-op implementation of Serializer Unmarshal method.
+func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
+
 // broadcaster is an interface for broadcasting messages.
 type broadcaster interface {
 	SendSync(Message) error
 	SendAsync(Message) error
-	SendTo(*Node, Message) error
+	SendTo(*topology.Node, Message) error
 }
 
 // Message is the interface implemented by all core pilosa types which can be serialized to messages.
@@ -49,7 +61,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
 func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
+func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
@@ -63,12 +75,14 @@ const (
 	messageTypeClusterStatus
 	messageTypeResizeInstruction
 	messageTypeResizeInstructionComplete
-	messageTypeSetCoordinator
-	messageTypeUpdateCoordinator
 	messageTypeNodeState
 	messageTypeRecalculateCaches
+	messageTypeLoadSchemaMessage
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
+	messageTypeResizeNodeMessage
+	messageTypeResizeAbortMessage
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -104,18 +118,22 @@ func getMessage(typ byte) Message {
 		return &ResizeInstruction{}
 	case messageTypeResizeInstructionComplete:
 		return &ResizeInstructionComplete{}
-	case messageTypeSetCoordinator:
-		return &SetCoordinatorMessage{}
-	case messageTypeUpdateCoordinator:
-		return &UpdateCoordinatorMessage{}
 	case messageTypeNodeState:
 		return &NodeStateMessage{}
 	case messageTypeRecalculateCaches:
 		return &RecalculateCaches{}
+	case messageTypeLoadSchemaMessage:
+		return &LoadSchemaMessage{}
 	case messageTypeNodeEvent:
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
+	case messageTypeResizeNodeMessage:
+		return &ResizeNodeMessage{}
+	case messageTypeResizeAbortMessage:
+		return &ResizeAbortMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -143,18 +161,22 @@ func getMessageType(m Message) byte {
 		return messageTypeResizeInstruction
 	case *ResizeInstructionComplete:
 		return messageTypeResizeInstructionComplete
-	case *SetCoordinatorMessage:
-		return messageTypeSetCoordinator
-	case *UpdateCoordinatorMessage:
-		return messageTypeUpdateCoordinator
 	case *NodeStateMessage:
 		return messageTypeNodeState
 	case *RecalculateCaches:
 		return messageTypeRecalculateCaches
+	case *LoadSchemaMessage:
+		return messageTypeLoadSchemaMessage
 	case *NodeEvent:
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
+	case *ResizeNodeMessage:
+		return messageTypeResizeNodeMessage
+	case *ResizeAbortMessage:
+		return messageTypeResizeAbortMessage
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

bsi.go

@@ -0,0 +1,296 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"math/bits"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// bsiData contains BSI-structured data.
+type bsiData []*Row
+
+// pivotDescending loops over nonzero BSI values in descending order.
+// For each value, the provided function is called with the value and a slice of the associated columns.
+// If limit or offset are not-nil, they will be applied.
+// Applying a limit or offset may modify the pointed-to value.
+func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+	// This "pivot" algorithm works by treating the BSI data as a tree.
+	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
+	// Each range is subdivided into 2 ranges of half size, which form lower branches.
+	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
+	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
+	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
+	// This function uses a depth-first search over this virtual tree.
+
+	switch {
+	case !filter.Any():
+		// There are no remaining data.
+
+	case offset != nil && *offset >= filter.Count():
+		// Skip this entire branch.
+		*offset -= filter.Count()
+
+	case limit != nil && *limit == 0:
+		// The limit has been reached.
+		// No more data is necessary.
+
+	case len(bsi) == 0:
+		// This is a leaf node.
+		cols := filter.Columns()
+		if offset != nil {
+			cols = cols[*offset:]
+			*offset = 0
+		}
+		if limit != nil {
+			if *limit < uint64(len(cols)) {
+				cols = cols[:*limit]
+			}
+			*limit -= uint64(len(cols))
+		}
+		fn(branch, cols...)
+
+	default:
+		// Pivot over the highest bit.
+		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
+		splitBit := bsi[len(bsi)-1]
+		lowerBits := bsi[:len(bsi)-1]
+		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+	}
+}
+
+/*
+// distribution generates a BSI histogram for the input.
+// TODO: I forgot what I was going to use this for.
+// Could probbably use this for:
+// - quartile queries
+// - TopN on int
+func (bsi bsiData) distribution(filter *Row) bsiData {
+	var dist bsiData
+	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+		dist.insert(count, uint64(len(values)))
+	})
+	return dist
+}
+*/
+
+var placeholderBitmap = roaring.NewBitmap()
+
+// addBSI adds two BSI bitmaps together.
+// It does not handle sign and has no concept of overflow.
+func addBSI(x, y bsiData) bsiData {
+	// Accumulate row segments.
+	segments := make([][]rowSegment, len(x)+len(y))
+	xsegs, ysegs := segments[:len(x)], segments[len(x):]
+	for i, r := range x {
+		xsegs[i] = r.segments
+	}
+	for i, r := range y {
+		ysegs[i] = r.segments
+	}
+
+	var dst bsiData
+	var xbitmaps, ybitmaps []*roaring.Bitmap
+	for {
+		// Find the next shard.
+		next := ^uint64(0)
+		for _, s := range segments {
+			if len(s) == 0 {
+				continue
+			}
+			shard := s[0].shard
+			if shard < next {
+				next = shard
+			}
+		}
+		if next == ^uint64(0) {
+			// There are no remaining shards.
+			break
+		}
+
+		// Accumulate bitmaps for this shard.
+		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
+		for i, segs := range xsegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			xsegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(xbitmaps) < i {
+				xbitmaps = append(xbitmaps, placeholderBitmap)
+			}
+			xbitmaps = append(xbitmaps, bm)
+		}
+		for i, segs := range ysegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			ysegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(ybitmaps) < i {
+				ybitmaps = append(ybitmaps, placeholderBitmap)
+			}
+			ybitmaps = append(ybitmaps, bm)
+		}
+
+		// Add the shard values together.
+		var out []*roaring.Bitmap
+		switch {
+		case len(xbitmaps) == 0:
+			// There are no values in x.
+			out = ybitmaps
+		case len(ybitmaps) == 0:
+			// There are no values in y.
+			out = xbitmaps
+		default:
+			out = roaring.Add(xbitmaps, ybitmaps)
+		}
+
+		// Convert the bitmaps to output segments.
+		for i, b := range out {
+			if !b.Any() {
+				continue
+			}
+			for len(dst) <= i {
+				dst = append(dst, NewRow())
+			}
+			dst[i].segments = append(dst[i].segments, rowSegment{
+				shard:    next,
+				writable: true,
+				data:     b,
+				n:        b.Count(),
+			})
+		}
+	}
+
+	return dst
+}
+
+// rowBuilder builds a row quickly from individual values.
+// It is optimized for the case in which values are generated sequentially.
+type rowBuilder struct {
+	bm    *roaring.Bitmap
+	mask  *[1024]uint64
+	array []uint16
+	key   uint64
+	n     int32
+}
+
+// flushKey flushes the data at the current key to the bitmap.
+func (b *rowBuilder) flushKey() {
+	var c *roaring.Container
+	switch {
+	case b.mask != nil:
+		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
+		b.mask = nil
+	case len(b.array) > 0:
+		c = roaring.NewContainerArrayCopy(b.array)
+		b.array = b.array[:0]
+	default:
+		return
+	}
+
+	if b.bm == nil {
+		b.bm = roaring.NewBitmap()
+	}
+	if old := b.bm.Containers.Get(b.key); old != nil {
+		c = roaring.Union(c, old)
+	}
+	b.bm.Containers.Put(b.key, c)
+}
+
+// Add a value to the bitmap.
+// Values must be added sequentially.
+func (b *rowBuilder) Add(v uint64) {
+	vkey := v / (1 << 16)
+	if b.key != vkey {
+		// This is a new key, so flush the old one.
+		b.flushKey()
+		b.key = vkey
+	}
+
+	if b.mask != nil {
+		// Add to the mask.
+		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
+		b.mask[uint16(v)/64] |= 1 << (v % 64)
+		return
+	}
+
+	// Add to an array.
+	b.array = append(b.array, uint16(v))
+	if len(b.array) >= roaring.ArrayMaxSize {
+		// The array is too big.
+		// Convert it to a bitmask.
+		m := [1024]uint64{}
+		for _, v := range b.array {
+			m[v/64] |= 1 << (v % 64)
+		}
+		b.n = int32(len(b.array))
+		b.array = b.array[:0]
+		b.mask = &m
+	}
+}
+
+// Build a Row from stored data.
+// This resets the builder.
+func (b *rowBuilder) Build() *Row {
+	// Flush the active key to the bitmap.
+	b.flushKey()
+
+	// Remove the bitmap and convert it to a Row.
+	bm := b.bm
+	b.bm = nil
+	if bm == nil {
+		return NewRow()
+	}
+	return NewRowFromBitmap(bm)
+}
+
+// bsiBuilder assembles BSI data.
+// It is optimized for the case in which values are generated sequentially.
+type bsiBuilder []rowBuilder
+
+// Insert a value into the BSI data.
+// Columns must be inserted sequentially, and duplicates are not allowed.
+func (b *bsiBuilder) Insert(col, val uint64) {
+	for val != 0 {
+		i := bits.TrailingZeros64(val)
+		val &^= 1 << i
+		for len(*b) <= i {
+			*b = append(*b, rowBuilder{})
+		}
+		(*b)[i].Add(col)
+	}
+}
+
+// Build BSI data.
+// This resets the builder.
+func (b *bsiBuilder) Build() bsiData {
+	builders := *b
+	*b = builders[:0]
+	rows := make(bsiData, len(builders))
+	for i := range builders {
+		rows[i] = builders[i].Build()
+	}
+	return rows
+}

bsi_test.go

@@ -0,0 +1,172 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+)
+
+// TestBSIAdd does a number of iterations. For each iteration, it
+// generates a random number of ids, and two random values for each id
+// to add together.
+func TestBSIAdd(t *testing.T) {
+	// TODO wouldn't it be cool if our test suite had a randomized
+	// burn-in mode where you could run any test which supported it
+	// with a random seed and way more iterations?
+	rnd := rand.New(rand.NewSource(99))
+	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
+	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
+	var builderA, builderB bsiBuilder
+	// a and b are generated slices of numbers to add together
+	var a, b []uint64
+	// idToIndex maps record ids to indexes in a and b
+	idToIndex := make(map[int]int)
+	// indexToID has the record id for each value in a and b
+	indexToID := []uint64{}
+
+	min := 999999999
+	max := 0
+
+	for iteration := 0; iteration < 1; iteration++ {
+		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
+			// reset generated data
+			a, b = a[:0], b[:0]
+			indexToID = indexToID[:0]
+			for k := range idToIndex {
+				delete(idToIndex, k)
+			}
+
+			// z generates the values, they can be fairly large, but are usually small
+			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
+			id := -1
+			for i := 0; true; i++ {
+				// get the next id, skipping a random amount
+				id = id + int(idZipf.Uint64()+1)
+				if id >= ShardWidth {
+					if i < min {
+						min = i
+					}
+					if max < i {
+						max = i
+					}
+					break
+				}
+				idToIndex[id] = int(i)
+				indexToID = append(indexToID, uint64(id))
+
+				// append a random value to each data slice
+				a = append(a, z.Uint64())
+				b = append(b, z.Uint64())
+			}
+
+			// build the BSIs based on the data slices and generated IDs
+			for index, id := range indexToID {
+				va, vb := a[index], b[index]
+				builderA.Insert(uint64(id), va)
+				builderB.Insert(uint64(id), vb)
+			}
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+
+			// build results from added bsiData; results[i] should hold a[i]+b[i]
+			results := make([]uint64, len(a))
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[idToIndex[int(id)]] = count
+				}
+			})
+
+			for i, res := range results {
+				if res != a[i]+b[i] {
+					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
+				}
+			}
+		})
+	}
+}
+
+type bsiAddCase struct {
+	positions []uint64
+	a         []uint64
+	b         []uint64
+}
+
+func (b bsiAddCase) Len() int {
+	return len(b.positions)
+}
+
+// Less reports whether the element with
+// index i should sort before the element with index j.
+func (b bsiAddCase) Less(i, j int) bool {
+	return b.positions[i] < b.positions[j]
+}
+
+// Swap swaps the elements with indexes i and j.
+func (b bsiAddCase) Swap(i, j int) {
+	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
+	b.a[i], b.a[j] = b.a[j], b.a[i]
+	b.b[i], b.b[j] = b.b[j], b.b[i]
+}
+
+// TestBSIAddCases tests specific cases of bsiAdd (would generally be
+// pulled from randomly generated ones from TestBSIAdd upon failure).
+func TestBSIAddCases(t *testing.T) {
+	tests := []bsiAddCase{
+		{
+			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
+			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
+			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
+		},
+		{
+			positions: []uint64{17570, 54607},
+			a:         []uint64{1010, 71},
+			b:         []uint64{25, 150},
+		},
+	}
+
+	var builderA, builderB bsiBuilder
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
+				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
+			}
+			sort.Sort(tst)
+
+			for i := 0; i < len(tst.a); i++ {
+				builderA.Insert(tst.positions[i], tst.a[i])
+				builderB.Insert(tst.positions[i], tst.b[i])
+			}
+
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+			// maps id to count
+			results := make(map[uint64]uint64)
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[id] = count
+				}
+			})
+
+			for i, id := range tst.positions {
+				if results[id] != tst.a[i]+tst.b[i] {
+					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
+				}
+			}
+		})
+	}
+}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +121,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -134,9 +138,11 @@ var _ cache = &lruCache{}
 
 // rankCache represents a cache with sorted entries.
 type rankCache struct {
-	mu       sync.Mutex
-	entries  map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	mu           sync.Mutex
+	entries      map[uint64]uint64
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -168,10 +174,16 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	// Ignore if the column count is below the threshold,
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -184,7 +196,13 @@ func (c *rankCache) Add(id uint64, n uint64) {
 func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -209,12 +227,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
-	for id := range c.entries {
-		a = append(a, id)
+	if len(c.entries) == 0 {
+		return nil
 	}
-	sort.Sort(uint64Slice(a))
-	return a
+	ids := make([]uint64, 0, len(c.entries))
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +249,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -236,27 +257,40 @@ func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,11 +306,14 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
+
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // SetStats defines the stats client used in the cache.
@@ -285,7 +322,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +358,68 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
+func (p *bitmapPairs) Len() int           { return len(*p) }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
+	}
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +495,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
+	}
+	copy(r.Pairs, p.Pairs)
+	return
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -431,13 +607,7 @@ func (p uint64Slice) merge(other []uint64) []uint64 {
 	return ret
 }
 
-// bitmapCache provides an interface for caching full bitmaps.
-type bitmapCache interface {
-	Fetch(id uint64) (*Row, bool)
-	Add(id uint64, b *Row)
-}
-
-// simpleCache implements BitmapCache
+// simpleCache implements a bitmap Rowcache.
 // it is meant to be a short-lived cache for cases where writes are continuing to access
 // the same row within a short time frame (i.e. good for write-heavy loads)
 // A read-heavy use case would cause the cache to get bigger, potentially causing the
@@ -452,6 +622,12 @@ func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
 	return m, ok
 }
 
+func newSimpleCache() *simpleCache {
+	return &simpleCache{
+		cache: make(map[uint64]*Row),
+	}
+}
+
 // Add adds the bitmap to the cache, keyed on the id. A nil row means
 // deleting the row from the cache.
 func (s *simpleCache) Add(id uint64, b *Row) {

cache_test.go

@@ -15,13 +15,14 @@
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +32,54 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
 }

catcher.go

@@ -0,0 +1,323 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	//txkey "github.com/pilosa/pilosa/v2/txkey"
+)
+
+// catcher is useful to report error locations with a
+// stack dump before the complexity
+// of the executor_test swallows up
+// the location of a panic.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
+	c.b.IncrementOpN(index, field, view, shard, changedN)
+}
+
+func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	return c.b.NewTxIterator(index, field, view, shard)
+}
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
+}
+
+func (c *catcherTx) Dump(short bool, shard uint64) {
+	c.b.Dump(short, shard)
+}
+
+func (c *catcherTx) Readonly() bool {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Readonly()
+}
+
+func (tx *catcherTx) Pointer() string {
+	return fmt.Sprintf("%p", tx)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) UseRowCache() bool {
+	return c.b.UseRowCache()
+}
+
+func (c *catcherTx) IsDone() bool {
+	return c.b.IsDone()
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, batched, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEach(index, field, view, shard, fn)
+}
+
+func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.UnionInPlace(index, field, view, shard, others...)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+
+func (c *catcherTx) Group() *TxGroup {
+	return c.b.Group()
+}
+
+func (c *catcherTx) Options() Txo {
+	return c.b.Options()
+}
+
+// Sn retreives the serial number of the Tx.
+func (c *catcherTx) Sn() int64 {
+	return c.b.Sn()
+}
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
+	return c.b.GetSortedFieldViewList(idx, shard)
+}
+
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

client.go

@@ -17,6 +17,10 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
+
+	pnet "github.com/pilosa/pilosa/v2/net"
+	"github.com/pilosa/pilosa/v2/topology"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -43,15 +47,18 @@ type FieldValue struct {
 // something hasn't been architected correctly.
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
+// Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client.
+// Doing that may obviate the need to refactor this.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
-	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
+	PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error
 	CreateIndex(ctx context.Context, index string, opt IndexOptions) error
-	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
-	Nodes(ctx context.Context) ([]*Node, error)
+	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error)
+	Nodes(ctx context.Context) ([]*topology.Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,40 +66,91 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
-	FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
-	BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
-	ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	SendMessage(ctx context.Context, uri *URI, msg []byte) error
-	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
-	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
+	BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
+	ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
+	RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
+	SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
+	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
+	ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
+
+	GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
+	GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
 }
 
 //===============
 
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error)
+
+	QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+
+	// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
+	TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error)
+
+	FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
+	FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
+
+	CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
+	CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
 }
 
 type nopInternalQueryClient struct{}
 
-func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
+func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) {
 	return nil, nil
 }
 
-func newNopInternalQueryClient() *nopInternalQueryClient {
-	return &nopInternalQueryClient{}
+func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
+	return nil, nil
+}
+
+func newNopInternalQueryClient() nopInternalQueryClient {
+	return nopInternalQueryClient{}
 }
 
 var _ InternalQueryClient = newNopInternalQueryClient()
 
 //===============
 
-type nopInternalClient struct{}
+type nopInternalClient struct{ nopInternalQueryClient }
 
 func newNopInternalClient() nopInternalClient {
 	return nopInternalClient{}
@@ -104,34 +162,40 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64,
 	return nil, nil
 }
 func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
-func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error {
+func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error {
 	return nil
 }
 
 func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
 	return nil
 }
-func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
+func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
 	return nil, nil
 }
-func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
+func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
 	return nil, nil
 }
 func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
-func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
-func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
+	return nil
+}
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -154,21 +218,45 @@ func (n nopInternalClient) CreateField(ctx context.Context, index, field string)
 func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
 	return nil
 }
-func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
+func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
 	return nil, nil
 }
-func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
+func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
 	return nil, nil, nil
 }
-func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
 	return nil, nil
 }
-func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
+func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
 	return nil, nil
 }
-func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
+func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
 	return nil
 }
-func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
+func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
 	return nil, nil
 }

cluster_internal_test.go

@@ -15,42 +15,80 @@
 package pilosa
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"os"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/gorilla/mux"
-	"github.com/pilosa/pilosa/v2/logger"
+	pnet "github.com/pilosa/pilosa/v2/net"
 	"github.com/pilosa/pilosa/v2/roaring"
-	"github.com/pkg/errors"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pilosa/pilosa/v2/topology"
 )
 
+// GlobalPortMap avoids many races and port conflicts when setting
+// up ports for test clusters. Used for tests only.
+var globalPortMap *GlobalPortMapper
+
+func init() {
+	globalPortMap = NewGlobalPortMapper(300)
+}
+
+// GlobalPortMapper maintains a pool of available ports by
+// holding them open until GetPort() is called.
+type GlobalPortMapper struct {
+	availPorts map[int]net.Listener
+}
+
+// reserve n ports
+func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) {
+
+	pm = &GlobalPortMapper{
+		availPorts: make(map[int]net.Listener),
+	}
+	for i := 0; i < n; i++ {
+		lsn, _ := net.Listen("tcp", ":0")
+		r := lsn.Addr()
+		port := r.(*net.TCPAddr).Port
+		pm.availPorts[port] = lsn
+	}
+	return
+}
+
+func (pm *GlobalPortMapper) GetPort() (port int, err error) {
+	for port, lsn := range pm.availPorts {
+		lsn.Close()
+		return port, nil
+	}
+	return -1, fmt.Errorf("no more ports available")
+}
+
+func (pm *GlobalPortMapper) MustGetPort() int {
+	port, err := pm.GetPort()
+	if err != nil {
+		panic(err)
+	}
+	return port
+}
+
 // Ensure that fragCombos creates the correct fragment mapping.
 func TestFragCombos(t *testing.T) {
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
 
 	c := newCluster()
 	c.addNodeBasicSorted(node0)
@@ -90,13 +128,34 @@ func TestFragCombos(t *testing.T) {
 	}
 }
 
-// newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+// newHolderWithTempPath returns a new instance of Holder.
+func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	cfg := mustHolderConfig()
+	cfg.StorageConfig.Backend = backend
+	h := NewHolder(path, cfg)
+	panicOn(h.Open())
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
+	return h
+}
+
+// newIndexWithTempPath returns a new instance of Index.
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
+	if err != nil {
+		panic(err)
+	}
+	h := NewHolder(path, nil)
+	panicOn(h.Open())
+	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -105,28 +164,27 @@ func newIndexWithTempPath(name string) *Index {
 
 // Ensure that fragSources creates the correct fragment mapping.
 func TestFragSources(t *testing.T) {
-
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri3, err := NewURIFromAddress("host3")
+	uri3, err := pnet.NewURIFromAddress("host3")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
-	node3 := &Node{ID: "node3", URI: *uri3}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
+	node3 := &topology.Node{ID: "node3", URI: *uri3}
 
 	c1 := newCluster()
 	c1.ReplicaN = 1
@@ -157,27 +215,53 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 101, nil)
+
+	// Obtain transaction.
+	var shard uint64
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 1
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*2+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 2
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*3+1, nil)
+	panicOn(tx.Commit())
+
+	shard = 3
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	panicOn(tx.Commit())
 
 	tests := []struct {
 		from     *cluster
@@ -194,8 +278,8 @@ func TestFragSources(t *testing.T) {
 				"node0": {},
 				"node1": {},
 				"node2": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -206,11 +290,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(1)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -221,11 +305,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(3)},
 				},
 				"node2": {},
 			},
@@ -274,37 +358,37 @@ func TestFragSources(t *testing.T) {
 // Ensure that fragSources creates the correct fragment mapping.
 func TestResizeJob(t *testing.T) {
 
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
 
 	tests := []struct {
-		existingNodes []*Node
-		node          *Node
+		existingNodes []*topology.Node
+		node          *topology.Node
 		action        string
 		expectedIDs   map[string]bool
 	}{
 		{
-			existingNodes: []*Node{node0, node1},
+			existingNodes: []*topology.Node{node0, node1},
 			node:          node2,
 			action:        resizeJobActionAdd,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
 		},
 		{
-			existingNodes: []*Node{node0, node1, node2},
+			existingNodes: []*topology.Node{node0, node1, node2},
 			node:          node2,
 			action:        resizeJobActionRemove,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false},
@@ -325,22 +409,27 @@ func TestResizeJob(t *testing.T) {
 // Ensure the cluster can fairly distribute partitions across the nodes.
 func TestCluster_Owners(t *testing.T) {
 	c := cluster{
-		nodes: []*Node{
+		noder: topology.NewLocalNoder([]*topology.Node{
 			{URI: NewTestURIFromHostPort("serverA", 1000)},
 			{URI: NewTestURIFromHostPort("serverB", 1000)},
 			{URI: NewTestURIFromHostPort("serverC", 1000)},
-		},
+		}),
 		Hasher:   NewTestModHasher(),
 		ReplicaN: 2,
 	}
 
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
 	// Verify nodes are distributed.
-	if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) {
+	if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*topology.Node{cNodes[0], cNodes[1]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 
 	// Verify nodes go around the ring.
-	if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) {
+	if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*topology.Node{cNodes[2], cNodes[0]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 }
@@ -351,7 +440,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := topology.ShardToShardPartition(index, shard, partitionN)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -381,7 +470,7 @@ func TestHasher(t *testing.T) {
 		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
 	} {
 		for i, v := range tt.bucket {
-			hasher := &jmphasher{}
+			hasher := &topology.Jmphasher{}
 			if got := hasher.Hash(tt.key, i+1); got != v {
 				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
 			}
@@ -391,9 +480,14 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
-	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
+	shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
 
 	if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
 		t.Fatalf("unexpected shars for node's index: %v", shards)
@@ -401,20 +495,22 @@ func TestCluster_ContainsShards(t *testing.T) {
 }
 
 func TestCluster_Nodes(t *testing.T) {
-	uri0 := NewTestURIFromHostPort("node0", 0)
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-	uri3 := NewTestURIFromHostPort("node3", 0)
+	const urisCount = 4
+	var uris []pnet.URI
+	arbitraryPorts := []int{17384, 17385, 17386, 17387}
+	for i := 0; i < urisCount; i++ {
+		uris = append(uris, NewTestURIFromHostPort(fmt.Sprintf("node%d", i), uint16(arbitraryPorts[i])))
+	}
 
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	node3 := &Node{ID: "node3", URI: uri3}
+	node0 := &topology.Node{ID: "node0", URI: uris[0]}
+	node1 := &topology.Node{ID: "node1", URI: uris[1]}
+	node2 := &topology.Node{ID: "node2", URI: uris[2]}
+	node3 := &topology.Node{ID: "node3", URI: uris[3]}
 
-	nodes := []*Node{node0, node1, node2}
+	nodes := []*topology.Node{node0, node1, node2}
 
 	t.Run("NodeIDs", func(t *testing.T) {
-		actual := Nodes(nodes).IDs()
+		actual := topology.Nodes(nodes).IDs()
 		expected := []string{node0.ID, node1.ID, node2.ID}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
@@ -422,24 +518,24 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Filter", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
-		expected := []URI{uri0, uri2}
+		actual := topology.Nodes(topology.Nodes(nodes).Filter(nodes[1])).URIs()
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("FilterURI", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
-		expected := []URI{uri0, uri2}
+		actual := topology.Nodes(topology.Nodes(nodes).FilterURI(uris[1])).URIs()
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("Contains", func(t *testing.T) {
-		actualTrue := Nodes(nodes).Contains(node1)
-		actualFalse := Nodes(nodes).Contains(node3)
+		actualTrue := topology.Nodes(nodes).Contains(node1)
+		actualFalse := topology.Nodes(nodes).Contains(node3)
 		if !reflect.DeepEqual(actualTrue, true) {
 			t.Errorf("expected: %v, but got: %v", true, actualTrue)
 		}
@@ -449,9 +545,9 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Clone", func(t *testing.T) {
-		clone := Nodes(nodes).Clone()
-		actual := Nodes(clone).URIs()
-		expected := []URI{uri0, uri1, uri2}
+		clone := topology.Nodes(nodes).Clone()
+		actual := topology.Nodes(clone).URIs()
+		expected := []pnet.URI{uris[0], uris[1], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
@@ -459,9 +555,9 @@ func TestCluster_Nodes(t *testing.T) {
 }
 
 func TestCluster_PreviousNode(t *testing.T) {
-	node0 := &Node{ID: "node0"}
-	node1 := &Node{ID: "node1"}
-	node2 := &Node{ID: "node2"}
+	node0 := &topology.Node{ID: "node0"}
+	node1 := &topology.Node{ID: "node1"}
+	node2 := &topology.Node{ID: "node2"}
 
 	t.Run("OneNode", func(t *testing.T) {
 		c := newCluster()
@@ -512,332 +608,6 @@ func TestCluster_PreviousNode(t *testing.T) {
 	})
 }
 
-// NEXT: move this test to internal and unexport IsCoordinator
-func TestCluster_Coordinator(t *testing.T) {
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-
-	c1 := *newCluster()
-	c1.Node = node1
-	c1.Coordinator = node1.ID
-	c2 := *newCluster()
-	c2.Node = node2
-	c2.Coordinator = node1.ID
-
-	t.Run("IsCoordinator", func(t *testing.T) {
-		if !c1.isCoordinator() {
-			t.Errorf("!IsCoordinator error: %v", c1.Node)
-		} else if c2.isCoordinator() {
-			t.Errorf("IsCoordinator error: %v", c2.Node)
-		}
-	})
-}
-
-func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
-
-	uri0 := NewTestURIFromHostPort("host0", 0)
-	uri1 := NewTestURIFromHostPort("host1", 0)
-	uri2 := NewTestURIFromHostPort("host2", 0)
-	invalid := NewTestURIFromHostPort("invalid", 0)
-
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
-
-	t.Run("AddNode", func(t *testing.T) {
-		err := c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// add the same host.
-		err = c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = c1.addNode(node2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		actual := c1.nodeIDs()
-		expected := []string{node0.ID, node1.ID, node2.ID}
-
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("ContainsID", func(t *testing.T) {
-		if !c1.Topology.ContainsID(node1.ID) {
-			t.Errorf("!ContainsHost error: %v", node1.ID)
-		} else if c1.Topology.ContainsID(nodeinvalid.ID) {
-			t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
-		}
-	})
-}
-
-// Ensure that general cluster functionality works as expected.
-func TestCluster_ResizeStates(t *testing.T) {
-
-	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		node := tc.Clusters[0]
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"some-other-host"},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		expected := "coordinator node0 is not in topology: [some-other-host]"
-		err := tc.Open()
-		if err == nil || errors.Cause(err).Error() != expected {
-			t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node0 := tc.Clusters[0]
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"node0", "node2"},
-		}
-		if err := tc.WriteTopology(node0.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		// Ensure that node is in state STARTING before the other node joins.
-		if node0.State() != ClusterStateStarting {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
-		}
-
-		// Expect an error by adding a node not in the topology.
-		expectedError := "host is not in topology: node1"
-		if err := tc.addNode(); err == nil || err.Error() != expectedError {
-			t.Errorf("did not receive expected error: %s", expectedError)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node2 := tc.Clusters[2]
-
-		// Ensure that node comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node2.State() != ClusterStateNormal {
-			t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Add Bit Data to node0.
-		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-
-		// Before starting the resize, get the CheckSum to use for
-		// comparison later.
-		node0Field := node0.holder.Field("i", "f")
-		node0View := node0Field.view("standard")
-		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
-
-		// addNode needs to block until the resize process has completed.
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes come up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Bits
-		// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
-		node1Field := node1.holder.Field("i", "f")
-		node1View := node1Field.view("standard")
-		node1Fragment := node1View.Fragment(1)
-
-		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
-			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-}
-
 func TestAE(t *testing.T) {
 	t.Run("AbortDoesn'tBlockUninitialized", func(t *testing.T) {
 		c := newCluster()
@@ -846,6 +616,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -857,11 +628,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@@ -893,97 +666,4 @@ func TestAE(t *testing.T) {
 			t.Fatalf("abort should not have blocked this long")
 		}
 	})
-
-}
-
-// Ensures that coordinator can be changed.
-func TestCluster_UpdateCoordinator(t *testing.T) {
-	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
-
-		oldNode := c.nodes[0]
-		newNode := c.nodes[1]
-
-		// Update coordinator to the same value.
-		if c.updateCoordinator(oldNode) {
-			t.Errorf("did not expect coordinator to change")
-		} else if c.Coordinator != oldNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
-		}
-
-		// Update coordinator to a new value.
-		if !c.updateCoordinator(newNode) {
-			t.Errorf("expected coordinator to change")
-		} else if c.Coordinator != newNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
-		}
-	})
-}
-
-func TestCluster_confirmNodeDownUp(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be up")
-	}
-
-}
-func TestCluster_confirmNodeDownTimeout(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
-}
-
-func TestCluster_confirmNodeDownDown(t *testing.T) {
-	uri := URI{}
-	uri.Scheme = "http"
-	uri.Host = "DoesntMatter"
-	uri.Port = 6666
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
 }

cmd/badloader/badloader.go

@@ -0,0 +1,158 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+
+	//"fmt"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	pnet "github.com/pilosa/pilosa/v2/net"
+
+	//"log"
+	"os"
+	//"path/filepath"
+	//"sort"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *http.InternalClient) error {
+	t0 := time.Now()
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	//vv("top of tar loop")
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				panic("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				panicOn(err)
+				//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+			}
+			return nil
+		}
+		//vv("got header '%v'", header.Name)
+		n++
+		if n%500 == 0 {
+			vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		//vv("parts = '%#v'", parts)
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+// badloader reproduce a union in place issue for us. slurp is
+// the new "good" loader, and should always be preferred now
+// when not trying to repro that bug. pulled from 85fa67e8
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	panicOn(UploadTar(tarSrcPath, c))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pnet.URI
+
+func init() {
+	var err error
+	globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
+	panicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
+	return globURI
+}

cmd/badloader/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/convert.go

@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
 			return inspector.Run(context.Background())
 		},
 	}
+	flags := inspectCmd.Flags()
+	flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+	flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+	flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+	flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+	flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+	flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
 	return inspectCmd
 }

cmd/export.go

@@ -50,7 +50,7 @@ The file does not contain any headers.
 	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
 	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
-	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
 
 	return exportCmd
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
@@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
-	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
 
 	return importCmd
 }

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
-					Min:       -10,
+					Max:       pql.NewDecimal(100, 0),
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/pilosa-bench/main.go

@@ -0,0 +1,358 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"expvar"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	phttp "github.com/pilosa/pilosa/v2/http"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	requestCountVar          = expvar.NewInt("request_count")
+	requestCurrentLatencyVar = expvar.NewFloat("request_current_latency") // seconds
+	requestAvgLatencyVar     = expvar.NewFloat("request_avg_latency")     // seconds
+	requestTotalLatencyVar   = expvar.NewFloat("request_total_latency")   // seconds
+	requestPerSecVar         = expvar.NewFloat("request_per_sec")
+)
+
+func main() {
+	if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) (err error) {
+	fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
+	hostport := fs.String("hostport", "localhost:10101", "")
+	typ := fs.String("type", "row", "query type (row)")
+	n := fs.Int("n", 1000, "number of queries")
+	rate := fs.Int("rate", 1, "number of queries per second")
+	verbose := fs.Bool("v", false, "verbose logging")
+	from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
+	to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse from/to time.
+	var opt queryOptions
+	if *from != "" {
+		if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
+			return fmt.Errorf("cannot parse -from time")
+		}
+	}
+	if *to != "" {
+		if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
+			return fmt.Errorf("cannot parse -to time")
+		}
+	}
+
+	if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
+		return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
+	}
+
+	// Clear time prefix on log.
+	log.SetFlags(0)
+	if !*verbose {
+		log.SetOutput(ioutil.Discard)
+	}
+
+	// Setup PRNG to have consistent values for the same set of data.
+	rand.Seed(0)
+
+	// Setup connection to pilosa.
+	client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
+	if err != nil {
+		return err
+	}
+
+	// Set up HTTP endpoint to provide /debug endpoints.
+	fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
+	go func() { _ = http.ListenAndServe(":7070", nil) }()
+
+	// Run separate goroutine to calculate the current req/sec & latency.
+	go monitor()
+
+	// Load all id/keys for each field.
+	log.Printf("loading field identifiers")
+	fieldIDMap, err := loadFields(ctx, client)
+	if err != nil {
+		return fmt.Errorf("cannot load field identifiers: %w", err)
+	} else if len(fieldIDMap) == 0 {
+		return fmt.Errorf("no field identifiers available, please verify data exists")
+	}
+
+	// Generate list of sorted keys.
+	fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
+	for k, f := range fieldIDMap {
+		switch *typ {
+		case "row-bsi":
+			if f.info.Options.Type != "int" {
+				continue
+			}
+		case "row-range", "topk":
+			if f.info.Options.Type != "time" {
+				continue
+			}
+		default:
+			if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
+				continue
+			}
+		}
+
+		fieldKeys = append(fieldKeys, k)
+	}
+	sort.Slice(fieldKeys, func(i, j int) bool {
+		return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
+	})
+
+	// Ensure we have appropriate fields for our query type.
+	if len(fieldKeys) == 0 {
+		return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
+	}
+
+	log.Printf("issuing %d queries at %d query/sec", *n, *rate)
+
+	// Repeatedly issue queries based on available row data.
+	ticker := time.NewTicker(time.Second / time.Duration(*rate))
+	var g errgroup.Group
+	for i := 0; i < *n; i++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		key := fieldKeys[rand.Intn(len(fieldKeys))]
+		q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
+		if err != nil {
+			return fmt.Errorf("cannot generate query: %w", err)
+		}
+
+		log.Printf("[query] %s", q)
+
+		g.Go(func() error {
+			t := time.Now()
+			_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
+			if err != nil {
+				return err
+			}
+			elapsed := time.Since(t).Seconds()
+			requestCountVar.Add(1)
+			requestTotalLatencyVar.Add(elapsed)
+			requestAvgLatencyVar.Set(requestTotalLatencyVar.Value() / float64(requestCountVar.Value()))
+			return nil
+		})
+	}
+
+	return g.Wait()
+}
+
+// monitor runs in a separate goroutine and updates metrics.
+func monitor() {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	var lastTime time.Time
+	var lastN int64
+	var lastLatency float64
+	for range ticker.C {
+		now, n := time.Now(), requestCountVar.Value()
+		latency := requestTotalLatencyVar.Value()
+
+		if !lastTime.IsZero() {
+			elapsed := lastTime.Sub(now).Seconds()
+			if n > 0 {
+				requestCurrentLatencyVar.Set((lastLatency - latency) / float64(n))
+			}
+			requestPerSecVar.Set(float64(lastN-n) / elapsed)
+		}
+		lastTime, lastN, lastLatency = now, n, latency
+	}
+}
+
+func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
+	switch typ {
+	case "row":
+		return generateRowQuery(index, field, identifiers), nil
+	case "row-bsi":
+		return generateRowBSIQuery(index, field), nil
+	case "row-range":
+		return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
+	case "count":
+		return generateCountQuery(index, field, identifiers), nil
+	case "intersect":
+		return generateIntersectQuery(index, field, identifiers), nil
+	case "union":
+		return generateUnionQuery(index, field, identifiers), nil
+	case "difference":
+		return generateDifferenceQuery(index, field, identifiers), nil
+	case "xor":
+		return generateXorQuery(index, field, identifiers), nil
+	case "groupby":
+		return generateGroupByQuery(index, field), nil
+	case "topk":
+		return generateTopKQuery(index, field, opt.from, opt.to), nil
+	default:
+		return "", fmt.Errorf("invalid query type: %q", typ)
+	}
+}
+
+func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
+	}
+	return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
+}
+
+func generateRowBSIQuery(index, field string) string {
+	return fmt.Sprintf("Row(%s > 0)", field)
+}
+
+func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+	}
+	return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	a := make([]string, rand.Intn(9)+1)
+	for i := range a {
+		a[i] = generateRowQuery(index, field, identifiers)
+	}
+	return strings.Join(a, ", ")
+}
+
+func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
+}
+
+func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateGroupByQuery(index, field string) string {
+	return fmt.Sprintf("GroupBy(Rows(%s))", field)
+}
+
+func generateTopKQuery(index, field string, from, to time.Time) string {
+	return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+// loadFields returns a mapping of index/field names to field info & identifiers.
+func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
+	indexes, err := client.Schema(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[fieldKey]*fieldInfo)
+	for _, ii := range indexes {
+		for _, f := range ii.Fields {
+			log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
+
+			switch f.Options.Type {
+			case "set", "mutex", "time":
+				identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
+				if err != nil {
+					return nil, fmt.Errorf("fetch fields: %w", err)
+				} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
+					m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
+				}
+
+			case "int":
+				m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
+			}
+		}
+	}
+	return m, nil
+}
+
+// fetchFieldIDs returns a list of field IDs or keys.
+func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
+	resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
+	if err != nil {
+		return nil, err
+	}
+
+	switch result := resp.Results[0].(type) {
+	case *pilosa.RowIdentifiers:
+		return result, nil
+	case pilosa.RowIdentifiers:
+		return &result, nil
+	default:
+		return nil, fmt.Errorf("unexpected result type: %T", result)
+	}
+}
+
+func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
+	return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
+}
+
+func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
+	return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
+}
+
+type fieldKey struct {
+	index string
+	field string
+}
+
+type fieldInfo struct {
+	info        *pilosa.FieldInfo
+	identifiers *pilosa.RowIdentifiers
+}
+
+func compareFieldKeys(x, y fieldKey) int {
+	if cmp := strings.Compare(x.index, y.index); cmp != 0 {
+		return cmp
+	}
+	return strings.Compare(x.field, y.field)
+}
+
+type queryOptions struct {
+	from, to time.Time
+}

cmd/random-query/main.go

@@ -0,0 +1,556 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math"
+	"math/rand"
+	nethttp "net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
+)
+
+// RandomQueryConfig
+type RandomQueryConfig struct {
+
+	// user facing flags
+	HostPort    string    // -hostport
+	TreeDepth   int       // -d
+	QueryCount  int       // -n
+	Verbose     bool      // -v
+	VeryVerbose bool      // -V
+	TimeFromArg string    // --time.from
+	TimeToArg   string    // --time.to
+	TimeFrom    time.Time // parsed time
+	TimeTo      time.Time // parsed time
+	TimeRange   int64     // hours between parsed times
+
+	IndexMap map[string]*Features
+
+	API  *pilosa.API
+	Info []*pilosa.IndexInfo
+
+	BitmapFunc []string
+
+	Rnd *rand.Rand
+}
+
+type API interface {
+
+	// InternalClient
+	Schema(ctx context.Context) ([]*pilosa.IndexInfo, error)
+	Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error)
+
+	// API for contrast; just a little different:
+	//Schema(ctx context.Context) []*IndexInfo
+	//Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error)
+}
+
+// have to wrap because the ugly little differences between InternalClient and API
+type wrapper struct {
+	api *pilosa.API
+}
+
+func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	return w.api.Schema(ctx, false)
+}
+
+func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	r, err := w.api.Query(ctx, queryRequest)
+	return &r, err
+}
+
+func wrapApiToInternalClient(api *pilosa.API) *wrapper {
+	return &wrapper{api: api}
+}
+
+// These times are copied from the "kitchen sink" data generator to serve as defaults.
+var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
+var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
+
+// call DefineFlags before myflags.Parse()
+func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
+	fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.")
+	fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.")
+	fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
+	fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results")
+	fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+	fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *RandomQueryConfig) ValidateConfig() error {
+	if c.TreeDepth < 1 {
+		return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth)
+	}
+	if c.QueryCount < 0 {
+		return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
+	}
+	var err error
+	c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg)
+	if err != nil {
+		return fmt.Errorf("-time.from value couldn't be parsed: %w", err)
+	}
+	c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg)
+	if err != nil {
+		return fmt.Errorf("-time.to value couldn't be parsed: %w", err)
+	}
+	c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours())
+	if c.TimeRange < 1 {
+		return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
+			c.TimeToArg, c.TimeFromArg)
+	}
+	return nil
+}
+
+var ProgramName = "random-query"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
+	cfg := NewRandomQueryConfig()
+	cfg.DefineFlags(myflags)
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+
+	err = cfg.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *RandomQueryConfig) Run() (err error) {
+	remoteClient := nethttp.DefaultClient
+	cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	totalQ := 0
+	loops := 0
+	t0 := time.Now()
+
+	report := func() {
+		dur := time.Since(t0)
+		if dur > 0 {
+			qps := 1e9 * float64(totalQ) / float64(dur)
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: %0.02f", totalQ, dur, qps)
+		} else {
+			AlwaysPrintf("totalQueries run: %v   elapsed: %v   qps: N/A", totalQ, dur)
+		}
+	}
+	defer report()
+
+NewSetup:
+	err = cfg.Setup(cli)
+	if err != nil {
+		return err
+	}
+
+	if len(cfg.IndexMap) == 0 {
+		return fmt.Errorf("no rows to query")
+	}
+
+	var indexes []string
+	for index := range cfg.IndexMap {
+		indexes = append(indexes, index)
+	}
+
+	for j := 0; ; j++ {
+		if cfg.QueryCount > 0 {
+			if j >= cfg.QueryCount {
+				break
+			}
+		} else {
+			// else keep doing queries forever...
+			if loops > 0 && loops%500 == 0 {
+				// ...but account for any new data arrived by getting
+				// the schema and rows again every so often.
+				loops++
+				goto NewSetup
+			}
+		}
+		if totalQ > 0 && totalQ%100 == 0 {
+			report()
+		}
+
+		index := indexes[cfg.Rnd.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		if cfg.Verbose {
+			fmt.Printf("pql = '%v'\n", pql)
+		}
+
+		// Query node0.
+		res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql)
+			return err
+		}
+		if cfg.VeryVerbose {
+			fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+		}
+		totalQ++
+		loops++
+
+	}
+
+	return nil
+}
+
+type Features struct {
+	Slc           []IndexFieldRow
+	Ranges        []IndexFieldRange
+	Distinctables []IndexFieldRange
+	SlcWeight     int
+	RangeWeight   int
+}
+
+// Pick either a feature entry or a random query on a range, weighted
+// by number of features and approximate weight of ranges
+func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight)
+	if r < f.SlcWeight {
+		return f.Slc[r].Query(cfg)
+	}
+	r = cfg.Rnd.Intn(len(f.Ranges))
+	return f.Ranges[r].Query(cfg)
+}
+
+func NewRandomQueryConfig() *RandomQueryConfig {
+	return &RandomQueryConfig{
+		IndexMap: make(map[string]*Features),
+	}
+}
+
+type IndexFieldRow struct {
+	Index    string
+	Field    string
+	RowID    uint64
+	RowKey   string
+	IsRowKey bool
+	HasTime  bool
+	IsInt    bool
+}
+
+func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
+	fromTo := ""
+	// 5% of queries on a time field will use the standard view
+	// anyway.
+	if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
+		startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
+		endHours := cfg.Rnd.Int63n(cfg.TimeRange-startHours) + 1 + startHours
+		startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
+		endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
+		fromTo = fmt.Sprintf(", from=%s, to=%s",
+			startTime.Format(pilosaTimeFmt),
+			endTime.Format(pilosaTimeFmt))
+	}
+	if fea.IsRowKey {
+		return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)}
+	}
+	return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
+}
+
+type IndexFieldRange struct {
+	Index           string
+	Field           string
+	Min, Max, Scale int64
+	ScaleDiv        float64
+	Range           uint64
+}
+
+// We want to pick one of (1) a single-operation filter, (2) a
+// between-filter of some kind.
+// So, that's one of <=, >=, ==, !=, >, <, or
+// one of [<, <], [<, <=], [<=, <=], [<=, <].
+var binaryOps = []string{
+	"<=", ">=", "==", "!=", "<", ">",
+}
+
+func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Int63n(10)
+	// this is unevenly weighted, but there's no Uint64N, and
+	// Int63n can't represent the whole range.
+	v1 := cfg.Rnd.Uint64() % i.Range
+	v2 := cfg.Rnd.Uint64() % i.Range
+	if v1 > v2 {
+		v1, v2 = v2, v1
+	}
+	v1 = v1 + uint64(i.Min)
+	v2 = v2 + uint64(i.Min)
+	var v1s, v2s string
+	if i.Scale != 0 {
+		v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1))/i.ScaleDiv)
+		v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2))/i.ScaleDiv)
+	} else {
+		v1s = strconv.FormatInt(int64(v1), 10)
+		v2s = strconv.FormatInt(int64(v2), 10)
+	}
+	if r < 4 {
+		lte := "<="
+		op1 := lte[:1+(r&1)]
+		op2 := lte[:1+((r>>1)&1)]
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)",
+			v1s, op1, i.Field, op2, v2s)}
+	} else {
+		if cfg.Rnd.Int63n(2) == 1 {
+			v1s = v2s
+		}
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r-4], v1s)}
+	}
+}
+
+// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
+// and spits back a PQL query
+//
+func (cfg *RandomQueryConfig) Setup(api API) (err error) {
+	ctx := context.Background()
+	cfg.Info, err = api.Schema(ctx)
+	if err != nil {
+		return err
+	}
+	foundIntField := false
+	for i, ii := range cfg.Info {
+		_ = i
+		for k, fld := range ii.Fields {
+			_ = k
+			switch fld.Options.Type {
+			case "set", "mutex", "time":
+				pql := fmt.Sprintf("Rows(%v)", fld.Name)
+
+				res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
+				panicOn(err)
+				if cfg.VeryVerbose {
+					fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
+				}
+				// if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers.
+				// e.g.
+				// success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"}
+				// success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}'
+
+				switch x := res.Results[0].(type) {
+				case *pilosa.RowIdentifiers:
+					// internalClient gets this
+					cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time")
+				case pilosa.RowIdentifiers:
+					// test gets this
+					cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time")
+				}
+			case "int":
+				foundIntField = true
+				fallthrough // I bet you thought you'd never see this used
+			case "decimal":
+				cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
+			default:
+				AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
+			}
+		}
+	}
+
+	cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
+	if foundIntField {
+		cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
+	}
+	seed := int64(42)
+	cfg.Rnd = rand.New(rand.NewSource(seed))
+
+	return nil
+}
+
+func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) {
+	for _, rowID := range x.Rows {
+		cfg.AddFeature(index, field, rowID, "", false, hasTime)
+	}
+	for _, rowKey := range x.Keys {
+		cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
+	}
+}
+
+const maxEffectiveRange = 1000000
+
+func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) {
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	if min.Scale != scale || max.Scale != scale {
+		panic(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
+			min.Scale, max.Scale, scale))
+	}
+
+	effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
+	// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
+	// wraps to 0. Anything closer together will be fine. We accept the loss
+	// of accuracy in the range from not representing quite the full value
+	// in that edge case.
+	if effectiveRange == 0 {
+		effectiveRange--
+	}
+
+	// we assume that the Value of the field is already scaled, I guess?
+	newRange := IndexFieldRange{
+		Index:    index,
+		Field:    field,
+		Min:      min.Value,
+		Max:      max.Value,
+		Scale:    scale,
+		ScaleDiv: math.Pow(10, float64(scale)),
+		Range:    effectiveRange,
+	}
+	f.Ranges = append(f.Ranges, newRange)
+	if !decimal {
+		f.Distinctables = append(f.Distinctables, newRange)
+	}
+	// We want to add more values for larger int fields, but the
+	// default KitchenSink field has a range of 1<<64 which would make
+	// it completely dominate weights, so...
+	if effectiveRange > maxEffectiveRange {
+		effectiveRange = maxEffectiveRange
+	}
+	f.RangeWeight += int(effectiveRange)
+}
+
+func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
+
+	tree := cfg.GenTree(index, cfg.TreeDepth)
+	pql = tree.ToPQL()
+
+	// avoid using too much bandwidth, just count the final bitmap.
+	pql = fmt.Sprintf("Count(%v)", pql)
+	return
+}
+
+type Tree struct {
+	Chd  []*Tree
+	S    string
+	Args []string // Extra args to pass after children, such as a field for Distinct.
+}
+
+func (tr *Tree) StringIndent(ind int) (s string) {
+	spc := strings.Repeat("    ", ind)
+	spc1 := strings.Repeat("    ", ind+1)
+	var chds []string
+	leaf := true
+	if len(tr.Chd) == 0 {
+		// leaf
+	} else {
+		leaf = false
+		for _, chd := range tr.Chd {
+			chds = append(chds, chd.StringIndent(ind+1))
+		}
+	}
+	if leaf {
+		s += fmt.Sprintf("%v %v\n", spc1, tr.S)
+	} else {
+		for i, c := range chds {
+			if i == 0 {
+				s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c)
+			} else {
+				s += fmt.Sprintf("%v", c)
+			}
+		}
+	}
+	return
+}
+
+const pilosaTimeFmt = "2006-01-02T15:04"
+
+func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
+	features := cfg.IndexMap[index]
+	if depth == 0 {
+		return features.RandomQuery(cfg)
+	}
+
+	r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
+	f := cfg.BitmapFunc[r]
+	tr = &Tree{S: f}
+	numChild := 2
+	switch f {
+	case "Union", "Intersect", "Xor":
+		numChild = cfg.Rnd.Intn(8) + 2
+	case "Not":
+		numChild = 1
+	case "Difference":
+		numChild = 2
+	case "Distinct":
+		numChild = 1
+		r = cfg.Rnd.Intn(len(features.Distinctables))
+		tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field))
+	}
+	for i := 0; i < numChild; i++ {
+		tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
+	}
+	return
+}
+
+func (tr *Tree) ToPQL() (s string) {
+
+	if len(tr.Chd) == 0 {
+		// leaf
+		return tr.S
+	}
+
+	var chds []string
+	for _, c := range tr.Chd {
+		chds = append(chds, c.ToPQL())
+	}
+	// If we had no extra args, this does nothing.
+	chds = append(chds, tr.Args...)
+	all := strings.Join(chds, ", ")
+	return fmt.Sprintf("%v(%v)", tr.S, all)
+}
+
+func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) {
+
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	f.Slc = append(f.Slc, IndexFieldRow{
+		Index:    index,
+		Field:    field,
+		RowID:    rowID,
+		RowKey:   rowKey,
+		IsRowKey: isRowKey,
+		HasTime:  hasTime,
+	})
+	f.SlcWeight++
+}

cmd/random-query/main_test.go

@@ -0,0 +1,165 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"math/rand"
+	"strconv"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func Test_RandomQuery(t *testing.T) {
+
+	cfg := NewRandomQueryConfig()
+
+	nNodes := 1
+	nReplicas := 1
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	defer c.Close()
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+	_ = dirs
+
+	ctx := context.Background()
+
+	indexes := []string{"rick", "morty"}
+	fieldName := []string{"f", "flying_car"}
+	idx := make([]*pilosa.Index, len(indexes))
+	field := make([]*pilosa.Field, len(indexes))
+
+	var err error
+
+	for i := range indexes {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	timestamp := int64(0)
+
+	for i := range indexes {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, uint64(j))
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the primary and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexes[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+		//vv("rowIDs = '%#v'", rowIDs)
+		//vv("colKeys = '%#v'", colKeys)
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+		//qcx.Reset()
+	}
+	// end of setup.
+
+	panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
+
+	for j := 0; j < 4; j++ {
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		panicOn(err)
+
+		//vv("pql = '%v'", pql)
+
+		// Query node0.
+		res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = res
+		//vv("success on pql = '%v'; res='%v'", pql, res.Results[0])
+	}
+}

cmd/random-query/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("# %s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) int64 {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return 0
+	}
+	return fi.Size()
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

cmd/rbf.go

@@ -0,0 +1,160 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newRBFCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "rbf",
+		Short: "Inspect RBF data files.",
+		Long: `
+Provides a set of commands for inspecting RBF data files.
+`,
+	}
+	cmd.AddCommand(newRBFCheckCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFDumpCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPagesCommand(stdin, stdout, stderr))
+	cmd.AddCommand(newRBFPageCommand(stdin, stdout, stderr))
+	return cmd
+}
+
+func newRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFCheckCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "check [flags] PATH",
+		Short: "Run consistency check on RBF data.",
+		Long: `
+Executes a consistency check on an RBF data directory.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFDumpCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "dump [flags] PATH PGNO [PGNO...]",
+		Short: "Prints RBF raw page data",
+		Long: `
+Dumps the raw hex data for one or more RBF pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}
+
+func newRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPagesCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "pages [flags] PATH",
+		Short: "Prints metadata for the list of all pages",
+		Long: `
+Prints a line for every page in the database with its type/status.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.BoolVar(&c.WithTree, "with-tree", false, "Display b-tree name for each row")
+	return cmd
+}
+
+func newRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
+	c := ctl.NewRBFPageCommand(stdin, stdout, stderr)
+	cmd := &cobra.Command{
+		Use:   "page [flags] PATH PGNO [PGNO...]",
+		Short: "Prints data for a page(s)",
+		Long: `
+Prints the header & cell data for one or more pages.
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) == 1 {
+				return fmt.Errorf("page number required")
+			}
+
+			c.Path = args[0]
+
+			for _, arg := range args[1:] {
+				pgno, err := strconv.Atoi(arg)
+				if err != nil {
+					return errors.New("invalid page number")
+				}
+				c.Pgnos = append(c.Pgnos, uint32(pgno))
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.Run(context.Background())
+		},
+	}
+	return cmd
+}

cmd/root.go

@@ -26,23 +26,18 @@ import (
 )
 
 func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	productName := "Pilosa " + pilosa.Version
-	if pilosa.EnterpriseEnabled {
-		productName = "Pilosa Enterprise " + pilosa.Version
-	}
 	rc := &cobra.Command{
-		Use:   "pilosa",
+		Use: "pilosa",
+		// TODO: These short/long descriptions could use some updating.
 		Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
-		// TODO - is documentation actually there?
 		Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
+tools for administering Pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
 at https://www.pilosa.com/docs/.
 
-` + productName + `
-Build Time: ` + pilosa.BuildTime + "\n",
+` + pilosa.VersionInfo() + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
 			err := setAllConfig(v, cmd.Flags(), "PILOSA")
@@ -74,6 +69,7 @@ Build Time: ` + pilosa.BuildTime + "\n",
 	rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr))
 	rc.AddCommand(newImportCommand(stdin, stdout, stderr))
 	rc.AddCommand(newInspectCommand(stdin, stdout, stderr))
+	rc.AddCommand(newRBFCommand(stdin, stdout, stderr))
 	rc.AddCommand(newServeCmd(stdin, stdout, stderr))
 	rc.AddCommand(newHolderCmd(stdin, stdout, stderr))
 

cmd/root_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/spf13/cobra"
 )
 
@@ -55,6 +56,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +68,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()
@@ -132,7 +136,7 @@ func executeDry(t *testing.T, tests []commandTest) {
 // a temp config file with the cfgFileContent string as its content.
 func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
 	// make config file
-	cfgFile, err := ioutil.TempFile("", "")
+	cfgFile, err := testhook.TempFile(t, "cmdconf")
 	failErr(t, err, "making temp file")
 	_, err = cfgFile.WriteString(ct.cfgFileContent)
 	failErr(t, err, "writing config to temp file")
@@ -145,6 +149,10 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
 		err = os.Setenv(name, val)
 		failErr(t, err, fmt.Sprintf("setting environment variable '%s' to '%s'", name, val))
 	}
+	// address common case where system might have postgres bind
+	// setting that an existing running Pilosa is using (causing test
+	// failures due to port conflict)
+	os.Setenv("PILOSA_POSTGRES_BIND", "")
 
 	// make command and set args
 	rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard)
@@ -173,20 +181,16 @@ func TestRootCommand(t *testing.T) {
 }
 
 func TestRootCommand_Config(t *testing.T) {
-	file, err := ioutil.TempFile("", "test.conf")
+	file, err := testhook.TempFile(t, "test.conf")
 	if err != nil {
-		panic(err)
+		t.Fatalf("creating config file: %v", err)
 	}
 	config := `data-dir = "/tmp/pil5_0"
 bind = "127.0.0.1:10101"
 
 [cluster]
   replicas = 2
-  partitions = 128
-  hosts = [
-    "127.0.0.1:10101",
-    "127.0.0.1:10111",
-  ]`
+  partitions = 128`
 	if _, err := file.Write([]byte(config)); err != nil {
 		t.Fatalf("writing config file: %v", err)
 	}

cmd/server_test.go

@@ -15,13 +15,14 @@
 package cmd_test
 
 import (
-	"io/ioutil"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/cmd"
 	_ "github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/toml"
 	"github.com/pkg/errors"
 )
@@ -34,17 +35,24 @@ func TestServerHelp(t *testing.T) {
 	}
 }
 
+// I have no idea why the linter in ci is complaining about this being unused.
+func nextPort() string { //nolint:unused
+	return fmt.Sprintf(`"localhost:%d"`, 0)
+}
+
 func TestServerConfig(t *testing.T) {
-	actualDataDir, err := ioutil.TempDir("", "")
+	t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
+	actualDataDir, err := testhook.TempDir(t, "")
 	failErr(t, err, "making data dir")
-	logFile, err := ioutil.TempFile("", "")
+	logFile, err := testhook.TempFile(t, "")
 	failErr(t, err, "making log file")
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
+				"PILOSA_LONG_QUERY_TIME":         "1m30s",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
 				"PILOSA_MAX_WRITES_PER_REQUEST":  "2000",
 				"PILOSA_PROFILE_BLOCK_RATE":      "9123",
@@ -52,15 +60,13 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
-	bind = "localhost:0"
+	bind = ` + nextPort() + `
+	bind-grpc = ` + nextPort() + `
 	max-writes-per-request = 3000
+	long-query-time = "1m10s"
 
 	[cluster]
-		disabled = true
 		replicas = 2
-		hosts = [
-			"localhost:19444",
-		]
 		long-query-time = "1m10s"
 	[profile]
 		block-rate = 100
@@ -71,7 +77,7 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.DataDir, actualDataDir)
 				v.Check(cmd.Server.Config.Bind, "localhost:42454")
 				v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
@@ -95,20 +101,15 @@ func TestServerConfig(t *testing.T) {
 				"PILOSA_PROFILE_MUTEX_FRACTION": "444",
 			},
 			cfgFileContent: `
-	bind = "localhost:0"
+	bind = ` + nextPort() + `
+	bind-grpc = ` + nextPort() + `
 	data-dir = "` + actualDataDir + `"
-	[cluster]
-		disabled = true
-		hosts = [
-			"localhost:19444",
-		]
 	[profile]
 		block-rate = 100
 		mutex-fraction = 10
 	`,
 			validation: func() error {
 				v := validator{}
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
 				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
 				v.Check(cmd.Server.Config.Profile.BlockRate, 4832)
@@ -122,11 +123,8 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
-	[cluster]
-		hosts = [
-			"localhost:19444",
-		]
 	[anti-entropy]
 		interval = "11m0s"
 	[metric]
@@ -139,7 +137,6 @@ func TestServerConfig(t *testing.T) {
 	`,
 			validation: func() error {
 				v := validator{}
-				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
 				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
 				v.Check(cmd.Server.Config.LogPath, logFile.Name())
 				v.Check(cmd.Server.Config.Metric.Service, "statsd")
@@ -163,28 +160,114 @@ func TestServerConfig(t *testing.T) {
 
 	// run server tests
 	for i, test := range tests {
-		com := test.setupCommand(t)
-		executed := make(chan struct{})
-		var execErr error
-		go func() {
-			execErr = com.Execute()
-			close(executed)
-		}()
-		select {
-		case <-cmd.Server.Started:
-		case <-executed:
-		}
-		if execErr != nil {
-			t.Fatalf("executing server command: %v", execErr)
-		}
-		err := cmd.Server.Close()
-		failErr(t, err, "closing pilosa server command")
-		<-executed
-		failErr(t, execErr, "executing command")
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			com := test.setupCommand(t)
+			executed := make(chan struct{})
+			var execErr error
+			go func() {
+				execErr = com.Execute()
+				close(executed)
+			}()
+			select {
+			case <-cmd.Server.Started:
+			case <-executed:
+			}
+			if execErr != nil {
+				t.Fatalf("executing server command: %v", execErr)
+			}
+			err := cmd.Server.Close()
+			failErr(t, err, "closing pilosa server command")
+			<-executed
+			failErr(t, execErr, "executing command")
 
-		if err := test.validation(); err != nil {
-			t.Fatalf("Failed test %d due to: %v", i, err)
-		}
-		test.reset()
+			if err := test.validation(); err != nil {
+				t.Fatalf("Failed test %d due to: %v", i, err)
+			}
+			test.reset()
+		})
+	}
+}
+func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
+	t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
+	actualDataDir, err := testhook.TempDir(t, "")
+	failErr(t, err, "making data dir")
+
+	tests := []commandTest{
+		// TEST 0
+		{
+			args: []string{"server", "--long-query-time", "1m10s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+             	data-dir = "` + actualDataDir + `"
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*70))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*70))
+				return v.Error()
+			},
+		},
+		// TEST 1
+		{
+			args: []string{"server", "--cluster.long-query-time", "1m20s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*80))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*80))
+				return v.Error()
+			},
+		},
+		// TEST 2: Use old value if both are provided because it is the simplest implementation
+		{
+			args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
+			env:  map[string]string{},
+			cfgFileContent: `
+            	bind = ` + nextPort() + `
+            	bind-grpc = ` + nextPort() + `
+`,
+			validation: func() error {
+				v := validator{}
+				v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*50))
+				v.Check(toml.Duration(cmd.Server.Config.Cluster.LongQueryTime), toml.Duration(time.Second*90))
+				v.Check(toml.Duration(cmd.Server.API.LongQueryTime()), toml.Duration(time.Second*90))
+				return v.Error()
+			},
+		},
+	}
+
+	// run server tests
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			com := test.setupCommand(t)
+			executed := make(chan struct{})
+			var execErr error
+			go func() {
+				execErr = com.Execute()
+				close(executed)
+			}()
+			select {
+			case <-cmd.Server.Started:
+			case <-executed:
+			}
+			if execErr != nil {
+				t.Fatalf("executing server command: %v", execErr)
+			}
+			err := cmd.Server.Close()
+			failErr(t, err, "closing pilosa server command")
+			<-executed
+			failErr(t, execErr, "executing command")
+
+			if err := test.validation(); err != nil {
+				t.Fatalf("Failed test %d due to: %v", i, err)
+			}
+			test.reset()
+		})
 	}
 }

cmd/slurp/slurp.go

@@ -0,0 +1,262 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	pnet "github.com/pilosa/pilosa/v2/net"
+)
+
+// slurp: slurp is a load-tester for importing bulk data.
+//        It allows us to measure write performance.
+
+type stateMachine struct {
+	viewData  map[string][]byte
+	lastIndex string
+	lastField string
+	lastShard uint64
+	state     string
+	client    *http.InternalClient
+	start     time.Time
+
+	profile string
+	host    string
+}
+
+func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
+	parts := strings.Split(h.Name, "/")
+	switch parts[0] {
+	case "roaring":
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		panicOn(err)
+		if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return err
+		}
+		if _, already := r.viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		r.viewData[view] = roaringData
+		r.lastIndex = index
+		r.lastField = field
+		r.lastShard = shard
+	case "bolt":
+		if r.state == "roaring" {
+			err := r.Upload()
+			if err != nil {
+				return err
+			}
+			vv("Finished import %v", time.Since(r.start))
+
+			if r.profile != "" {
+				stopProfile(r.host, r.profile)
+			}
+		}
+		//
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+
+		switch v := parts[len(parts)-1]; v {
+		case "keys":
+			index := parts[1]
+			fieldName := parts[2]
+			if fieldName == "_keys" {
+				//skip index keys are not not real fields so will have no need for field keys
+				return nil
+
+			}
+
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
+			if err != nil {
+				return err
+			}
+		default:
+			pilosa.VV("%v", h.Name)
+			index := parts[1]
+			partition, err := strconv.ParseUint(v, 10, 64)
+			if err != nil {
+				return err
+			}
+			byteData, err := ioutil.ReadAll(tr)
+			panicOn(err)
+
+			br := bytes.NewReader(byteData)
+			err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	r.state = parts[0]
+	return nil
+}
+func (r *stateMachine) Upload() error {
+	if len(r.viewData) > 0 {
+		request := &pilosa.ImportRoaringRequest{
+			Views: r.viewData,
+		}
+		uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
+		err := r.client.ImportRoaring(context.Background(), uri, r.lastIndex, r.lastField, r.lastShard, false, request)
+		if err != nil {
+			return err
+		}
+		r.viewData = make(map[string][]byte)
+	}
+	return nil
+}
+
+func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error {
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	runner := &stateMachine{
+		viewData: make(map[string][]byte),
+		start:    time.Now(),
+		profile:  profile,
+		host:     host,
+	}
+	runner.client = client
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			_ = runner.Upload()
+			break
+		}
+		if err != nil {
+			panicOn(err)
+		}
+		err = runner.NewHeader(header, tarReader)
+		panicOn(err)
+	}
+	return nil
+}
+
+func main() {
+	var host string
+	var profile string
+	var tarSrcPath string
+	flag.StringVar(&host, "host", "127.0.0.1:10101", "host to import into")
+	flag.StringVar(&profile, "profile", "", "profile and save a cpu profile of the import to this file")
+	flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import")
+	flag.Parse()
+
+	uri, err := pnet.NewURIFromAddress(host)
+	panicOn(err)
+	globURI = uri
+
+	h := &gohttp.Client{}
+	if profile != "" {
+		startProfile(host)
+	}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	t0 := time.Now()
+	println("uploading", tarSrcPath)
+	panicOn(UploadTar(tarSrcPath, c, profile, host))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+func startProfile(host string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/start",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+}
+
+func stopProfile(host, outfile string) {
+	cli := &gohttp.Client{}
+	req := &gohttp.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   host,
+			Path:   "/cpu-profile/stop",
+		},
+	}
+	resp, err := cli.Do(req)
+	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		panic(err)
+	}
+
+	fd, err := os.Create(outfile)
+	panicOn(err)
+	defer fd.Close()
+	_, err = io.Copy(fd, resp.Body)
+	panicOn(err)
+
+}
+
+var globURI *pnet.URI
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
+	return globURI
+}

cmd/slurp/vprint.go

@@ -0,0 +1,177 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize

const_amd64.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,11 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-// +build enterprise
 
-package server
+// +build amd64
 
-import (
-	_ "github.com/pilosa/pilosa/v2/enterprise"
-)
+package pilosa
+
+const TxInitialMmapSize = 4 << 30 // 4GB

const_other.go

@@ -0,0 +1,21 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !amd64
+
+package pilosa
+
+// this is a stubbed out file to let 386/arm build.
+
+const TxInitialMmapSize = 1 << 30 // 1GB

ctl/check_test.go

@@ -16,20 +16,22 @@ package ctl
 
 import (
 	"bytes"
-	"encoding/hex"
 	"io"
-	"io/ioutil"
-	"math/rand"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
 	"context"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestCheckCommand_RunCacheFile(t *testing.T) {
-	cacheFile := TempFileName("test", ".cache")
+	fi, err := testhook.TempFile(t, "test*.cache")
+	if err != nil {
+		t.Fatalf("creating test file: %v", err)
+	}
+	cacheFile := fi.Name()
 
 	rder := []byte{}
 	stdin := bytes.NewReader(rder)
@@ -37,7 +39,7 @@ func TestCheckCommand_RunCacheFile(t *testing.T) {
 	cm := NewCheckCommand(stdin, w, w)
 	cm.Paths = []string{cacheFile}
 
-	err := cm.Run(context.Background())
+	err = cm.Run(context.Background())
 	w.Close()
 	var buf bytes.Buffer
 	if _, err := io.Copy(&buf, r); err != nil {
@@ -50,7 +52,11 @@ func TestCheckCommand_RunCacheFile(t *testing.T) {
 }
 
 func TestCheckCommand_RunSnapshot(t *testing.T) {
-	snapshotFile := TempFileName("test", ".snapshotting")
+	fi, err := testhook.TempFile(t, "test*.snapshotting")
+	if err != nil {
+		t.Fatalf("creating test file: %v", err)
+	}
+	snapshotFile := fi.Name()
 
 	rder := []byte{}
 	stdin := bytes.NewReader(rder)
@@ -58,7 +64,7 @@ func TestCheckCommand_RunSnapshot(t *testing.T) {
 	cm := NewCheckCommand(stdin, w, w)
 	cm.Paths = []string{snapshotFile}
 
-	err := cm.Run(context.Background())
+	err = cm.Run(context.Background())
 	w.Close()
 	var buf bytes.Buffer
 	if _, err := io.Copy(&buf, r); err != nil {
@@ -71,10 +77,11 @@ func TestCheckCommand_RunSnapshot(t *testing.T) {
 }
 
 func TestCheckCommand_Run(t *testing.T) {
-	file, err := ioutil.TempFile("", "")
+	file, err := testhook.TempFile(t, "run-command")
 	if err != nil {
 		t.Fatal(err)
 	}
+	fname := file.Name()
 	if _, err := file.Write([]byte("1234,1223")); err != nil {
 		t.Fatalf("writing to temp file: %v", err)
 	}
@@ -84,7 +91,7 @@ func TestCheckCommand_Run(t *testing.T) {
 	stdin := bytes.NewReader(rder)
 	r, w, _ := os.Pipe()
 	cm := NewCheckCommand(stdin, w, w)
-	cm.Paths = []string{file.Name()}
+	cm.Paths = []string{fname}
 
 	err = cm.Run(context.Background())
 	w.Close()
@@ -99,10 +106,3 @@ func TestCheckCommand_Run(t *testing.T) {
 	}
 	//	Todo: need correct roaring file for happy path
 }
-
-// TempFileName generates a temporary filename with extension
-func TempFileName(prefix, suffix string) string {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix)
-}

ctl/common.go

@@ -31,12 +31,12 @@ type CommandWithTLSSupport interface {
 }
 
 // SetTLSConfig creates common TLS flags
-func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
-	flags.StringVarP(certificatePath, "tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
-	flags.StringVarP(certificateKeyPath, "tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
-	flags.StringVarP(caCertPath, "tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
-	flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
-	flags.BoolVarP(enableClientVerification, "tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
+func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, certificateKeyPath *string, caCertPath *string, skipVerify *bool, enableClientVerification *bool) {
+	flags.StringVarP(certificatePath, prefix+"tls.certificate", "", "", "TLS certificate path (usually has the .crt or .pem extension)")
+	flags.StringVarP(certificateKeyPath, prefix+"tls.key", "", "", "TLS certificate key path (usually has the .key extension)")
+	flags.StringVarP(caCertPath, prefix+"tls.ca-certificate", "", "", "TLS CA certificate path (usually has the .pem extension)")
+	flags.BoolVarP(skipVerify, prefix+"tls.skip-verify", "", false, "Skip TLS certificate server verification (not secure)")
+	flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
 }
 
 // commandClient returns a pilosa.InternalHTTPClient for the command

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -257,7 +261,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, useColumnKeys, useRowK
 func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowKeys bool, bits []pilosa.Bit) error {
 	logger := log.New(cmd.Stderr, "", log.LstdFlags)
 
-	// If keys are used, all bits are sent to the primary translate store (i.e. coordinator).
+	// If keys are used, all bits are sent to the primary translate store.
 	if useColumnKeys || useRowKeys {
 		logger.Printf("importing keys: n=%d", len(bits))
 		if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
-				return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-			}
+			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
 
-		// Parse FieldValue.
-		value, err := strconv.ParseInt(record[1], 10, 64)
-		if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
-		}
-		val.Value = value
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
 			}
-			a = a[:0]
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
+			value, err := strconv.ParseInt(record[1], 10, 64)
+			if err != nil {
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
+			}
+			req.Values = append(req.Values, value)
+		}
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,9 +25,11 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestImportCommand_Validation(t *testing.T) {
@@ -57,7 +59,7 @@ func TestImportCommand_Basic(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import.csv")
+		file, err := testhook.TempFile(t, "import.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -72,7 +74,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -89,7 +91,7 @@ func TestImportCommand_Basic(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import.csv")
+		file, err := testhook.TempFile(t, "import.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -101,7 +103,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -122,7 +124,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import-value.csv")
+		file, err := testhook.TempFile(t, "import-value.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -134,7 +136,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -161,7 +163,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 		buf := bytes.Buffer{}
 		stdin, stdout, stderr := GetIO(buf)
 		cm := NewImportCommand(stdin, stdout, stderr)
-		file, err := ioutil.TempFile("", "import-value.csv")
+		file, err := testhook.TempFile(t, "import-value.csv")
 		if err != nil {
 			t.Fatalf("creating tempfile: %v", err)
 		}
@@ -176,7 +178,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -206,7 +208,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -218,7 +220,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -246,7 +248,7 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -270,8 +272,8 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	c := test.MustRunCluster(t, 2)
 	defer c.Close()
-	cmd0 := c[0]
-	cmd1 := c[1]
+	cmd0 := c.GetNode(0)
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -299,20 +301,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
-		qry := "Count(Row(f=foo0))"
-		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
-		if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
-		}
+		if err := test.RetryUntil(2*time.Second, func() error {
+			qry := "Count(Row(f=foo0))"
+			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
+			if err != nil {
+				return fmt.Errorf("Querying data for validation: %s", err)
+			}
 
-		// Read body and unmarshal response.
-		exp := `{"results":[100]}` + "\n"
-		if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
-		} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+			// Read body and unmarshal response.
+			exp := `{"results":[100]}` + "\n"
+			if body, err := ioutil.ReadAll(resp.Body); err != nil {
+				return fmt.Errorf("reading: %s", err)
+			} else if !reflect.DeepEqual(body, []byte(exp)) {
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.
@@ -320,7 +328,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-key.csv")
+	file, err := testhook.TempFile(t, "import-key.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -332,7 +340,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -358,7 +366,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -366,7 +374,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 	cm.Host = cmd.API.Node().URI.HostPort()
 	cm.Index = "i"
 	cm.Field = "f"
-	file, err := ioutil.TempFile("", "import.csv")
+	file, err := testhook.TempFile(t, "import.csv")
 	if err != nil {
 		t.Fatalf("creating tempfile: %v", err)
 	}
@@ -380,7 +388,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid row id on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatalf("creating tempfile: %v", err)
 	}
@@ -394,7 +402,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid column id on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -408,7 +416,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
 		t.Fatalf("expect error: invalid timestamp on row, actual: %s", err)
 	}
 
-	file, err = ioutil.TempFile("", "import1.csv")
+	file, err = testhook.TempFile(t, "import1.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -446,12 +454,12 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
 	cm := NewImportCommand(stdin, stdout, stderr)
-	file, err := ioutil.TempFile("", "import-value.csv")
+	file, err := testhook.TempFile(t, "import-value.csv")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -483,7 +491,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	}
 
 	file.Close()
-	file, err = ioutil.TempFile("", "import-value2.csv")
+	file, err = testhook.TempFile(t, "import-value2.csv")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -498,7 +506,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	}
 
 	file.Close()
-	file, err = ioutil.TempFile("", "import-value3.csv")
+	file, err = testhook.TempFile(t, "import-value3.csv")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -522,7 +530,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -540,7 +548,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 	cm.Field = "f"
 
 	t.Run("Valid", func(t *testing.T) {
-		file, err := ioutil.TempFile("", "import-bool.csv")
+		file, err := testhook.TempFile(t, "import-bool.csv")
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -558,7 +566,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	// Ensure that invalid bool values return an error.
 	t.Run("Invalid", func(t *testing.T) {
-		file, err := ioutil.TempFile("", "import-invalid-bool.csv")
+		file, err := testhook.TempFile(t, "import-invalid-bool.csv")
 		if err != nil {
 			t.Fatal(err)
 		}

ctl/inspect.go

@@ -16,15 +16,23 @@ package ctl
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
+	"hash/fnv"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
-	"unsafe"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -33,6 +41,12 @@ import (
 type InspectCommand struct {
 	// Path to data file
 	Path string
+	// don't list details of objects
+	Quiet bool
+	// list only this many objects
+	Max int
+	// Filters:
+	InspectOpts pilosa.InspectRequest
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
+type pointerContext struct {
+	from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(cmd.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+		if cmd.Max != 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(cmd.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+		if cmd.Max > 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
 // Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
 	if err != nil {
@@ -58,7 +183,172 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "statting file")
 	}
+	if fi.IsDir() {
+		total := 0
+		infos, err := f.Readdir(0)
+		if err != nil {
+			return err
+		}
+		if len(infos) == 0 {
+			return errors.New("directory contains no files")
+		}
 
+		names := make([]string, len(infos))
+		nameToInfo := make(map[string]os.FileInfo, len(infos))
+		// find numeric-only names; we'll operate on
+		// either those, or the whole holder if we find
+		// a .topology file.
+		n := 0
+		for _, fi := range infos {
+			name := fi.Name()
+			if name == ".topology" {
+				return cmd.InspectHolder(ctx, cmd.Path)
+			}
+			if _, err := strconv.Atoi(name); err == nil {
+				names[n] = name
+				nameToInfo[name] = fi
+				n++
+			}
+		}
+		if n == 0 {
+			return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+		}
+		names = names[:n]
+		fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+		for _, name := range names {
+			f2, err := os.Open(filepath.Join(cmd.Path, name))
+			if err != nil {
+				return fmt.Errorf("opening %q: %v", name, err)
+			}
+			fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+			err = cmd.InspectFile(f2, nameToInfo[name])
+			total++
+			f2.Close()
+			if err != nil {
+				return fmt.Errorf("inspecting %q: %v", name, err)
+			}
+		}
+		return nil
+	}
+	return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology internal.Topology, myID string, err error) {
+	buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, errors.Wrap(err, "reading file")
+	}
+	if err := proto.Unmarshal(buf, &topology); err != nil {
+		return topology, myID, errors.Wrap(err, "unmarshalling")
+	}
+	sort.Slice(topology.NodeIDs,
+		func(i, j int) bool {
+			return topology.NodeIDs[i] < topology.NodeIDs[j]
+		})
+	buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, nil
+	}
+	myID = strings.TrimSpace(string(buf))
+	return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+	var shardMap map[uint64]int
+	var ok bool
+	if shardMap, ok = partitions[index]; !ok {
+		shardMap = make(map[uint64]int)
+		partitions[index] = shardMap
+	}
+	if partition, ok = shardMap[shard]; !ok {
+		var buf [8]byte
+		binary.BigEndian.PutUint64(buf[:], shard)
+
+		// Hash the bytes and mod by partition count.
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write(buf[:])
+		partition = int(h.Sum64() % uint64(partitionN))
+		shardMap[shard] = partition
+	}
+	return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+	parts := strings.Split(path, "/")
+	shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+	holder := pilosa.NewHolder(path, nil)
+	holder.Opts.Inspect = true
+	holder.Opts.ReadOnly = true
+	err := holder.Open()
+	if err != nil {
+		return fmt.Errorf("%s: holder open: %v", path, err)
+	}
+	holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+	if err != nil {
+		return fmt.Errorf("%s: inspect: %v", path, err)
+	}
+	myPartition := 0
+	topology, myID, err := loadTopology(path)
+	if err == nil {
+		fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+		if len(topology.NodeIDs) > 1 {
+			fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+		}
+		found := false
+		for i := range topology.NodeIDs {
+			if topology.NodeIDs[i] == myID {
+				found = true
+				myPartition = i
+				break
+			}
+		}
+		if !found {
+			fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+		}
+	} else {
+		fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+	}
+	for _, name := range holderInfo.FragmentNames {
+		partition, err := findPartitionPath(name, len(topology.NodeIDs))
+		if err != nil {
+			fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+		} else {
+			if partition == myPartition {
+				fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+			} else {
+				fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+			}
+		}
+		details := holderInfo.FragmentInfo[name]
+		cmd.DisplayInfo(details.BitmapInfo)
+		if details.BlockChecksums != nil {
+			fmt.Fprintf(cmd.Stdout, "  Checksums [%d total]:\n", len(details.BlockChecksums))
+			for _, block := range details.BlockChecksums {
+				fmt.Fprintf(cmd.Stdout, "   %8d: %x\n", block.ID, block.Checksum)
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
 	// Memory map the file.
 	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
@@ -72,39 +362,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	}()
 	// Attach the mmap file to the bitmap.
 	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
-	bm := roaring.NewBitmap()
-	if err := bm.UnmarshalBinary(data); err != nil {
-		return errors.Wrap(err, "unmarshalling")
+	fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
+	var info roaring.BitmapInfo
+	_, _, err = roaring.InspectBinary(data, true, &info)
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	cmd.DisplayInfo(info)
+	if err != nil {
+		return errors.Wrap(err, "inspecting")
 	}
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	return nil
+}
 
-	// Retrieve stats.
-	t = time.Now()
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
-	info := bm.Info()
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
+	pC := pointerContext{
+		from: info.From,
+		to:   info.To,
+	}
 
 	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintf(cmd.Stdout, "  Bitmap Info:\n")
+	fmt.Fprintf(cmd.Stdout, "    Bits: %d\n", info.BitCount)
+	fmt.Fprintf(cmd.Stdout, "    Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Operations: %d (%d bits)\n", info.Ops, info.OpN)
 	fmt.Fprintln(cmd.Stdout, "")
 
 	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
-	for _, ci := range info.Containers {
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
-			ci.Key,
-			ci.Type,
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
+	if !cmd.Quiet {
+		if info.ContainerCount > 0 {
+			cmd.PrintContainers(info, pC)
+		}
+		if info.Ops > 0 {
+			cmd.PrintOps(info)
+		}
 	}
-	tw.Flush()
-
-	return nil
 }

ctl/inspect_test.go

@@ -18,10 +18,11 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestInspectCommand_Run(t *testing.T) {
@@ -30,7 +31,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	r, w, _ := os.Pipe()
 
 	cm := NewInspectCommand(stdin, w, w)
-	file, err := ioutil.TempFile("", "inspectTest")
+	file, err := testhook.TempFile(t, "inspectTest")
 	if err != nil {
 		t.Fatalf("Error creating tempfile: %s", err)
 	}
@@ -41,7 +42,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	file.Close()
 	cm.Path = file.Name()
 	err = cm.Run(context.Background())
-	expectedError := "unmarshalling: "
+	expectedError := "inspecting: "
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Fatalf("expected error '%s', got '%v'", expectedError, err)
 	}
@@ -52,7 +53,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	if err != nil {
 		t.Fatalf("copying data: %v", err)
 	}
-	if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+	if !strings.Contains(buf.String(), "inspecting bitmap...") {
 		t.Fatalf("Inspect doesn't work: %s", err)
 	}
 

ctl/rbf_check.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFCheckCommand represents a command for running a consistency check on RBF.
+type RBFCheckCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFCheckCommand returns a new instance of RBFCheckCommand.
+func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckCommand {
+	return &RBFCheckCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Run check on the database.
+	if err := db.Check(); err != nil {
+		return err
+	}
+
+	// If successful, print a success message.
+	fmt.Fprintln(cmd.Stdout, "ok")
+
+	return nil
+}

ctl/rbf_dump.go

@@ -0,0 +1,111 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFDumpCommand represents a command for dumping raw data for an RBF page.
+type RBFDumpCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFDumpCommand returns a new instance of RBFDumpCommand.
+func NewRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFDumpCommand {
+	return &RBFDumpCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFDumpCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch each page & dump.
+	for _, pgno := range cmd.Pgnos {
+		buf, err := tx.PageData(pgno)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(cmd.Stdout, "## PAGE %d\n", pgno)
+		fmt.Fprintln(cmd.Stdout, compressedHexDump(buf))
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func compressedHexDump(b []byte) string {
+	const prefixN = len("00000000")
+
+	var output []string
+	var prev string
+	var ellipsis bool
+
+	lines := strings.Split(strings.TrimSpace(hex.Dump(b)), "\n")
+	for i, line := range lines {
+		// Add line to output if it is not repeating or the last line.
+		if i == 0 || i == len(lines)-1 || trimPrefixN(line, prefixN) != trimPrefixN(prev, prefixN) {
+			output = append(output, line)
+			prev, ellipsis = line, false
+			continue
+		}
+
+		// Add an ellipsis for the first duplicate line.
+		if !ellipsis {
+			output = append(output, "...")
+			ellipsis = true
+			continue
+		}
+	}
+
+	return strings.Join(output, "\n")
+}
+
+// trimPrefixN trims n bytes from the beginning of a string.
+func trimPrefixN(s string, n int) string {
+	if len(s) < n {
+		return ""
+	}
+	return s[n:]
+}

ctl/rbf_page.go

@@ -0,0 +1,140 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+)
+
+// RBFPageCommand represents a command for printing data for a single RBF page.
+type RBFPageCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Page numbers to print.
+	Pgnos []uint32
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPageCommand returns a new instance of RBFPageCommand.
+func NewRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPageCommand {
+	return &RBFPageCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPageCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Fetch the page.
+	pages, err := tx.Pages(cmd.Pgnos)
+	if err != nil {
+		return err
+	}
+
+	for _, page := range pages {
+		switch page := page.(type) {
+		case *rbf.MetaPage:
+			cmd.printMetaPage(page)
+		case *rbf.RootRecordPage:
+			cmd.printRootRecordPage(page)
+		case *rbf.LeafPage:
+			cmd.printLeafPage(page)
+		case *rbf.BranchPage:
+			cmd.printBranchPage(page)
+		case *rbf.BitmapPage:
+			cmd.printBitmapPage(page)
+		case *rbf.FreePage:
+			cmd.printFreePage(page)
+		default:
+			return fmt.Errorf("unexpected page type %T", page)
+		}
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+
+	return nil
+}
+
+func (cmd *RBFPageCommand) printMetaPage(page *rbf.MetaPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: meta\n")
+	fmt.Fprintf(cmd.Stdout, "PageN: %d\n", page.PageN)
+	fmt.Fprintf(cmd.Stdout, "WALID: %d\n", page.WALID)
+	fmt.Fprintf(cmd.Stdout, "Root Record Pgno: %d\n", page.RootRecordPageNo)
+	fmt.Fprintf(cmd.Stdout, "Freelist Pgno: %d\n", page.FreelistPageNo)
+}
+
+func (cmd *RBFPageCommand) printRootRecordPage(page *rbf.RootRecordPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: root record\n")
+	fmt.Fprintf(cmd.Stdout, "Next: %d\n", page.Next)
+	fmt.Fprintf(cmd.Stdout, "Records: n=%d\n", len(page.Records))
+	for i, rec := range page.Records {
+		fmt.Fprintf(cmd.Stdout, "[%d]: name=%q pgno=%d\n", i, rec.Name, rec.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printLeafPage(page *rbf.LeafPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: leaf\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		if cell.Type == rbf.ContainerTypeBitmapPtr {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s pgno=%d\n", i, cell.Key, cell.Type, cell.Pgno)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "[%d]: key=%d type=%s values=%v\n", i, cell.Key, cell.Type, cell.Values)
+		}
+	}
+}
+
+func (cmd *RBFPageCommand) printBranchPage(page *rbf.BranchPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: branch\n")
+	fmt.Fprintf(cmd.Stdout, "Cells: n=%d\n", len(page.Cells))
+	for i, cell := range page.Cells {
+		fmt.Fprintf(cmd.Stdout, "[%d]: key=%d flags=%d pgno=%d\n", i, cell.Key, cell.Flags, cell.Pgno)
+	}
+}
+
+func (cmd *RBFPageCommand) printBitmapPage(page *rbf.BitmapPage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: bitmap\n")
+	fmt.Fprintf(cmd.Stdout, "Values: %v\n", page.Values)
+}
+
+func (cmd *RBFPageCommand) printFreePage(page *rbf.FreePage) {
+	fmt.Fprintf(cmd.Stdout, "Pgno: %d\n", page.Pgno)
+	fmt.Fprintf(cmd.Stdout, "Type: free\n")
+}

ctl/rbf_pages.go

@@ -0,0 +1,149 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/txkey"
+)
+
+// RBFPagesCommand represents a command for printing a list of RBF page metadata.
+type RBFPagesCommand struct {
+	// Filepath to the RBF database.
+	Path string
+
+	// Print the b-tree key with each row.
+	WithTree bool
+
+	// Standard input/output
+	*pilosa.CmdIO
+}
+
+// NewRBFPagesCommand returns a new instance of RBFPagesCommand.
+func NewRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFPagesCommand {
+	return &RBFPagesCommand{
+		CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
+	}
+}
+
+// Run executes the export.
+func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
+	// Open database.
+	db := rbf.NewDB(cmd.Path, nil)
+	if err := db.Open(); err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Execute with a transaction.
+	tx, err := db.Begin(false)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Iterate over each page and grab info.
+	infos, err := tx.PageInfos()
+	if err != nil {
+		return err
+	}
+
+	// Write header.
+	fmt.Fprint(cmd.Stdout, "ID       ")
+	fmt.Fprint(cmd.Stdout, "TYPE       ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "TREE                           ")
+	}
+	fmt.Fprintln(cmd.Stdout, "EXTRA")
+
+	fmt.Fprint(cmd.Stdout, "======== ")
+	fmt.Fprint(cmd.Stdout, "========== ")
+	if cmd.WithTree {
+		fmt.Fprint(cmd.Stdout, "============================== ")
+	}
+	fmt.Fprintln(cmd.Stdout, "====================")
+
+	// Print one line for each page.
+	for pgno, info := range infos {
+		switch info := info.(type) {
+		case *rbf.MetaPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "meta")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo)
+
+		case *rbf.RootRecordPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "rootrec")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "next=%d\n", info.Next)
+
+		case *rbf.LeafPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "leaf")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BranchPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "branch")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "flags=x%x,celln=%d\n", info.Flags, info.CellN)
+
+		case *rbf.BitmapPageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "bitmap")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", prefixToString(info.Tree))
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		case *rbf.FreePageInfo:
+			fmt.Fprintf(cmd.Stdout, "%-8d ", pgno)
+			fmt.Fprintf(cmd.Stdout, "%-10s ", "free")
+			if cmd.WithTree {
+				fmt.Fprintf(cmd.Stdout, "%-30q ", "")
+			}
+			fmt.Fprintf(cmd.Stdout, "-\n")
+
+		default:
+			panic(fmt.Sprintf("unexpected page info type %T", info))
+		}
+	}
+
+	return nil
+}
+
+func prefixToString(s string) (ret string) {
+	defer func() {
+		if err := recover(); err != nil {
+			ret = s
+		}
+	}()
+	return txkey.PrefixToString([]byte(s))
+}

ctl/server.go

@@ -15,72 +15,96 @@
 package ctl
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/storage"
 	"github.com/spf13/cobra"
 )
 
 // BuildServerFlags attaches a set of flags to the command for a server instance.
 func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
+	flags.StringVar(&srv.Config.Name, "name", srv.Config.Name, "Name of the node in the cluster.")
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
-	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
+	flags.StringVar(&srv.Config.AdvertiseGRPC, "advertise-grpc", srv.Config.AdvertiseGRPC, "Address to advertise externally for gRPC.")
+	flags.IntVar(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
 	flags.Uint64Var(&srv.Config.MaxMapCount, "max-map-count", srv.Config.MaxMapCount, "Limits the maximum number of active mmaps. Pilosa will fall back to reading files once this is exhausted. Set below your system's vm.max_map_count.")
 	flags.Uint64Var(&srv.Config.MaxFileCount, "max-file-count", srv.Config.MaxFileCount, "Soft limit on the maximum number of fragment files Pilosa keeps open simultaneously.")
+	flags.DurationVar((*time.Duration)(&srv.Config.LongQueryTime), "long-query-time", time.Duration(srv.Config.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
+	flags.IntVar(&srv.Config.QueryHistoryLength, "query-history-length", srv.Config.QueryHistoryLength, "Number of queries to remember in history.")
 
 	// TLS
-	SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
+	SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
 
 	// Handler
-	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).")
+	flags.StringSliceVar(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
 
 	// Cluster
-	flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
-	flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
-	flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
-	flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
+	flags.IntVar(&srv.Config.Cluster.ReplicaN, "cluster.replicas", 1, "Number of hosts each piece of data should be stored on.")
+	flags.DurationVar((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", time.Duration(srv.Config.Cluster.LongQueryTime), "RENAMED TO 'long-query-time': Duration that will trigger log and stat messages for slow queries.") // negative duration indicates invalid value because 0 is meaningful
+	flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.")
 
 	// Translation
-	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
-	flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
+	flags.StringVar(&srv.Config.Translation.PrimaryURL, "translation.primary-url", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
+	flags.IntVar(&srv.Config.Translation.MapSize, "translation.map-size", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
 
-	// Gossip
-	flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
-	flags.StringVarP(&srv.Config.Gossip.AdvertiseHost, "gossip.advertise-host", "", srv.Config.Gossip.AdvertiseHost, "Host on which memberlist should advertise.")
-	flags.StringVarP(&srv.Config.Gossip.AdvertisePort, "gossip.advertise-port", "", srv.Config.Gossip.AdvertisePort, "Port on which memberlist should advertise.")
-
-	flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
-	flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
-	flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
-	flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
+	// Etcd
+	// Etcd.Name used Config.Name for it's value.
+	// Etcd.Dir defaults to a directory under the pilosa data directory.
+	flags.StringVar(&srv.Config.Etcd.LClientURL, "etcd.listen-client-address", srv.Config.Etcd.LClientURL, "Listen client address.")
+	flags.StringVar(&srv.Config.Etcd.AClientURL, "etcd.advertise-client-address", srv.Config.Etcd.AClientURL, "Advertise client address. If not provided, uses the listen client address.")
+	flags.StringVar(&srv.Config.Etcd.LPeerURL, "etcd.listen-peer-address", srv.Config.Etcd.LPeerURL, "Listen peer address.")
+	flags.StringVar(&srv.Config.Etcd.APeerURL, "etcd.advertise-peer-address", srv.Config.Etcd.APeerURL, "Advertise peer address. If not provided, uses the listen peer address.")
+	flags.StringVar(&srv.Config.Etcd.ClusterURL, "etcd.cluster-url", srv.Config.Etcd.ClusterURL, "Cluster URL to join.")
+	// Etcd.ClusterName uses Cluster.Name for its value.
+	flags.StringVar(&srv.Config.Etcd.InitCluster, "etcd.initial-cluster", srv.Config.Etcd.InitCluster, "Initial cluster name1=apurl1,name2=apurl2")
 
 	// AntiEntropy
-	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
+	flags.DurationVar((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
-	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
-	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
-	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
+	flags.StringVar(&srv.Config.Metric.Service, "metric.service", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
+	flags.StringVar(&srv.Config.Metric.Host, "metric.host", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
+	flags.DurationVar((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
+	flags.BoolVar((&srv.Config.Metric.Diagnostics), "metric.diagnostics", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
-	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
-	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
+	flags.StringVar(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
+	flags.StringVar(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
+	flags.Float64Var(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling
 	flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
 	flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
+
+	// Storage
+	// Note: the default for --storage.backend must be kept "" empty string.
+	// Otherwise we cannot detect and honor the PILOSA_STORAGE_BACKEND env var
+	// over-ride.
+	// TODO: the comment above was carried over from the PILOSA_TXSRC flag, but
+	// we should confirm that this still applies.
+	flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
+	flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk")
+
+	// RowcacheOn
+	flags.BoolVar((&srv.Config.RowcacheOn), "rowcache-on", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)")
+
+	// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.
+	srv.Config.RBFConfig.DefineFlags(flags)
+
+	// Postgres endpoint
+	flags.StringVar(&srv.Config.Postgres.Bind, "postgres.bind", srv.Config.Postgres.Bind, "Address to which to bind a postgres endpoint (leave blank to disable)")
+	SetTLSConfig(flags, "postgres.", &srv.Config.Postgres.TLS.CertificatePath, &srv.Config.Postgres.TLS.CertificateKeyPath, &srv.Config.Postgres.TLS.CACertPath, &srv.Config.Postgres.TLS.SkipVerify, &srv.Config.Postgres.TLS.EnableClientVerification)
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.StartupTimeout), "postgres.startup-timeout", time.Duration(srv.Config.Postgres.StartupTimeout), "Timeout for postgres connection startup. (set 0 to disable)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.ReadTimeout), "postgres.read-timeout", time.Duration(srv.Config.Postgres.ReadTimeout), "Timeout for reads on a postgres connection. (set 0 to disable; does not include connection idling)")
+	flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
+	flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
+	flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
 }

dbshard_internal_test.go

@@ -0,0 +1,369 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/rbf"
+	"github.com/pilosa/pilosa/v2/shardwidth"
+	txkey "github.com/pilosa/pilosa/v2/short_txkey"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// Shard per db evaluation
+func TestShardPerDB_SetBit(t *testing.T) {
+	f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
+	_ = idx
+	defer f.Clean(t)
+
+	// Set bits on the fragment.
+	if _, err := f.setBit(tx, 120, 1); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 120, 6); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(tx, 121, 0); err != nil {
+		t.Fatal(err)
+	}
+	// should have two containers set in the fragment.
+
+	// Verify counts on rows.
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count: %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count: %d", n)
+	}
+
+	// commit the change, and verify it is still there
+	panicOn(tx.Commit())
+
+	// Close and reopen the fragment & verify the data.
+	err := f.Reopen() // roaring data not being flushed? red on roaring
+	if err != nil {
+		t.Fatal(err)
+	}
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
+	defer tx.Rollback()
+
+	if n := f.mustRow(tx, 120).Count(); n != 2 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	} else if n := f.mustRow(tx, 121).Count(); n != 1 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	}
+}
+
+// test that we find all *local* shards
+func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
+	tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly")
+	panicOn(err)
+	defer os.RemoveAll(tmpdir)
+
+	v2s := NewFieldView2Shards()
+	stdShardSet := newShardSet()
+	for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+		stdShardSet.add(shard)
+	}
+	for _, field := range []string{"f", "_exists"} {
+		v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+	}
+
+	for _, src := range []string{"roaring", "bolt", "rbf"} {
+		cfg := mustHolderConfig()
+		cfg.StorageConfig.Backend = src
+		holder := NewHolder(tmpdir, cfg)
+
+		index := "rick"
+		idx := makeSampleRoaringDir(t, tmpdir, index, src, 1, holder, v2s)
+		if idx == nil {
+			idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
+			panicOn(err)
+		}
+		estd := "rick/fields/_exists/views/standard"
+		std := "rick/fields/f/views/standard"
+
+		shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
+		panicOn(err)
+
+		for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+			if !shards[shard] {
+				panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+			}
+		}
+		if src == "roaring" {
+			// check estd too
+			shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
+			panicOn(err)
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				if !shards[shard] {
+					panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
+				}
+			}
+
+			// check GetSortedFieldViewList() and roaringGetFieldView2Shards()
+			vs, err := roaringGetFieldView2Shards(idx)
+			panicOn(err)
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				panicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+
+				for _, fv := range fvs {
+					if !vs.has(fv.Field, fv.View, shard) {
+						panic(fmt.Sprintf("vs did not contain fv='%#v' for shard %v", fv, shard))
+					}
+				}
+				tx.Rollback()
+			}
+		} else {
+			// non-roaring: rbf, bolt
+
+			for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
+				tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
+				fvs, err := tx.GetSortedFieldViewList(idx, shard)
+				panicOn(err)
+				// expect these same two field/views for all 6 shards
+				expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
+				expect1 := txkey.FieldView{Field: "f", View: "standard"}
+				if len(fvs) != 2 {
+					panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
+				}
+				if fvs[0] != expect0 {
+					panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
+				}
+				if fvs[1] != expect1 {
+					panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
+				}
+				tx.Rollback()
+			}
+		}
+		holder.Close()
+	}
+}
+
+// data for Test_DBPerShard_GetShardsForIndex
+//
+var sampleRoaringDirList = map[string]string{"roaring": `
+rick/fields/f/views/standard/fragments/215.cache
+rick/fields/f/views/standard/fragments/221.cache
+rick/fields/f/views/standard/fragments/223.cache
+rick/fields/f/views/standard/fragments/93.cache
+rick/fields/f/views/standard/fragments/217.cache
+rick/fields/f/views/standard/fragments/219.cache
+rick/fields/f/views/standard/fragments/217
+rick/fields/f/views/standard/fragments/219
+rick/fields/f/views/standard/fragments/215
+rick/fields/f/views/standard/fragments/221
+rick/fields/f/views/standard/fragments/223
+rick/fields/f/views/standard/fragments/93
+rick/fields/_exists/views/standard/fragments/221
+rick/fields/_exists/views/standard/fragments/215
+rick/fields/_exists/views/standard/fragments/217
+rick/fields/_exists/views/standard/fragments/93
+rick/fields/_exists/views/standard/fragments/219
+rick/fields/_exists/views/standard/fragments/223
+`,
+	"bolt": `
+rick/backends/backend-boltdb/shard.0093-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0215-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0217-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0219-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0221-bolt/bolt.db
+rick/backends/backend-boltdb/shard.0223-bolt/bolt.db
+`,
+	"rbf": `
+rick/backends/backend-rbf/shard.0093-rbf
+rick/backends/backend-rbf/shard.0215-rbf
+rick/backends/backend-rbf/shard.0217-rbf
+rick/backends/backend-rbf/shard.0219-rbf
+rick/backends/backend-rbf/shard.0221-rbf
+rick/backends/backend-rbf/shard.0223-rbf
+`,
+}
+
+func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
+	shards := []uint64{0, 93, 215, 217, 219, 221, 223}
+	fns := strings.Split(sampleRoaringDirList[backend], "\n")
+	firstDone := false
+
+	for i, fn := range fns {
+		// This check is here because in sampleRoaringDirList, the first entry
+		// of each map value is a line feed, so the strings.Split() above
+		// results in a blank entry for the first item. This means that the
+		// slice of shards above has an initial entry "0" which is not used.
+		if fn == "" {
+			continue
+		}
+		var shard uint64
+		switch backend {
+		case "bolt", "rbf":
+			shard = shards[i]
+
+			idx = helperCreateDBShard(h, index, shard)
+
+			// first time only, we'll actually make all the shards at this point because
+			// view2shards has them all anyway.
+			if !firstDone {
+				firstDone = true
+				makeTxTestDBWithViewsShards(h, idx, view2shards)
+			}
+			continue
+		case "roaring":
+		default:
+			t.Fatalf("invalid backend: %s", backend)
+		}
+
+		path := root + sep + filepath.Dir(fn)
+		panicOn(os.MkdirAll(path, 0755))
+		fd, err := os.Create(root + sep + fn)
+		panicOn(err)
+		if minBytes > 0 {
+			_, err := fd.Write(make([]byte, minBytes))
+			panicOn(err)
+		}
+		fd.Close()
+	}
+	return
+}
+
+func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
+	panicOn(err)
+	// TODO: It's not clear that this is actually doing anything.
+	dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
+	panicOn(err)
+	_ = dbs
+	return idx
+}
+
+// keep the ocd linter happy
+var _ = makeBolttestDB
+var _ = makeRBFtestDB
+
+func makeBolttestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+	w, _ := mustOpenEmptyBoltWrapper(path)
+	BoltMustSetBitvalue(w, "index", "field", "view", shard, i)
+	w.Close()
+}
+
+func makeRBFtestDB(path string, h *Holder, shard uint64) {
+	i := uint64(1)
+
+	db := rbf.NewDB(path, nil)
+	err := db.Open()
+	panicOn(err)
+	defer db.Close()
+
+	tx, err := db.Begin(true)
+	panicOn(err)
+
+	err = tx.CreateBitmap("x")
+	panicOn(err)
+
+	_, err = tx.Add("x", i)
+	panicOn(err)
+
+	err = tx.Commit()
+	panicOn(err)
+}
+
+func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shards) {
+
+	// TODO(jea): need date time quantum views!!
+	batched := false
+	for field, viewmap := range exp.m {
+		for view, shset := range viewmap {
+
+			ss := shset.CloneMaybe()
+			for shard := range ss {
+
+				// simply write 1 bit to each shard to force its creation.
+				bits := []uint64{(shard << shardwidth.Exponent) + 1}
+				tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+				changeCount, err := tx.Add(idx.name, field, view, shard, batched, bits...)
+				panicOn(err)
+				if changeCount != len(bits) {
+					panic(fmt.Sprintf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount))
+				}
+
+				panicOn(tx.Commit())
+			}
+		}
+	}
+
+}
+
+// test that rbf can give us a map[view]*shardSet
+func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
+	tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetFieldView2Shards_map_from_RBF")
+	panicOn(err)
+	defer os.RemoveAll(tmpdir)
+
+	cfg := mustHolderConfig()
+	cfg.StorageConfig.Backend = "rbf"
+	holder := NewHolder(tmpdir, cfg)
+	defer holder.Close()
+
+	index := "rick"
+	field := "f"
+
+	cim := &CreateIndexMessage{
+		Index:     index,
+		CreatedAt: 0,
+		Meta:      IndexOptions{},
+	}
+
+	idx, err := holder.createIndex(cim, false)
+	panicOn(err)
+
+	exp := NewFieldView2Shards()
+
+	stdShardSet := newShardSet()
+	stdShardSet.add(12)
+	stdShardSet.add(15)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
+
+	hrShardSet := newShardSet()
+	hrShardSet.add(7)
+	exp.addViewShardSet(txkey.FieldView{Field: field, View: "standard_2019092416"}, hrShardSet)
+
+	makeTxTestDBWithViewsShards(holder, idx, exp)
+
+	// setup is done
+	view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx)
+	panicOn(err)
+
+	// compare against setup
+	if !view2shard.equals(exp) {
+		panic(fmt.Sprintf("expected '%v' but got view2shard '%v'", exp, view2shard))
+	}
+}

dbshard_test.go

@@ -0,0 +1,111 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/test"
+)
+
+func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c.GetNode(0)
+
+	t.Run("RowIDColumnKey", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "rick"
+		fieldName := "f"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+
+		rowID := uint64(1)
+		timestamp := int64(0)
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		for i := 1; i <= 10; i++ {
+			rowIDs = append(rowIDs, rowID)
+			timestamps = append(timestamps, timestamp)
+		}
+
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
+		// Import data with keys to the primary and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
+			Shard:          0, // import is all on shard 0, why are we making bocu other shards? b/c this is ignored.
+			RowIDs:         rowIDs,
+			ColumnKeys:     colKeys,
+			Timestamps:     timestamps,
+		}
+
+		qcx := m0.API.Txf().NewQcx()
+		if err := m0.API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		panicOn(qcx.Finish())
+
+		//select {}
+
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+			t.Fatalf("unexpected column keys: %#v", keys)
+		}
+	})
+
+}

debugstats/stats.go

@@ -0,0 +1,140 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debugstats
+
+import (
+	"fmt"
+	"math"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+type CallStats struct {
+	// protect elap
+	mu sync.Mutex
+
+	// track how much time each call took.
+	elap map[string]*elapsed
+}
+
+type elapsed struct {
+	dur []float64
+}
+
+func NewCallStats() *CallStats {
+	w := &CallStats{}
+	w.Reset()
+	return w
+}
+
+func (w *CallStats) Reset() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	w.elap = make(map[string]*elapsed)
+}
+
+type LineSorter struct {
+	Line string
+	Tot  float64
+}
+
+type SortByTot []*LineSorter
+
+func (p SortByTot) Len() int {
+	return len(p)
+}
+func (p SortByTot) Less(i, j int) bool {
+	return p[i].Tot < p[j].Tot
+}
+func (p SortByTot) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (c *CallStats) Report(title string) (r string) {
+	r = fmt.Sprintf("CallStats: (%v)\n", title)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var lines []*LineSorter
+	for id, elap := range c.elap {
+		slc := elap.dur
+		n := len(slc)
+		if n == 0 {
+			continue
+		}
+		mean, sd, totaltm := computeMeanSd(slc)
+		if n == 1 {
+			sd = 0
+			mean = slc[0]
+			totaltm = slc[0]
+		}
+		line := fmt.Sprintf("  %20v  N=%8v   avg/op: %12v   sd: %12v  total: %12v\n", id, n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm))
+		lines = append(lines, &LineSorter{Line: line, Tot: totaltm})
+	}
+	sort.Sort(SortByTot(lines))
+	for i := range lines {
+		r += lines[i].Line
+	}
+
+	if false {
+		var m1 runtime.MemStats
+		runtime.ReadMemStats(&m1)
+		r += fmt.Sprintf("\n m1.TotalAlloc = %v\n", m1.TotalAlloc)
+	}
+
+	return
+}
+
+var NaN = math.NaN()
+
+func computeMeanSd(slc []float64) (mean, sd, tot float64) {
+	if len(slc) < 2 {
+		return NaN, NaN, NaN
+	}
+	for _, v := range slc {
+		tot += v
+	}
+	n := float64(len(slc))
+	mean = tot / n
+
+	variance := 0.0
+	for _, v := range slc {
+		tmp := (v - mean)
+		variance += tmp * tmp
+	}
+	variance = variance / n // biased, but we don't care b/c we can have very small n
+	sd = math.Sqrt(variance)
+	if sd < 1e-8 {
+		// sd is super close to zero, NaN out the z-score rather than +/- Inf
+		sd = NaN
+	}
+	return
+}
+
+func (c *CallStats) Add(k string, dur time.Duration) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.elap[k]
+	if !ok {
+		e = &elapsed{}
+		c.elap[k] = e
+	}
+	e.dur = append(e.dur, float64(dur))
+}

debugstats/stats_test.go

@@ -0,0 +1,54 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debugstats
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestCallStats(t *testing.T) {
+
+	callStats := NewCallStats()
+
+	for j := 0; j < 4; j++ {
+		t0 := time.Now()
+		doOperation0()
+		callStats.Add("op0", time.Since(t0))
+
+		t1 := time.Now()
+		doOperation1()
+		callStats.Add("op1", time.Since(t1))
+
+		t2 := time.Now()
+		doOperation2()
+		callStats.Add("op2", time.Since(t2))
+	}
+
+	fmt.Printf("report = \n%v\n", callStats.Report("test"))
+}
+
+func doOperation0() {
+	time.Sleep(50 * time.Millisecond)
+}
+
+func doOperation1() {
+	time.Sleep(100 * time.Millisecond)
+}
+
+func doOperation2() {
+	time.Sleep(200 * time.Millisecond)
+}

diagnostics.go

@@ -229,11 +229,11 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 	timeQuantumEnabled := false
 
 	for _, index := range d.server.holder.Indexes() {
-		numShards += index.AvailableShards().Count()
+		numShards += index.AvailableShards(includeRemote).Count()
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {
@@ -275,6 +275,7 @@ type SystemInfo interface {
 	CPUCores() (physical int, logical int, err error)
 	CPUMHz() (int, error)
 	CPUArch() string
+	DiskCapacity(string) (uint64, error)
 }
 
 // newNopSystemInfo creates a no-op implementation of SystemInfo.
@@ -345,3 +346,8 @@ func (n *nopSystemInfo) CPUMHz() (int, error) {
 func (n *nopSystemInfo) CPUCores() (physical, logical int, err error) {
 	return 0, 0, nil
 }
+
+// DiskCapacity returns the disk capacity
+func (n *nopSystemInfo) DiskCapacity(path string) (uint64, error) {
+	return 0, nil
+}

diagnostics_internal_test.go

@@ -27,8 +27,11 @@ import (
 )
 
 func TestDiagnosticsClient(t *testing.T) {
+	t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
+
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)
@@ -111,6 +114,8 @@ func TestDiagnosticsVersion_Compare(t *testing.T) {
 }
 
 func TestDiagnosticsVersion_Check(t *testing.T) {
+	t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
+
 	// Mock server.
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -121,6 +126,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 			t.Fatalf("couldn't encode version response: %v", err)
 		}
 	}))
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector("localhost:10101")
@@ -144,6 +150,8 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 	}
 }
 
+var _ = compareJSON
+
 func compareJSON(a, b []byte) (bool, error) {
 	var j1, j2 interface{}
 	if err := json.Unmarshal(a, &j1); err != nil {
@@ -156,8 +164,10 @@ func compareJSON(a, b []byte) (bool, error) {
 }
 
 func BenchmarkDiagnostics(b *testing.B) {
+
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)

disco/disco.go

@@ -0,0 +1,481 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package disco
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+var (
+	ErrTooManyResults    error = fmt.Errorf("too many results")
+	ErrNoResults         error = fmt.Errorf("no results")
+	ErrKeyDeleted        error = fmt.Errorf("key deleted")
+	ErrIndexExists       error = fmt.Errorf("index already exists")
+	ErrIndexDoesNotExist error = fmt.Errorf("index does not exist")
+	ErrFieldExists       error = fmt.Errorf("field already exists")
+	ErrFieldDoesNotExist error = fmt.Errorf("field does not exist")
+	ErrViewExists        error = fmt.Errorf("view already exists")
+	ErrViewDoesNotExist  error = fmt.Errorf("view does not exist")
+)
+
+type Peer struct {
+	URL string
+	ID  string
+}
+
+func (p *Peer) String() string {
+	return fmt.Sprintf(`{"ID": "%s", "URL": "%s"}`, p.ID, p.URL)
+}
+
+type DisCo interface {
+	io.Closer
+
+	Start(ctx context.Context) (InitialClusterState, error)
+	IsLeader() bool
+	ID() string
+	Leader() *Peer
+	Peers() []*Peer
+	DeleteNode(ctx context.Context, id string) error
+}
+
+type (
+	InitialClusterState string
+
+	// ClusterState represents the state returned in the /status endpoint.
+	ClusterState string
+)
+
+const (
+	InitialClusterStateNew      InitialClusterState = "new"
+	InitialClusterStateExisting InitialClusterState = "existing"
+
+	ClusterStateUnknown  ClusterState = "UNKNOWN"  // default cluster state. It is returned when we are not able to get the real actual state.
+	ClusterStateStarting ClusterState = "STARTING" // cluster is starting and some internal services are not ready yet.
+	ClusterStateDegraded ClusterState = "DEGRADED" // cluster is running but we've lost some # of hosts >0 but < replicaN. Only read queries are allowed.
+	ClusterStateNormal   ClusterState = "NORMAL"   // cluster is up and running.
+	ClusterStateResizing ClusterState = "RESIZING" // cluster is replicating data to other nodes.
+	ClusterStateDown     ClusterState = "DOWN"     // cluster is unable to serve queries.
+)
+
+type NodeState string
+
+const (
+	NodeStateUnknown  NodeState = "UNKNOWN"
+	NodeStateStarting NodeState = "STARTING"
+	NodeStateStarted  NodeState = "STARTED"
+	NodeStateResizing NodeState = "RESIZING"
+)
+
+type Stator interface {
+
+	// Started will mark the actual node as already started.
+	// It must be called after all initialization processes
+	// are up and running.
+	Started(ctx context.Context) error
+
+	// ClusterState considers the state of all nodes and gives
+	// a general cluster state. The output calculation is as follows:
+	// - If any of the nodes are still starting: "STARTING"
+	// - If all nodes are up and running: "NORMAL"
+	// - If number of DOWN nodes is lower than number of replicas: "DEGRADED"
+	// - If number of unresponsive nodes is greater than (or equal to) the number of replicas: "DOWN"
+	// - If any of the nodes started a resize operation, or a new
+	// node was specifically added or removed from the cluster: "RESIZING"
+	ClusterState(context.Context) (ClusterState, error)
+
+	// NodeState returns the specific state of a node given its ID.
+	NodeState(context.Context, string) (NodeState, error)
+
+	// NodeStates will return all the states by node ID of the actual nodes in the cluster.
+	NodeStates(context.Context) (map[string]NodeState, error)
+}
+
+// Schema is a map of all indexes, each of those being a map of fields, then
+// views.
+type Schema map[string]*Index
+
+// Index is a struct which contains the data encoded for the index as well as
+// for each of its fields.
+type Index struct {
+	Data   []byte
+	Fields map[string]*Field
+}
+
+// Field is a struct which contains the data encoded for the field as well as
+// for each of its views.
+type Field struct {
+	Data  []byte
+	Views map[string]struct{}
+}
+
+// Schemator is the source of truth for different schema elements.
+// All nodes will store and retrieve information from the same source,
+// having the same information at the same time.
+type Schemator interface {
+
+	// Schema return the actual pilosa schema. If the schema is not present, an error is returned.
+	Schema(ctx context.Context) (Schema, error)
+
+	// Index gets a specific index data by name.
+	Index(ctx context.Context, name string) ([]byte, error)
+
+	CreateIndex(ctx context.Context, name string, val []byte) error
+	DeleteIndex(ctx context.Context, name string) error
+	Field(ctx context.Context, index, field string) ([]byte, error)
+	CreateField(ctx context.Context, index, field string, val []byte) error
+	DeleteField(ctx context.Context, index, field string) error
+	View(ctx context.Context, index, field, view string) (bool, error)
+	CreateView(ctx context.Context, index, field, view string) error
+	DeleteView(ctx context.Context, index, field, view string) error
+}
+
+// Metadator is in charge of storing specific metadata per node.
+// This metadata can be retrieved by any node using the specific peerID.
+type Metadator interface {
+	Metadata(ctx context.Context, peerID string) ([]byte, error)
+	SetMetadata(ctx context.Context, metadata []byte) error
+}
+
+// Resizer triggers resizing the node and changes cluster state into RESIZING.
+// We can also return some kind of handler from Resize function (e.g. key-value)
+type Resizer interface {
+	// Resize will trigger a resize event. Node state will change to RESIZE state.
+	// The returned function can be used to send info about the resize process to other nodes.
+	Resize(ctx context.Context) (func([]byte) error, error)
+
+	// DoneResize will mark the resize event as done. This will be called when all the resize actions are done.
+	DoneResize() error
+
+	// Watch will give information about a resize event in another node, using its peerID.
+	// onUpdate function will be called per each event sent by the node in RESIZE state.
+	Watch(ctx context.Context, peerID string, onUpdate func([]byte) error) error
+}
+
+// Sharder is an interface used to maintain the set of availableShards bitmaps
+// per field.
+type Sharder interface {
+	Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error)
+	AddShard(ctx context.Context, index, field string, shard uint64) error
+	AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error)
+	RemoveShard(ctx context.Context, index, field string, shard uint64) error
+}
+
+// NopDisCo represents a DisCo that doesn't do anything.
+var NopDisCo DisCo = &nopDisCo{}
+
+type nopDisCo struct{}
+
+// Close no-op.
+func (n *nopDisCo) Close() error {
+	return nil
+}
+
+// Start is a no-op implementation of the DisCo Start method.
+func (n *nopDisCo) Start(ctx context.Context) (InitialClusterState, error) {
+	return InitialClusterStateNew, nil
+}
+
+// ID is a no-op implementation of the DisCo ID method.
+func (n *nopDisCo) ID() string {
+	return ""
+}
+
+// IsLeader is a no-op implementation of the DisCo IsLeader method.
+func (n *nopDisCo) IsLeader() bool {
+	return false
+}
+
+// Leader is a no-op implementation of the DisCo Leader method.
+func (n *nopDisCo) Leader() *Peer {
+	return nil
+}
+
+// Peers is a no-op implementation of the DisCo Peers method.
+func (n *nopDisCo) Peers() []*Peer {
+	return nil
+}
+
+// DeleteNode a no-op implementation of the DisCo DeleteNode method.
+func (n *nopDisCo) DeleteNode(context.Context, string) error {
+	return nil
+}
+
+// NopStator represents a Stator that doesn't do anything.
+var NopStator Stator = &nopStator{}
+
+type nopStator struct{}
+
+// ClusterState is a no-op implementation of the Stator ClusterState method.
+func (n *nopStator) ClusterState(context.Context) (ClusterState, error) {
+	return ClusterStateUnknown, nil
+}
+
+func (n *nopStator) Started(ctx context.Context) error {
+	return nil
+}
+
+func (n *nopStator) NodeState(context.Context, string) (NodeState, error) {
+	return NodeStateUnknown, nil
+}
+
+func (n *nopStator) NodeStates(context.Context) (map[string]NodeState, error) {
+	return nil, nil
+}
+
+// NopMetadator represents a Metadator that doesn't do anything.
+var NopMetadator Metadator = &nopMetadator{}
+
+type nopMetadator struct{}
+
+func (*nopMetadator) Metadata(context.Context, string) ([]byte, error) {
+	return nil, nil
+}
+func (*nopMetadator) SetMetadata(context.Context, []byte) error {
+	return nil
+}
+
+// NopResizer represents a Resizer that doesn't do anything.
+var NopResizer Resizer = &nopResizer{}
+
+type nopResizer struct{}
+
+func (*nopResizer) Resize(context.Context) (func([]byte) error, error)      { return nil, nil }
+func (*nopResizer) DoneResize() error                                       { return nil }
+func (*nopResizer) Watch(context.Context, string, func([]byte) error) error { return nil }
+
+// NopSharder represents a Sharder that doesn't do anything.
+var NopSharder Sharder = &nopSharder{}
+
+type nopSharder struct{}
+
+// Shards is a no-op implementation of the Sharder Shards method.
+func (n *nopSharder) Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error) {
+	return nil, nil
+}
+
+// AddShard is a no-op implementation of the Sharder AddShard method.
+func (n *nopSharder) AddShard(ctx context.Context, index, field string, shard uint64) error {
+	return nil
+}
+
+// AddShards is a no-op implementation of the Sharder AddShards method.
+func (n *nopSharder) AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error) {
+	return nil, nil
+}
+
+// RemoveShard is a no-op implementation of the Sharder RemoveShard method.
+func (n *nopSharder) RemoveShard(ctx context.Context, index, field string, shard uint64) error {
+	return nil
+}
+
+// NopSchemator represents a Schemator that doesn't do anything.
+var NopSchemator Schemator = &nopSchemator{}
+
+type nopSchemator struct{}
+
+// Schema is a no-op implementation of the Schemator Schema method.
+func (*nopSchemator) Schema(ctx context.Context) (Schema, error) { return nil, nil }
+
+// Index is a no-op implementation of the Schemator Index method.
+func (*nopSchemator) Index(ctx context.Context, name string) ([]byte, error) { return nil, nil }
+
+// CreateIndex is a no-op implementation of the Schemator CreateIndex method.
+func (*nopSchemator) CreateIndex(ctx context.Context, name string, val []byte) error { return nil }
+
+// DeleteIndex is a no-op implementation of the Schemator DeleteIndex method.
+func (*nopSchemator) DeleteIndex(ctx context.Context, name string) error { return nil }
+
+// Field is a no-op implementation of the Schemator Field method.
+func (*nopSchemator) Field(ctx context.Context, index, field string) ([]byte, error) { return nil, nil }
+
+// CreateField is a no-op implementation of the Schemator CreateField method.
+func (*nopSchemator) CreateField(ctx context.Context, index, field string, val []byte) error {
+	return nil
+}
+
+// DeleteField is a no-op implementation of the Schemator DeleteField method.
+func (*nopSchemator) DeleteField(ctx context.Context, index, field string) error { return nil }
+
+// View is a no-op implementation of the Schemator View method.
+func (*nopSchemator) View(ctx context.Context, index, field, view string) (bool, error) {
+	return false, nil
+}
+
+// CreateView is a no-op implementation of the Schemator CreateView method.
+func (*nopSchemator) CreateView(ctx context.Context, index, field, view string) error {
+	return nil
+}
+
+// DeleteView is a no-op implementation of the Schemator DeleteView method.
+func (*nopSchemator) DeleteView(ctx context.Context, index, field, view string) error { return nil }
+
+// InMemSchemator represents a Schemator that manages the schema in memory. The
+// intention is that this would be used for testing.
+var InMemSchemator Schemator = &inMemSchemator{
+	schema: make(Schema),
+}
+
+type inMemSchemator struct {
+	mu     sync.RWMutex
+	schema Schema
+}
+
+// Schema is an in-memory implementation of the Schemator Schema method.
+func (s *inMemSchemator) Schema(ctx context.Context) (Schema, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.schema, nil
+}
+
+// Index is an in-memory implementation of the Schemator Index method.
+func (s *inMemSchemator) Index(ctx context.Context, name string) ([]byte, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	idx, ok := s.schema[name]
+	if !ok {
+		return nil, ErrIndexDoesNotExist
+	}
+	return idx.Data, nil
+}
+
+// CreateIndex is an in-memory implementation of the Schemator CreateIndex method.
+func (s *inMemSchemator) CreateIndex(ctx context.Context, name string, val []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if idx, ok := s.schema[name]; ok {
+		// The current logic in pilosa doesn't allow us to return ErrIndexExists
+		// here, so for now we just update the Data value if the index already
+		// exists.
+		idx.Data = val
+		return nil
+	}
+	s.schema[name] = &Index{
+		Data:   val,
+		Fields: make(map[string]*Field),
+	}
+	return nil
+}
+
+// DeleteIndex is an in-memory implementation of the Schemator DeleteIndex method.
+func (s *inMemSchemator) DeleteIndex(ctx context.Context, name string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.schema, name)
+	return nil
+}
+
+// Field is an in-memory implementation of the Schemator Field method.
+func (s *inMemSchemator) Field(ctx context.Context, index, field string) ([]byte, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return nil, ErrIndexDoesNotExist
+	}
+	fld, ok := idx.Fields[field]
+	if !ok {
+		return nil, ErrFieldDoesNotExist
+	}
+	return fld.Data, nil
+}
+
+// CreateField is an in-memory implementation of the Schemator CreateField method.
+func (s *inMemSchemator) CreateField(ctx context.Context, index, field string, val []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return ErrIndexDoesNotExist
+	}
+	if fld, ok := idx.Fields[field]; ok {
+		// The current logic in pilosa doesn't allow us to return ErrFieldExists
+		// here, so for now we just update the Data value if the field already
+		// exists.
+		fld.Data = val
+		return nil
+	}
+	idx.Fields[field] = &Field{
+		Data:  val,
+		Views: make(map[string]struct{}),
+	}
+	return nil
+}
+
+// DeleteField is an in-memory implementation of the Schemator DeleteField method.
+func (s *inMemSchemator) DeleteField(ctx context.Context, index, field string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return ErrIndexDoesNotExist
+	}
+	delete(idx.Fields, field)
+	return nil
+}
+
+// View is an in-memory implementation of the Schemator View method.
+func (s *inMemSchemator) View(ctx context.Context, index, field, view string) (bool, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return false, ErrIndexDoesNotExist
+	}
+	fld, ok := idx.Fields[field]
+	if !ok {
+		return false, ErrFieldDoesNotExist
+	}
+	_, ok = fld.Views[view]
+	return ok, nil
+}
+
+// CreateView is an in-memory implementation of the Schemator CreateView method.
+func (s *inMemSchemator) CreateView(ctx context.Context, index, field, view string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return ErrIndexDoesNotExist
+	}
+	fld, ok := idx.Fields[field]
+	if !ok {
+		return ErrFieldDoesNotExist
+	}
+	// The current logic in pilosa doesn't allow us to return ErrViewExists
+	// here, so for now we just update the value if the view already exists.
+	fld.Views[view] = struct{}{}
+	return nil
+}
+
+// DeleteView is an in-memory implementation of the Schemator DeleteView method.
+func (s *inMemSchemator) DeleteView(ctx context.Context, index, field, view string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.schema[index]
+	if !ok {
+		return ErrIndexDoesNotExist
+	}
+	fld, ok := idx.Fields[field]
+	if !ok {
+		return ErrFieldDoesNotExist
+	}
+	delete(fld.Views, view)
+	return nil
+}

docs/README.md

@@ -1,5 +0,0 @@
-Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/).
-
-Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search. Internal links will only work on the website.
-
-Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request!

docs/administration.md

@@ -1,327 +0,0 @@
-+++
-title = "Administration"
-weight = 13
-nav = [
-    "Installing in production",
-    "Importing and Exporting Data",
-    "Versioning",
-    "Resizing the Cluster",
-    "Backup/restore",
-]
-+++
-
-## Administration Guide
-
-### Installing in production
-
-#### Hardware
-
-Pilosa is a standalone, compiled Go application, so there is no need to worry about running and configuring a Java VM. Pilosa can run on very small machines and works well with even a medium sized dataset on a personal laptop. If you are reading this section, you are likely ready to deploy a cluster of Pilosa servers handling very large datasets or high velocity data. These are guidelines for running a cluster; specific needs may differ.
-
-#### Memory
-
-Pilosa holds all row/column bitmap data in main memory. While this data is compressed more than a typical database, available memory is a primary concern. In a production environment, we recommend choosing hardware with a large amount of memory >= 64GB. Prefer a small number of hosts with lots of memory per host over a larger number with less memory each. Larger clusters tend to be less efficient overall due to increased inter-node communication.
-
-#### CPUs
-
-Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload.
-
-#### Disk
-
-Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application.
-
-#### Network
-
-Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups.
-
-#### Overview
-
-While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
-
-### Open File Limits
-
-Pilosa requires a large number of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa.
-
-On Mac OS X, `ulimit` does not behave predictably. The Mac OS X system has a utility called csrutil that prevents you from changing the open file limit easily. One workaround that may work for you involves disabling the csrutil program. To disable the csrutil program, restart your laptop and when the start up screen pops up, hold down command + R to enter Recovery Mode. Open a terminal and enter `csrutil disable`, then restart your computer as you normally would. Now that the csrutil is disabled, you can change the open file limit. The open file limit can be changed by creating the following files and changing their ownership:
-
-Copy the contents of [this](https://github.com/wilsonmar/mac-setup/blob/master/configs/limit.maxfiles.plist) file into a new file on your system located at /Library/LaunchDaemons/limit.maxfiles.plist, then run:
-
-```
-sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
-```
-
-Copy the contents of [this](https://github.com/wilsonmar/mac-setup/blob/master/configs/limit.maxproc.plist) file into a new file on your system located at /Library/LaunchDaemons/limit.maxproc.plist, then run:
-
-```
-sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist
-```
-
-To ensure the open file limit has successfully changed, run `ulimit -a`. Your open files should be set to a number greater than 256 (in the range of 524288) and your max users processes should be greater than 709 (in the range of 2048).
-
-### Importing and Exporting Data
-
-#### Importing
-
-The import API expects a csv of the format `Row,Column`.
-
-When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
-
-```
-pilosa import --sort -i project -f stargazer project-stargazer.csv
-```
-
-We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at:
-* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md)
-* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md)
-* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md)
-
-##### Importing Integer Values
-
-If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`.
-
-```
-pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
-```
-
-##### Importing Boolean Values
-
-If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true).
-
-For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row.
-```
-0,3
-0,9
-1,1
-1,2
-1,4
-1,8
-```
-
-<div class="note">
-    <p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
-</div>
-
-#### Clearing Data via Import
-
-By using the `--clear` flag with the import command, Pilosa will clear the values provided in the import payload.
-
-For example, importing a file with the following contents along with the `--clear` flag will result in data being cleared from row 0, column 9; row 1, columns 2 and 8; and row 3, column 12. Clearing a value that doesn't exists is allowed.
-```
-0,9
-1,2
-1,8
-3,12
-```
-
-#### Exporting
-
-Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column.
-```request
-curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \
-     --header "Accept: text/csv"
-```
-```response
-2,10
-2,30
-3,426
-4,2
-...
-```
-
-### Versioning
-
-Pilosa follows [Semantic Versioning](http://semver.org/).
-
-MAJOR.MINOR.PATCH:
-
-* MAJOR version when you make incompatible API changes,
-* MINOR version when you add functionality in a backwards-compatible manner, and
-* PATCH version when you make backwards-compatible bug fixes.
-
-#### PQL versioning
-
-The Pilosa server should support PQL versioning using HTTP headers. On each request, the client should send a Content-Type header and an Accept header. The server should respond with a Content-Type header that matches the client Accept header. The server should also optionally respond with a Warning header if a PQL version is in a deprecation period, or an HTTP 400 error if a PQL version is no longer supported.
-
-#### Upgrading
-
-To upgrade Pilosa:
-
-1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`.
-2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases).
-3. Shut down the Pilosa cluster.
-4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node.
-5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make.
-6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes.
-
-##### Version 1.4
-
-Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release.
-
-### Resizing the Cluster
-
-If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process.
-
-#### Adding a Node
-
-You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries.
-
-If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately.
-
-#### Removing a Node
-
-In order to  remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself.
-
-To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing a `/status` request to the node. The node's ID is in the `localID` field:
-``` request
-curl localhost:10101/status
-```
-``` response
-{
-    "state":"NORMAL",
-    "nodes":[
-        {"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
-        {"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
-        {"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
-    ],
-    "localID": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"
-}
-```
-
-If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node:
-``` request
-curl localhost:10101/status
-```
-``` response
-{
-    "state":"NORMAL",
-    "nodes":[
-        {"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
-        {"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
-        {"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
-    ],
-    "localID": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"
-}
-```
-
-Once you have the ID of the node that you want to remove from the cluster, issue the following request:
-```
-curl localhost:10101/cluster/resize/remove-node \
-     -X POST \
-     -d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}'
-```
-At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries.
-
-Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator).
-
-#### Aborting a Resize Job
-
-If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node.
-For example, if your coordinator node is `localhost:10101`, then you can run:
-```
-curl localhost:10101/cluster/resize/abort -X POST
-```
-This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began.
-
-#### Changing the Coordinator
-
-In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator.
-```
-curl localhost:10101/cluster/resize/set-coordinator \
-     -X POST \
-     -d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}'
-```
-
-### Backup/restore
-
-Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster.
-
-Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node.
-
-For larger datasets and to make this process faster you could copy the relevant data files from the other nodes to the new one before startup.
-
-Note: This will only work when the replication factor is >= 2
-
-#### Using Index Sync
-
-- Shutdown the cluster.
-- Modify config file to replace existing node address with new node.
-- Restart all nodes in the cluster.
-- Wait for auto Index sync to replicate data from existing nodes to new node.
-
-#### Copying data files manually
-
-- To accomplish this you will first need:
-  - List of all indexes on your cluster
-  - List of all fields in your indexes
-  - Max shard per index, listed in the `/internal/shards/max` endpoint
-- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard
-- Using the list of shards owned by this node you will then need to manually:
-  - setup a directory structure similar to the other nodes with a path for each Index/Field
-  - copy each owned shard for an existing node to this new node
-- Modify the cluster config file to replace the previous node address with the new node address.
-- Restart the cluster
-- Wait for the first sync (10 minutes) to validate Index connections
-
-### Diagnostics
-
-Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
-
-- **Version:** Version string of the build.
-- **Host:** Host URI.
-- **Cluster:** List of nodes in the cluster.
-- **NumNodes:** Number of nodes in the cluster.
-- **NumCPU:** Number of cores per node
-- **BSIEnabled:** Bit Sliced Index Fields in use.
-- **TimeQuantumEnabled:** Time Quantum Fields in use.
-- **NumIndexes:** Number of indexes in the Cluster.
-- **NumFields:** Number of fields in the Cluster.
-- **NumShards:** Number of shards in the Cluster.
-- **NumViews:** Number of views in the Cluster.
-- **OpenFiles:** Open file handle count.
-- **GoRoutines:** Go routine count.
-
-You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
-
-### Metrics
-
-Pilosa can be configured to emit metrics pertaining to its internal processes in one of three formats: Expvar, StatsD, or Prometheus. Metric recording is disabled by default.
-The metrics configuration options are:
-
-  - [Host](../configuration/#metric-host): specify host that receives metric events
-  - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
-  - [Service](../configuration/#metric-service): declare type StatsD or Expvar
-
-#### Tags
-StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
-
-- NodeID
-- Index
-- Field
-- View
-- Shard
-
-#### Events
-We currently track the following events
-
-- **Index:** The creation of a new index.
-- **Field:** The creation of a new field.
-- **MaxShard:** The creation of a new Shard.
-- **SetBit:** Count of set bits.
-- **ClearBit:** Count of cleared bits.
-- **ImportBit:** During a bulk data import this represents the count of bits created.
-- **SetRowAttrs:** Count of attributes set per row.
-- **SetColumnAttrs:** Count of attributes set per column.
-- **Bitmap:** Count of Bitmap queries.
-- **TopN:** Count of TopN queries.
-- **Union:** Count of Union queries.
-- **Intersection:** Count of Intersection queries.
-- **Difference:** Count of Difference queries.
-- **Xor:** Count of Xor queries.
-- **Not:** Count of Not queries.
-- **Count:** Count of Count queries.
-- **Range:** Count of ranged Row queries.
-- **Snapshot:** Event count when the snapshot process is triggered.
-- **BlockRepair:** Count of data blocks that were out of sync and repaired.
-- **GarbageCollection:** Event count when garbage collection occurs.
-- **Goroutines:** Number of running goroutines.
-- **OpenFiles:** Number of open file handles associated with running Pilosa process ID.

docs/api-reference.md

@@ -1,352 +0,0 @@
-+++
-title = "API Reference"
-weight = 10
-nav = []
-+++
-
-
-## API Reference
-
-### List all index schemas
-
-`GET /index`
-
-Is equivalent to `GET /schema` and returns the same response.
-
-### List index schema
-
-`GET /index/<index-name>`
-
-Returns the schema of the specified index in JSON.
-
-``` request
-curl -XGET localhost:10101/index/user
-```
-``` response
-{
-    "fields": [
-        {
-            "name": "event",
-            "options": {
-                "keys": false,
-                "timeQuantum": "YMD",
-                "type": "time"
-            }
-        }
-    ],
-    "name": "user",
-    "options": {
-        "keys": false,
-        "trackExistence": true
-    }
-}
-```
-
-### Create index
-
-`POST /index/<index-name>`
-
-Creates an index with the given name.
-
-The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object with the following options:
-
-* `keys` (bool): Enables using column keys instead of column IDs.
-* `trackExistence` (bool): Enables or disables existence tracking on the index. Required for [Not](../query-language/#not) queries. It is `true` by default.
-
-``` request
-curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
-```
-``` response
-{"success":true}
-```
-
-### Remove index
-
-`DELETE /index/index-name`
-
-Removes the given index.
-
-``` request
-curl -XDELETE localhost:10101/index/user
-```
-``` response
-{"success":true}
-```
-
-### Query index
-
-`POST /index/<index-name>/query`
-
-Sends a [query](../query-language/) to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
-
-``` request
-curl localhost:10101/index/user/query \
-     -X POST \
-     -d 'Row(language=5)'
-```
-``` response
-{
-    "results": [
-        {
-            "attrs": {},
-            "columns": [
-                100
-            ]
-        }
-    ]
-}
-```
-
-In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
-
-The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
-
-The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices.
-
-``` request
-curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \
-     -X POST \
-     -d 'Row(language=5)'
-```
-``` response
-{
-    "columnAttrs": [
-        {
-            "attrs": {
-                "name": "Klingon"
-            },
-            "id": 100
-        }
-    ],
-    "results": [
-        {
-            "attrs": {},
-            "columns": [
-                100
-            ]
-        }
-    ]
-}
-```
-
-By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
-
-### Import Data
-
-`POST /index/<index-name>/field/<field-name>/import`
-
-Supports high-rate data ingest to a particular shard of a particular field. The
-official client libraries use this endpoint for their import functionality - it
-is not usually necessary to use this endpoint directly. See the documentation for
-imports for
-<a href="https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md">Go</a>,
-<a href="https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md">Java</a>,
-and <a href="https://github.com/pilosa/python-pilosa/tree/master/docs/imports.md">Python</a>.
-
-The request payload is protobuf encoded with the following schema. The RowKeys
-and/or ColumnKeys fields are used if the pilosa field or index are configured
-for keys respectively. Otherwise, the RowIDs and ColumnIDs fields are used. They
-must have the same number of items, and each index into those two lists
-represents a particular bit to be set. Timestamps are optional, but if they
-exist must also contain the same number of items as rows and columns. The
-column IDs must all be in the shard specified in the request.
-
-```
-message ImportRequest {
-	string Index = 1;
-	string Field = 2;
-	uint64 Shard = 3;
-	repeated uint64 RowIDs = 4;
-	repeated uint64 ColumnIDs = 5;
-	repeated string RowKeys = 7;
-	repeated string ColumnKeys = 8;
-	repeated int64 Timestamps = 6;
-}
-```
-
-
-### Create field
-
-`POST /index/<index-name>/field/<field-name>`
-
-Creates a field in the given index with the given name.
-
-The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which must contain a `type`:
-
-* `type` (string): Sets the field type and type options.
-* `keys` (bool): Enables using column keys instead of column IDs (optional).
-
-Valid `type`s and correspondonding options are listed below:
-
-* `set`
-    * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `ranked`.
-    * `cacheSize` (int): Number of rows to keep in the cache. Default is 50,000.
-* `int`
-    * `min` (int): Minimum integer value allowed for the field.
-    * `max` (int): Maximum integer value allowed for the field.
-* `bool`
-    * (boolean fields take no arguments)
-* `time`
-    * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
-* `mutex`
-    * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `ranked`.
-    * `cacheSize` (int): Number of rows to keep in the cache. Default is 50,000.
-
-The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
-
-``` request
-curl localhost:10101/index/user/field/quantity \
-     -X POST \
-     -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
-```
-``` response
-{"success":true}
-```
-
-Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
-
-``` request
-curl localhost:10101/index/user/field/language -X POST
-```
-``` response
-{"success":true}
-```
-
-``` request
-curl localhost:10101/index/repository/field/stats \
-    -X POST \
-    -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
-```
-``` response
-{"success":true}
-```
-
-### Remove field
-
-`DELETE /index/<index-name>/field/<field-name>`
-
-Removes the given field.
-
-``` request
-curl -XDELETE localhost:10101/index/user/field/language
-```
-``` response
-{"success":true}
-```
-
-### List all index schemas
-
-`GET /schema`
-
-Returns the schema of all indexes in JSON.
-
-``` request
-curl -XGET localhost:10101/schema
-```
-``` response
-{
-    "indexes": [
-        {
-            "fields": [
-                {
-                    "name": "event",
-                    "options": {
-                        "keys": false,
-                        "timeQuantum": "YMD",
-                        "type": "time"
-                    }
-                },
-                {
-                    "name": "language",
-                    "options": {
-                        "cacheSize": 50000,
-                        "cacheType": "ranked",
-                        "keys": false,
-                        "type": "set"
-                    }
-                }
-            ],
-            "name": "user",
-            "options": {
-                "keys": false,
-                "trackExistence": true
-            }
-        }
-    ]
-}
-```
-
-### Duplicate schema into empty Pilosa cluster
-
-`POST /schema`
-
-To duplicate one Pilosa cluster's schema to another, it's possible to
-pass the output of `GET /schema` as the request body of `POST /schema`
-and all the indexes and fields in the schema will be created in
-Pilosa. As of this writing, the behavior of POSTing a schema to a
-non-empty Pilosa cluster is undefined. These semantics will likely be
-ironed out in a future version.
-
-``` request
-# after (e.g.) curl -XGET localhost:10101/schema > schema.json
-curl -XPOST localhost:10101/schema --data-binary @schema.json
-```
-
-Response: `204 No Content`
-
-### Get version
-
-`GET /version`
-
-Returns the version of the Pilosa server.
-
-``` request
-curl -XGET localhost:10101/version
-```
-``` response
-{"version":"v0.6.0"}
-```
-
-### Get status
-
-`GET /status`
-
-Returns the status of the cluster.
-
-```request
-curl -XGET localhost:10101/status
-```
-```response
-{
-    "localID": "d3369125-29d8-4305-a351-b4474d14a542",
-    "nodes": [
-        {
-            "id": "d3369125-29d8-4305-a351-b4474d14a542",
-            "isCoordinator": true,
-            "uri": {
-                "host": "localhost",
-                "port": 10101,
-                "scheme": "http"
-            }
-        }
-    ],
-    "state": "NORMAL"
-}
-```
-
-### Recalculate Caches
-
-`POST /recalculate-caches`
-
-Recalculates the caches on demand. The cache is recalculated every 10
-seconds by default. This endpoint can be used to recalculate the cache
-before the 10 second interval. This should probably only be used in
-integration tests and not in a typical production workflow. Note that
-in a multi-node cluster, the cache is only recalculated on the node
-that receives the request.
-
-``` request
-curl -XPOST localhost:10101/recalculate-caches
-```
-
-Response: `204 No Content`

docs/architecture.md

@@ -1,25 +0,0 @@
-+++
-title = "Architecture"
-weight = 6
-nav = []
-+++
-
-## Architecture
-
-### Roaring bitmap storage format
-
-Bitmaps are persisted to disk using a file format very similar to the [Roaring Bitmap format spec](https://github.com/RoaringBitmap/RoaringFormatSpec). Pilosa's format uses 64-bit IDs, so it is not binary-compatible with the spec. Some parts of the format are simpler, and an additional section is included. Specific differences include:
-
-* The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3.
-* The cookie includes file format version in bytes 2-3 (currently equal to zero).
-* The descriptive header includes, for each container, a 64-bit key, a 16-bit cardinality, and a 16-bit container type (which only uses two bits now). This makes the runFlag bitset unnecessary. This is in contrast to the spec, which stores a 16-bit key and a 16-bit cardinality.
-* The offset header section is always included.
-* RLE runs are serialized as [start, last], not [start, length].
-* After the container storage section is an operation log, of unspecified length.
-
-![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png)
-*Pilosa Roaring storage format diagram*
-
-All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.
-
-Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa.

docs/client-libraries.md

@@ -1,18 +0,0 @@
-+++
-title = "Client Libraries"
-weight = 12
-nav = [
-    "Go",
-    "Python",
-    "Java",
-]
-+++
-
-## Client Libraries
-
-We have the following official client libraries. You can find more information in their repositories:
-* [Go client repository](https://github.com/pilosa/go-pilosa)
-* [Java client repository](https://github.com/pilosa/java-pilosa)
-* [Python client repository](https://github.com/pilosa/python-pilosa)
-
-Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients.

docs/configuration.md

@@ -1,638 +0,0 @@
-+++
-title = "Configuration"
-weight = 7
-nav = [
-    "Command line flags",
-    "Environment variables",
-    "Config file",
-    "All Options",
-]
-+++
-
-## Configuration
-
-Pilosa can be configured through command line flags, environment variables, and/or a configuration file; configured options take precedence in that order. So if an option is specified in a command line flag, it will take precedence over the same option specified in the environment, which will take precedence over that same option specified in the configuration file.
-
-All options are available in all three configuration types with the exception of the `--config` option which specifies the location of the config file, and therefore will not be used if it is present in the config file.
-
-The syntax for each option is slightly different between each of the configuration types, but follows a simple formula. See the following three sections for an explanation of each configuration type.
-
-### Command line flags
-
-Pilosa uses GNU/POSIX style flags. Most flags you specify as `--flagname=value` although some have a short form that is a single character and can be specified with a single dash like `-f value`. Running `pilosa server --help` will give an overview of the available flags as well as their short forms (if applicable).
-
-### Environment variables
-
-Every command line flag has a corresponding environment variable. The environment variable is the flag name in all caps, prefixed by `PILOSA_`, and with dots and dashes replaced by underscores. For example: `--scope.flag-name` becomes `PILOSA_SCOPE_FLAG_NAME`.
-
-### Config file
-
-The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file:
-```toml
-[cluster]
-  coordinator = true
-  replicas = 1
-```
-
-### All Options
-
-#### Advertise
-
-* Description: Address advertised by the server to other nodes in the cluster and to clients via the `/status` endpoint. Host defaults to the IP address represented by `bind` and port to 10101. If `bind` is set to `0.0.0.0` and `advertise` is not specified, then Pilosa will try to determine a reasonable, external IP address to use for `advertise`.
-* Flag: `--advertise="192.168.1.100:10101"`
-* Env: `PILOSA_BIND="192.168.1.100:10101"`
-* Config:
-
-    ```toml
-    advertise = 192.168.1.100:10101
-    ```
-
-#### Anti Entropy Interval
-
-* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
-* Flag: `--anti-entropy.interval="10m0s"`
-* Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"`
-* Config:
-
-    ```toml
-    [anti-entropy]
-    interval = "10m0s"
-    ```
-
-#### Bind
-
-* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. If `bind` is set to `0.0.0.0` then Pilosa will listen on all available interfaces.
-* Flag: `--bind="localhost:10101"`
-* Env: `PILOSA_BIND="localhost:10101"`
-* Config:
-
-    ```toml
-    bind = localhost:10101
-    ```
-
-#### CORS (Cross-Origin Resource Sharing) Allowed Origins
-
-* Description: List of allowed origin URIs for CORS
-* Flag: `--handler.allowed-origins="https://myapp.com,https://myapp.org"`
-* Env: `PILOSA_HANDLER_ALLOWED_ORIGINS="https://myapp.com,https://myapp.org"`
-* Config:
-
-    ```toml
-    [handler]
-    allowed-origins = ["https://myapp.com", "https://myapp.org"]
-    ```
-
-#### Data Dir
-
-* Description: Directory to store Pilosa data files.
-* Flag: `--data-dir="~/.pilosa"`
-* Env: `PILOSA_DATA_DIR="~/.pilosa"`
-* Config:
-
-    ```toml
-    data-dir = "~/.pilosa"
-    ```
-
-#### Log Path
-
-* Description: Path of log file.
-* Flag: `--log-path="/path/to/logfile"`
-* Env: `PILOSA_LOG_PATH="/path/to/logfile"`
-* Config:
-
-    ```toml
-    log-path = "/path/to/logfile"
-    ```
-
-#### Verbose
-
-* Description: Enable verbose logging.
-* Flag: `--verbose`
-* Env: `PILOSA_VERBOSE`
-* Config:
-
-    ```toml
-    verbose = true
-    ```
-
-#### Max Map Count
-
-* Description: Maximum number of active memory maps Pilosa will use for fragment
-  files (actual total usage may be slightly higher). Best practice is to set
-  this ~10% lower than your system's maximum map count (obtained via `sysctl
-  vm.max_map_count` on Linux). If you plan on having lots of fragments per host,
-  it's a good idea to raise both the system's max map count, and Pilosa's. The
-  number of fragments is a function of the number of shards, fields, and time
-  quantums. Using, for example, YMDH time quantum fields with a wide range of
-  timestamps will create lots of fragments. When Pilosa exhausts the
-  max-map-count it falls back to reading files directly into memory. This can be
-  a bit slower, and cause slower restarts, but is generally fine.
-  * Flag: `--max-map-count=1000000`
-  * Env: `PILOSA_MAX_MAP_COUNT=1000000`
-  * Config:
-
-      ```toml
-      max-map-count = 1000000
-      ```
-
-#### Max Writes Per Request
-
-* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs.
-* Flag: `--max-writes-per-request=5000`
-* Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000`
-* Config:
-
-    ```toml
-    max-writes-per-request = 5000
-    ```
-
-#### Max File Count
-
-* Description: A soft limit on the maximum number of files that Pilosa will keep
-  open simultaneously. When past this limit, Pilosa will only keep files open
-  for as long as it needs to write updates. This will negatively affect
-  performance in cases where Pilosa is doing lots of small updates.
-* Flag: `--max-file-count=1000000`
-* Env: `PILOSA_MAX_FILE_COUNT=1000000`
-* Config:
-
-    ```toml
-    max-file-count = 1000000
-    ```
-
-#### Gossip Advertise Host
-
-* Description: Host on which memberlist should advertise. Defaults to `advertise` host.
-* Flag: `--gossip.advertise-host=192.168.1.100`
-* Env: `PILOSA_GOSSIP_ADVERTISE_HOST=192.168.1.100
-* Config:
-
-    ```toml
-    [gossip]
-      advertise-host = 192.168.1.100
-    ```
-
-#### Gossip Advertise Port
-
-* Description: Port on which memberlist should advertise. Defaults to `advertise` port.
-* Flag: `--gossip.advertise-port=15001`
-* Env: `PILOSA_GOSSIP_ADVERTISE_PORT=15001`
-* Config:
-
-    ```toml
-    [gossip]
-      advertise-port = 15001
-    ```
-
-#### Gossip Port
-
-* Description: Port to which Pilosa should bind for internal communication. If more than one Pilosa server is running on the same host, the gossip port for each server must be unique.
-* Flag: `--gossip.port=11101`
-* Env: `PILOSA_GOSSIP_PORT=11101`
-* Config:
-
-    ```toml
-    [gossip]
-      port = 11101
-    ```
-
-#### Gossip Seeds
-
-* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typically this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms.
-* Flag: `--gossip.seeds="localhost:11101,localhost:11110"`
-* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101,localhost:11110"`
-* Config:
-
-    ```toml
-    [gossip]
-      seeds = ["localhost:11101", "localhost:11110"]
-    ```
-
-#### Gossip Key
-
-* Description: Path to the file which contains the key to encrypt gossip communication. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 encryption. You can read from `/dev/random` device on UNIX-like systems to create the key file; e.g., `head -c 32 /dev/random > gossip.key32` creates a key file to use AES-256.
-* Flag: `--gossip.key="/var/secret/gossip.key32"`
-* Env: `PILOSA_GOSSIP_KEY="/var/secret/gossip.key32"`
-* Config:
-
-    ```toml
-    [gossip]
-      key = "/var/secret/gossip.key32"
-    ```
-
-#### Cluster Coordinator
-
-* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
-* Flag: `cluster.coordinator`
-* Env: `PILOSA_CLUSTER_COORDINATOR`
-* Config:
-
-    ```toml
-    [cluster]
-    coordinator = true
-    ```
-
-#### Cluster Long Query Time
-
-* Description: Duration that will trigger log and stat messages for slow queries.
-* Flag: `cluster.long-query-time="1m0s"`
-* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
-* Config:
-
-    ```toml
-    [cluster]
-    long-query-time = "1m0s"
-    ```
-
-#### Cluster Replicas
-
-* Description: Number of hosts each piece of data should be stored on. 
-* Flag: `cluster.replicas=1`
-* Env: `PILOSA_CLUSTER_REPLICAS=1`
-* Config:
-
-    ```toml
-    [cluster]
-    replicas = 1
-    ```
-
-#### Cluster Type
-
-* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip].
-  * static - Messaging between nodes is disabled. This is primarily used for testing.
-  * gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip.
-* Flag: `cluster.type="gossip"`
-* Env: `PILOSA_CLUSTER_TYPE="gossip"`
-* Config:
-
-    ```toml
-    [cluster]
-    type = "gossip"
-    ```
-
-#### Profile CPU
-
-* Description: If this is set to a path, collect a cpu profile and store it there.
-* Flag: `--profile.cpu="/path/to/somewhere"`
-* Env: `PILOSA_PROFILE_CPU="/path/to/somewhere"`
-* Config:
-
-    ```toml
-    [profile]
-    cpu = "/path/to/somewhere"
-    ```
-
-#### Profile CPU Time
-
-* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set.
-* Flag: `--profile.cpu-time="30s"`
-* Env: `PILOSA_PROFILE_CPU_TIME="30s"`
-* Config:
-
-    ```toml
-    [profile]
-    cpu-time = "30s"
-    ```
-
-#### Metric Service
-* Description: Which stats service to use for collecting [metrics](../administration/#metrics). Choose from [statsd, expvar, prometheus, none].
-* Flag: `--metric.service=statsd`
-* Env: `PILOSA_METRIC_SERVICE=statsd`
-* Config:
-
-    ```toml
-    [metric]
-    service = “statsd”
-    ```
-
-#### Metric Host
-* Description: Address of the StatsD service host.
-* Flag: `--metric.host=localhost:8125`
-* Env: `PILOSA_METRIC_HOST=localhost:8125`
-* Config:
-
-    ```toml
-    [metric]
-    host = "localhost:8125"
-    ```
-
-#### Metric Poll Interval
-
-* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
-* Flag: `metric.poll-interval=”0m15s”`
-* Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
-* Config:
-
-    ```toml
-    [metric]
-    poll-interval = "0m15s"
-    ```
-
-#### Metric Diagnostics
-
-* Description: Enable [reporting](../administration/#diagnostics) of limited usage statistics to Pilosa developers. To disable, set to false.
-* Flag: `metric.diagnostics`
-* Env: `PILOSA_METRIC_DIAGNOSTICS`
-* Config:
-
-    ```toml
-    [metric]
-    diagnostics = true
-    ```
-
-
-#### TLS Certificate
-
-* Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of `.crt` or `.pem` extensions.
-* Flag: `tls.certificate=/srv/pilosa/certs/server.crt`
-* Env: `PILOSA_TLS_CERTIFICATE=/srv/pilosa/certs/server.crt`
-* Config:
-
-    ```toml
-    [tls]
-    certificate = "/srv/pilosa/certs/server.crt"
-    ```
-
-#### TLS Certificate Key
-
-* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension.
-* Flag: `tls.key=/srv/pilosa/certs/server.key`
-* Env: `PILOSA_TLS_KEY=/srv/pilosa/certs/server.key`
-* Config:
-
-    ```toml
-    [tls]
-    key = "/srv/pilosa/certs/server.key"
-    ```
-
-#### TLS CA Certificate
-
-* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has one of `.crt` or `.pem` extensions.
-* Flag: `tls.ca-certificate=/srv/pilosa/certs/ca-chain.pem`
-* Env: `PILOSA_TLS_CA_CERTIFICATE=/srv/pilosa/certs/ca-chain.pem`
-* Config:
-
-    ```toml
-    [tls]
-    ca-certificate = "/srv/pilosa/certs/ca-chain.pem"
-    ```
-
-#### TLS Skip Verify
-
-* Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial.
-* Flag: `tls.skip-verify`
-* Env: `PILOSA_TLS_SKIP_VERIFY`
-* Config:
-
-    ```toml
-    [tls]
-    skip-verify = true
-    ```
-
-#### TLS Enable Client Certificate Verification
-
-* Description: Enables verification of client certificates on incoming HTTPS requests for mutual TLS authentication.
-* Flag: `tls.enable-client-verification`
-* Env: `PILOSA_TLS_ENABLE_CLIENT_VERIFICATION`
-* Config:
-
-    ```toml
-    [tls]
-    enable-client-verification = true
-    ```
-
-#### Tracing Sampler Type
-
-* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely.
-* Flag: `tracing.sampler-type`
-* Env: `PILOSA_TRACING_SAMPLER_TYPE`
-* Config:
-
-    ```toml
-    [tracing]
-    sampler-type = "remote"
-    ```
-
-#### Tracing Sampler Parameter
-
-* Description: Jaeger sampler parameter (number)
-* Flag: `tracing.sampler-param`
-* Env: `PILOSA_TRACING_SAMPLER_PARAM`
-* Config:
-
-    ```toml
-    [tracing]
-    sampler-param = 0.001
-    ```
-
-#### Tracing Agent Host/Port
-
-* Description: Jaeger agent host:port
-* Flag: `tracing.agent-host-port`
-* Env: `PILOSA_TRACING_AGENT_HOST_PORT`
-* Config:
-
-    ```toml
-    [tracing]
-    agent-host-port = "localhost:6831"
-    ```
-
-#### Profile Block Rate
-
-* Description: Block Rate is passed directly to Go's
-  [runtime.SetBlockProfileRate](https://golang.org/pkg/runtime/#SetBlockProfileRate). Goroutine blocking events will be sampled at 1
-  per `rate` nanoseconds. A value of "1" samples every event, and 0 disables
-  profiling.
-* Flag: `--profile.block-rate=10000000`
-* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000`
-* Config:
-
-   ```toml
-   [profile]
-   block-rate = 10000000
-   ```
-
-#### Profile Mutex Fraction
-
-* Description: Mutex Fraction is passed directly to Go's
-  [runtime.SetMutexProfileFraction](https://golang.org/pkg/runtime/#SetMutexProfileFraction). 1/`fraction` of events will be sampled. 
-* Flag: `--profile.mutex-fraction=100`
-* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100`
-* Config:
-
-   ```toml
-   [profile]
-   mutex-fraction = 100
-   ```
-
-#### Translation Map Size
-
-* Description: Size in bytes of mmap to allocate for key translation
-* Flag: `translation.map-size`
-* Env: `PILOSA_TRANSLATION_MAP_SIZE`
-* Config:
-
-    ```toml
-    [translation]
-    map-size = 10737418240
-    ```
-
-### Example Cluster Configuration
-
-A three node cluster running on different hosts could be minimally configured as follows:
-
-#### Node 0
-
-    data-dir = "/home/pilosa/data"
-    bind = "node0.pilosa.com:10101"
-    
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-
-    [cluster]
-      replicas = 1
-      coordinator = true
-
-#### Node 1
-
-    data-dir = "/home/pilosa/data"
-    bind = "node1.pilosa.com:10101"
-    
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-#### Node 2
-
-    data-dir = "/home/pilosa/data"
-    bind = "node2.pilosa.com:10101"
-    
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-
-### Example Cluster Configuration (HTTPS)
-
-The same cluster which uses HTTPS instead of HTTP can be configured as follows. Note that we explicitly specify `https` as the protocol in `bind` and `cluster.hosts` configuration. It is not required to use a gossip key but it is highly recommended: 
-
-#### Node 0
-
-    data-dir = "/home/pilosa/data"
-    bind = "https://node0.pilosa.com:10101"
-
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = true
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"
-
-#### Node 1
-
-    data-dir = "/home/pilosa/data"
-    bind = "https://node1.pilosa.com:10101"
-
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"
-      
-#### Node 2
-
-    data-dir = "/home/pilosa/data"
-    bind = "https://node2.pilosa.com:10101"
-
-    [gossip]
-      port = 12000
-      seeds = ["node0.pilosa.com:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"
-
-### Example Cluster Configuration (HTTPS, same host)
-
-You can run a cluster on the same host using the configuration above with a few changes. Gossip port and bind address should be different for each node and a data directory should be accessed only by a single node.
-
-#### Node 0
-
-    data-dir = "/home/pilosa/data0"
-    bind = "https://localhost:10100"
-
-    [gossip]
-      port = 12000
-      seeds = ["localhost:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = true
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"
-
-#### Node 1
-
-    data-dir = "/home/pilosa/data1"
-    bind = "https://localhost:10101"
-
-    [gossip]
-      port = 12001
-      seeds = ["localhost:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"
-      
-#### Node 2
-
-    data-dir = "/home/pilosa/data2"
-    bind = "https://localhost:10102"
-
-    [gossip]
-      port = 12002
-      seeds = ["localhost:12000"]
-      key = "/home/pilosa/private/gossip.key32"
-
-    [cluster]
-      replicas = 1
-      coordinator = false
-
-    [tls]
-      certificate = "/home/pilosa/private/server.crt"
-      key = "/home/pilosa/private/server.key"

docs/console.md

@@ -1,58 +0,0 @@
-+++
-title = "Console"
-weight = 9
-nav = [
-    "Installation",
-    "Query",
-    "Cluster Admin",
-]
-+++
-
-## Console
-
-A web-based app called Pilosa Console is available in a separate package. This can be used for constructing queries and viewing the cluster status.
-
-### Installation
-
-Releases are [available on Github](https://github.com/pilosa/console/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
-
-Installing on a Mac with Homebrew is simple; just run:
-
-```
-brew tap pilosa/homebrew-pilosa
-brew install pilosa-console
-```
-
-You may also build from source by checking out the [repo on Github](https://github.com/pilosa/console) and running:
-
-```
-make install
-```
-
-### Query
-
-The Query tab allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
-
-Each query's result will be displayed in the Output section along with the query time. 
-
-The Console will keep a record of each query and its result with the latest query on top.
-
-![Console screenshot](/img/docs/webui-console.png)
-*Console query screenshot*
-
-In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
-
-- `:create index <indexname>`
-- `:delete index <indexname>`
-- `:use <indexname>`
-- `:create field <fieldname>`
-- `:delete field <fieldname>`
-
-Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field).
-
-- `:create field <fieldname> cacheSize=10000`
-
-
-### Cluster Admin
-
-Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields.

docs/data-model.md

@@ -1,205 +0,0 @@
-+++
-title = "Data Model"
-weight = 5
-nav = [
-    "Overview",
-    "Index",
-    "Column",
-    "Row",
-    "Field",
-    "Time Quantum",
-    "Attribute",
-    "Shard",
-]
-+++
-
-## Data Model
-
-### Overview
-
-The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit; if the bit is set, it indicates that a relationship exists between that particular row and column.
-
-Rows and columns can represent anything (they could even represent the same set of things as in a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
-
-Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation—such as Intersect or Union—on multiple rows, are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
-
-Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 2<sup>63</sup> on a single-node cluster, for example, will not work well due to memory limitations.
-
-![basic data model diagram](/img/docs/data-model.png)
-*Basic data model diagram*
-
-### Index
-
-The purpose of the Index is to represent a data namespace. You cannot perform cross-index queries.
-
-### Column
-
-Column ids are sequential, increasing integers and they are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
-
-### Row
-
-Row ids are sequential, increasing integers namespaced to each Field within an Index.
-
-### Field
-
-Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field.
-
-#### Relational Analogy
-
-The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here.
-
-Entities:
-
- Relational  | Pilosa
--------------|----------------------------------------------
- Database    | N/A *(internal: Holder)*
- Table       | Index
- Row         | Column
- Column      | Field
- Value       | Row
- Value (int) | Field.Value (see [BSI](#bsi-range-encoding))
-
-Simple queries:
-
- Relational                                    | Pilosa
------------------------------------------------|------------------------------------
- `select ID from People where Name = 'Bob'`    | `Row(Name="Bob")`
- `select ID from People where Age > 30`        | `Row(Age > 30)`
- `select ID from People where Member = true`   | `Row(Member=0)`
-
-Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join:
-
-```sql
-select AVG(p.Age) from People p
-inner join PersonCar pc on pc.PersonID=p.ID
-inner join Cars c on pc.CarID=c.ID
-where c.Make = 'Ford'
-```
-
-can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed):
-
-```pql
-Sum(Row(Car-Make="Ford"), field=Age)
-```
-
-This is one major component of Pilosa's ability to combine relationships from multiple data stores.
-
-#### Ranked
-
-Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation.
-
-![ranked field diagram](/img/docs/field-ranked.png)
-*Ranked field diagram*
-
-#### LRU
-
-The LRU cache maintains the most recently accessed Rows.
-
-![lru field diagram](/img/docs/field-lru.png)
-*LRU field diagram*
-
-### Time Quantum
-
-Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to `YMD`, ranged Row queries down to the granularity of a day are supported.
-
-### Attribute
-
-Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure.
-
-Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row.
-
-### Shard
-
-Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
-
-Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
-
-### Field Type
-
-Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`.
-
-#### Set
-
-Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
-
-``` request
-curl localhost:10101/index/repository/field/info \
-     -X POST \
-     -d '{"options": {"type": "set", "cacheType": "ranked", "cacheSize":100000}}'
-```
-``` response
-{"success":true}
-```
-
-#### Int
-Fields of type `int` are used to store integer values. Integer fields share the same columns as the other fields in the index, but values for the field must be integers that fall between the `min` and `max` values specified when creating the field. The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
-
-``` request
-curl localhost:10101/index/repository/field/quantity \
-     -X POST \
-     -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
-```
-``` response
-{"success":true}
-```
-
-##### BSI Range-Encoding
-
-Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Row`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
-
-Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
-
-For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below:
-
-```
-Set(1, A=1)
-Set(2, A=2)
-Set(3, A=3)
-Set(4, A=7)
-Set(2, B=1)
-Set(3, B=6)
-```
-
-![BSI field diagram](/img/docs/field-bsi.png)
-*BSI field diagram*
-
-Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.
-
-
-###### BSI Deprecated Format
-
-The original implementation of BSI required a fixed bit depth when creating fields because the existence bit was written to the bit above the highest bit. The second version of BSI moves the existence bit to the beginning, adds a negative bit as the second bit, and shifts all remaining bits up by two.
-
-Pilosa automatically converts all old data to the new format on startup, however, this can cause issues when upgrading Pilosa and then reverting back to an old version. This documentation section exists as a record for anyone who experiences unusual behavior in BSI between versions.
-
-
-#### Time
-
-Time fields are similar to `set` fields, but in addition to row and column information, they also store a per-bit time value down to a defined granularity. The following example creates a `time` field called "event" which stores timestamp information down to a day granularity.
-
-``` request
-curl localhost:10101/index/repository/field/event \
-     -X POST \
-     -d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
-```
-``` response
-{"success":true}
-```
-
-With `time` fields, data views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
-
-```
-Set(3, A=8, 2017-05-18T00:00)
-Set(3, A=8, 2017-05-19T00:00)
-```
-
-![time quantum field diagram](/img/docs/field-time-quantum.png)
-*Time quantum field diagram*
-
-#### Mutex
-
-Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field.
-
-#### Boolean
-
-A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values.

docs/examples.md

@@ -1,221 +0,0 @@
-+++
-title = "Examples"
-weight = 4
-nav = [
-    "Transportation",
-]
-+++
-
-## Examples
-
-### Transportation
-
-#### Introduction
-
-New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set.
-
-Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
-
-We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
-
-After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
-
-#### Data Model
-
-The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at:
-
-* Distance: miles, floating point
-* Fare: dollars, floating point
-* Number of passengers: integer
-* Dropoff location: latitude and longitude, floating point
-* Pickup location: latitude and longitude, floating point
-* Dropoff time: timestamp
-* Pickup time: timestamp
-
-We import these fields, creating one or more Pilosa fields from each of them:
-
-field	|mapping
-------------|---------------------
-cab_type	|direct map of enum int → row ID
-dist_miles	|round(dist) → row ID
-total_amount_dollars	|round(dist) → row ID
-passenger_count	|direct map of integer value → row ID
-drop_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
-drop_year	|year(timestamp) → row ID
-drop_month	|month(timestamp) → row ID
-drop_day	|day(timestamp) → row ID
-drop_time	|time of day mapped to one of 48 half-hour buckets
-pickup_grid_id	|(lat, lon) → 100x100 rectangular grid → cell ID
-pickup_year	|year(timestamp) → row ID
-pickup_month	|month(timestamp) → row ID
-pickup_day	|day(timestamp) → row ID
-pickup_time	|time of day mapped to one of 48 half-hour buckets → row ID
-
-We also created two extra fields that represent the duration and average speed of each ride:
-
-field	|mapping
---------------------|-------------
-duration_minutes	|round(drop_timestamp - pickup_timestamp) → row ID
-speed_mph	|round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
-
-#### Mapping
-
-Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
-
-##### 0 columns → 1 field
-
-**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant.
-
-##### 1 column → 1 field
-
-The following three fields are mapped in a simple direct way from single columns of the original data.
-
-**dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
-
-In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code:
-```go
-lfm := pdk.LinearFloatMapper{
-    Min: -0.5,
-    Max: 3600.5,
-    Res: 3601,
-}
-```
-
-`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
-
-This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). <!-- TODO update so this makes sense -->
-```go
-pdk.ColumnMapper{
-    Field:   "dist_miles",
-    Mapper:  lfm,
-    Parsers: []pdk.Parser{pdk.FloatParser{}},
-    Fields:  []int{fields["trip_distance"]},
-},
-```
-
-These same objects are represented in the JSON definition file:
-```go
-{
-    "Fields": {
-        "Trip_distance": 10
-    },
-    "Mappers": [
-        {
-            "Name": "lfm0",
-            "Min": -0.5,
-            "Max": 3600.5,
-            "Res": 3600
-        }
-    ],
-    "ColumnMappers": [
-        {
-            "Field": "dist_miles",
-            "Mapper": {
-                "Name": "lfm0"
-            },
-            "Parsers": [
-                {"Name": "FloatParser"}
-            ],
-            "Fields": "Trip_distance"
-        }
-    ]
-}
-```
-
-Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of ColumnMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the ColumnMapper definitions to keep things human-readable.
-
-**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The ColumnMapper definition is very similar to the previous one.
-
-**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
-
-##### 1 column → multiple fields
-
-When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
-
-We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day".
-
-We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day".
-
-We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
-
-##### Multiple columns → 1 field
-
-The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
-
-We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id.
-
-Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
-
-##### Complex mappings
-
-We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
-```go
-durm := pdk.CustomMapper{
-    Func: func(fields ...interface{}) interface{} {
-        start := fields[0].(time.Time)
-        end := fields[1].(time.Time)
-        return end.Sub(start).Minutes()
-    },
-    Mapper: lfm,
-}
-```
-
-#### Import process
-
-After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. For more details, see the [PDK](../pdk/) section, or check out the [code](https://github.com/pilosa/pdk/tree/master/usecase/taxi) itself.
-
-#### Queries
-
-Now we can run some example queries.
-
-Count per cab type can be retrieved, sorted, with a single PQL call.
-
-```request
-TopN(cab_type)
-```
-```response
-{"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]}
-```
-
-High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
-
-```request
-TopN(pickup_grid_id)
-```
-```response
-{"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]}
-```
-
-Average of `total_amount` per `passenger_count` can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
-
-```python
-import pilosa
-
-client = pilosa.Client()
-schema = client.schema()
-taxi = schema.index("taxi")
-passenger_count = taxi.field("passenger_count")
-total_amount_dollars = taxi.field("total_amount_dollars")
-
-queries = []
-pcounts = range(10)
-for i in pcounts:
-    queries.append(total_amount_dollars.topn(passenger_count.row(i))
-query = taxi.batch_query(**queries)
-results = client.query(query)
-resp = requests.post(qurl, data=queries)
-
-average_amounts = []
-for pcount, result in zip(pcounts, resp.results):
-    wsum = sum([r.count * r.id for r in result.count_items])
-    count = sum([r.count for r in result.count_items])
-    average_amounts.append(float(wsum)/count)
-```
-
-<div class="note">
-Note that the <a href="../data-model/#bsi-range-encoding">BSI</a>-powered <a href="../query-language/#sum">Sum</a> query now provides an alternative approach to this kind of query.
-</div>
-
-<!-- Disabled until we have the time to update the Jupyter notebook --YT
-For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
--->

docs/faq.md

@@ -1,43 +0,0 @@
-+++
-title = "FAQ"
-weight = 15
-nav = []
-+++
-
-## FAQ
-
-### What is Pilosa?
-
-Pilosa is an in-memory, distributed index that is layered over persistent storage. It supports fast ad-hoc queries and segmentation. Pilosa does not require the underlying data to be moved, rather it can be populated in conjunction with data writes, or it can be backfilled asynchronously from any other data store or event processing system. This allows Pilosa to support sub-second queries against very large underlying data sets.
-
-### Is Pilosa a database?
-
-Pilosa is not a database in the traditional sense. While Pilosa does store data (both in-memory as well as persisted to disk), it wouldn't typically be used as a primary data store. Instead, one would likely use Pilosa as an index of the data stored in a traditional database or in a data warehouse.
-
-### Where does Pilosa fit in my stack?
-
-Pilosa was designed to index the relationships in your data. Pilosa runs along with your existing stack, integrating with one or more backing data stores. Pilosa can connect through a stream platform like Kafka or application integration via [PDK](../pdk/).
-
-### How is Pilosa different from Elasticsearch since they are both indexes?
-
-Elasticsearch is a search engine based on Lucene, and is therefore very good at indexing and searching large volumes of unstructured text. As it matures, Elasticsearch has continued to move into the analytics space, but its core data object is still the "document". Pilosa is specifically designed to index structured data and improve query speed. By representing data as the relationship between objects, and then storing those relationships in bitmaps, Pilosa can very efficiently search and compare many millions of data points while still maintaining a small memory footprint.
-
-### How do I get my data into Pilosa?
-
-There are typically two methods for getting data into Pilosa: importing large batches of data from an existing data set, and continuously updating Pilosa as data is added or updated.
-
-In the first case, one would use the `pilosa import` command to bulk load structured data into Pilosa. In order to improve this process, one can use the Pilosa Development Kit (PDK) to map structured data in the original data set onto the Pilosa schema.
-
-For the case where data is continually mutating, one would apply a parallel data writer at the point at which data is written to the persistent data store. This new writer would simultaneously write to Pilosa. An example use case would be one where Kafka was employed as the message broker in your data pipeline, you could introduce an additional Kafka consumer to read from the message log and write mutated data to Pilosa.
-
-### What languages can I use with it?
-
-There is currently [client support](../client-libraries/) for [Go](https://github.com/pilosa/go-pilosa), [Python](https://github.com/pilosa/python-pilosa), and [Java](https://github.com/pilosa/java-pilosa). If you want to use Pilosa with a different language, you can access Pilosa via the [Pilosa API](../api-reference/).
-
-### Do you query Pilosa using SQL?
-
-One can access Pilosa directly via the terminal using the [Pilosa Query Language](../query-language/) (PQL), but a typical implementation would use one of the Pilosa client libraries to integrate with an existing codebase. There is currently client support for Go, Python, and Java.
-
-### Replication on each node?
-
-Pilosa supports a replication factor greater than or equal to one. When replication is configured to be greater than one, then all mutations will be replicated to additional nodes in the cluster. For example, in a five-node cluster consisting of nodes A-B-C-D-E and with replication factor of three, then a write to node B will result in data being written to nodes B, C, and D. If the replication factor is greater than the number of nodes in the cluster, the data will be replicated to every node in the cluster only once.

docs/getting-started.md

@@ -1,970 +0,0 @@
-+++
-title = "Getting Started"
-weight = 3
-nav = [
-     "Starting Pilosa",
-     "Sample Project",
-     "Using Curl",
-     "Using Go",
-     "Using Java",
-     "Using Python",
-     "What's Next?",
-]
-+++
-
-## Getting Started
-
-Pilosa supports an HTTP interface which uses JSON by default.
-Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. However, the best way to interface with the Pilosa server is through one of our three official client libraries. Pilosa currently supports [Go](https://github.com/pilosa/go-pilosa), [Java](https://github.com/pilosa/java-pilosa), and [Python](https://github.com/pilosa/python-pilosa).
-
-<div class="note">
-    <p>Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. See <a href="/docs/administration/#open-file-limits">Open File Limits</a> for more details.</p>
-</div>
-
-### Starting Pilosa
-
-Follow the steps in the [Installation](../installation/) document to install Pilosa.
-Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at [localhost:10101](http://localhost:10101)):
-```
-pilosa server
-```
-
-Let's make sure Pilosa is running:
-``` request
-curl localhost:10101/status
-```
-``` response
-{"state":"NORMAL","nodes":[{"id":"91715a50-7d50-4c54-9a03-873801da1cd1","uri":{"scheme":"http","host":"localhost","port
-":10101},"isCoordinator":true}],"localID":"91715a50-7d50-4c54-9a03-873801da1cd1"}
-```
-
-### Sample Project
-
-In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about 1,000 popular Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language and stargazers—people who have starred a project.
-
-Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and stargazers. We can better organize the rows by grouping them into sets called Fields. So the "repository" index might have a "languages" field as well as a "stargazers" field. You can learn more about indexes and fields in the [Data Model](../data-model/) section of the documentation.
-
-<div class="note">
-    <p>If at any time you want to verify the data structure, you can request the schema as follows:</p>
-</div>
-
-```request
-curl localhost:10101/schema
-```
-```response
-{
-  "indexes": [
-    {
-      "name": "repository",
-      "options": {
-        "keys": false,
-        "trackExistence": true
-      },
-      "fields": [
-        {
-          "name": "language",
-          "options": {
-            "type": "set",
-            "cacheType": "ranked",
-            "cacheSize": 50000,
-            "keys": false
-          }
-        },
-        {
-          "name": "stargazer",
-          "options": {
-            "type": "time",
-            "timeQuantum": "YMDH",
-            "keys": false,
-            "noStandardView": false
-          }
-        }
-      ],
-      "shardWidth": 1048576
-    }
-  ]
-}
-```
-<div class="note">
-    <p>Note: This is the response you should receive once completing this project. It has also been formatted using <a href="https://stedolan.github.io/jq/"><code>jq</code></a>. </p>
-</div>
-
-#### Using Curl
-
-##### Create the Schema
-
-Before we can import data or run queries, we need to create our indexes and the fields within them. Let's create the `repository` index first:
-``` request
-curl localhost:10101/index/repository -X POST
-```
-``` response
-{"success":true}
-```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
-
-Let's create the `stargazer` field which has user IDs of stargazers as its rows:
-``` request
-curl localhost:10101/index/repository/field/stargazer \
-     -X POST \
-     -d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
-```
-``` response
-{"success":true}
-```
-
-Since our data contains time stamps which represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
-
-Next up is the `language` field, which will contain IDs for programming languages:
-``` request
-curl localhost:10101/index/repository/field/language \
-     -X POST
-```
-``` response
-{"success":true}
-```
-
-The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options.
-
-##### Import Data From CSV Files
-
-Download the `stargazer.csv` and `language.csv` files here:
-
-```
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
-```
-
-Run the following commands to import the data into Pilosa:
-
-```
-pilosa import -i repository -f stargazer stargazer.csv
-pilosa import -i repository -f language language.csv
-```
-
-If you are using a Docker container for Pilosa (with name `pilosa`), you should instead copy the `*.csv` file into the container and then import them:
-```
-docker cp stargazer.csv pilosa:/stargazer.csv
-docker exec -it pilosa /pilosa import -i repository -f stargazer /stargazer.csv
-docker cp language.csv pilosa:/language.csv
-docker exec -it pilosa /pilosa import -i repository -f language /language.csv
-```
-
-Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
-
-##### Make Some Queries
-
-Which repositories did user 14 star:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Row(stargazer=14)'
-```
-``` response
-{
-    "results":[
-        {
-            "attrs":{},
-            "columns":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
-        }
-    ]
-}
-```
-
-What are the top 5 languages in the sample data:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'TopN(language, n=5)'
-```
-``` response
-{
-    "results":[
-        [
-            {"id":5,"count":119},
-            {"id":1,"count":50},
-            {"id":4,"count":48},
-            {"id":9,"count":31},
-            {"id":13,"count":25}
-        ]
-    ]
-}
-```
-
-Which repositories were starred by user 14 and 19:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Intersect(
-            Row(stargazer=14), 
-            Row(stargazer=19)
-        )'
-```
-``` response
-{
-    "results":[
-        {
-            "attrs":{},
-            "columns":[2,3,362,396,416,461,464,466,470,486]
-        }
-    ]
-}
-```
-
-Which repositories were starred by user 14 or 19:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Union(
-            Row(stargazer=14), 
-            Row(stargazer=19)
-        )'
-```
-``` response
-{
-    "results":[
-        {
-            "attrs":{},
-            "columns":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
-        }
-    ]
-}
-```
-
-Which repositories were starred by user 14 and 19 and also were written in language 1:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Intersect(
-            Row(stargazer=14), 
-            Row(stargazer=19),
-            Row(language=1)
-        )'
-```
-``` response
-{
-    "results":[
-        {
-            "attrs":{},
-            "columns":[2,362,416,461]
-        }
-    ]
-}
-```
-
-Set user 99999 as a stargazer for repository 77777:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Set(77777, stargazer=99999)'
-```
-``` response
-{"results":[true]}
-```
-
-Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. 
-Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
-
-#### Using Go
-
-Pilosa follows the Go policy of supporting the two most recent major versions of Go.
-
-##### Create the Environment
-
-Interacting with Pilosa in your go program is best accomplished using our client, go-pilosa. To install go-pilosa, open a new  terminal and download the library to your `GOPATH` using:
-``` 
-go get github.com/pilosa/go-pilosa
-```
-
-Create a project folder:
-```
-mkdir getting-started && cd getting-started
-```
-
-In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
-```
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
-```
-
-We will also create a file called `startrace.go` as follows:
-```
-touch startrace.go
-```
-This file will be used in the following sections.
-
-##### Create the Schema
-
-Before we can import data or run queries, we need to create our schema. You can see two imports from the go-pilosa repo, go-pilosa for the client, and csv for the CSV reader. Create the schema by creating a client (which will communicate our schema to Pilosa), creating a schema locally (which will contain our indexes and fields), and syncing with Pilosa. This is all done in the `startrace.go` file:
-``` 
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/pilosa/go-pilosa"
-	"github.com/pilosa/go-pilosa/csv"
-	"io/ioutil"
-	"log"
-)
-
-func main() {
-	// Create the Schema
-	client := pilosa.DefaultClient()
-	schema, _ := client.Schema()
-	// This is where the index will go later
-	// This is where the fields will go later
-	err := client.SyncSchema(schema)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-```
-
-Next, let's create the `repository` index:
-```
-	repository := schema.Index("repository")
-```
-
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
-
-Let's create the `stargazer` field which has user IDs of stargazers as its rows:
-```
-	stargazer := repository.Field("stargazer")
-```
-
-Next up is the `language` field, which will contain IDs for programming languages:
-```
-	language := repository.Field("language")
-```
-
-Your `startrace.go` file should look like:
-```
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/pilosa/go-pilosa"
-	"github.com/pilosa/go-pilosa/csv"
-	"io/ioutil"
-	"log"
-)
-
-func main() {
-	// Create the Schema
-	client := pilosa.DefaultClient()
-	schema, _ := client.Schema()
-	repository := schema.Index("repository")
-	stargazer := repository.Field("stargazer")
-	language := repository.Field("language")
-	err := client.SyncSchema(schema)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-```
-
-##### Import Data From CSV Files
-
-Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
-
-First, we will load our data into the `stargazer` field:
-```
-	stargazerFile, err := ioutil.ReadFile("stargazer.csv")
-	if err != nil {
-		log.Fatal(err)
-	}
-	format := "2006-01-02T15:04"
-	iterator = csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, bytes.NewReader(stargazerFile), format)
-	err = client.ImportField(stargazer, iterator)
-	if err != nil {
-		log.Fatal(err)
-	}
-```
-Since our `stargazer` data contains time stamps, which represent the time users starred repos, we will be using the `csv.NewColumnIteratorWithTimeStampFormat` function from the go-pilosa/csv package. This function takes the format of the csv files (`csv.RowIDColumnID`), an `io.Reader` (`bytes.NewReader(stargazerFile)`), and the time quantum format (`format`) and translates the csv file into a format Pilosa can read. Time quantum is the resolution of the time we want to use.
-
-Next, we will load our data into the `language` field:
-```
-	languageFile, err := ioutil.ReadFile("language.csv")
-	if err != nil {
-		log.Fatal(err)
-	}
-	iterator := csv.NewColumnIterator(csv.RowIDColumnID, bytes.NewReader(languageFile))
-	err = client.ImportField(language, iterator)
-	if err != nil {
-		log.Fatal(err)
-	}
-```
-Since our `language` data doesn't contain time stamps, we will use the `csv.NewColumnIterator` function in place of `csv.NewColumnIteratorWithTimeStampFormat`.
-
-Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
-
-For more information on imports in go-pilosa, please see the go-pilosa [site](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md).
-
-##### Make Some Queries
-
-Now that we have a working schema, we can query it.
-
-Which repositories did user 14 star:
-``` request
-response, err := client.Query(stargazer.Row(14))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("User 14 starred: ", response.Result().Row().Columns)
-```
-``` response
-User 14 starred:  [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514]
-```
-
-What are the top 5 languages in the sample data:
-``` request
-response, err = client.Query(language.TopN(5))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("Top Languages: ", response.Result().CountItems())
-```
-``` response
-Top Languages:  [{5  119} {1  50} {4  48} {9  31} {13  25}]
-```
-
-Which repositories were starred by user 14 and 19:
-``` request
-response, err = client.Query(repository.Intersect(stargazer.Row(14), stargazer.Row(19)))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("Both user 14 and 19 starred: ", response.Result().Row().Columns)
-```
-``` response
-Both user 14 and 19 starred:  [2 3 362 396 416 461 464 466 470 486]
-```
-
-Which repositories were starred by user 14 or 19:
-``` request
-response, err = client.Query(repository.Union(stargazer.Row(14), stargazer.Row(19)))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("User 14 or 19 starred: ", response.Result().Row().Columns)
-```
-``` response
-User 14 or 19 starred:  [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514]
-```
-
-Which repositories were starred by user 14 and 19 and also were written in language 1:
-``` request
-response, err = client.Query(repository.Intersect(stargazer.Row(14), stargazer.Row(19), language.Row(1)))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("Both user 14 and 19 starred and were written in language 1: ", response.Result().Row().Columns)
-```
-``` response
-Both user 14 and 19 starred and were written in language 1:  [2 362 416 461]
-```
-
-Set user 99999 as a stargazer for repository 77777:
-``` request
-client.Query(stargazer.Set(99999, 77777))
-response, err = client.Query(stargazer.Row(99999))
-if err != nil {
-	log.Fatal(err)
-}
-fmt.Println("Set user 99999 as a stargazer for repository 77777")
-```
-``` response
-Set user 99999 as a stargazer for repository 77777
-```
-
-Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. 
-Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
-
-For more information about go-pilosa, please see our Go client library at [go-pilosa](https://github.com/pilosa/go-pilosa) or checkout the go-pilosa [Data Model and Queries](https://github.com/pilosa/go-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
-
-#### Using Java
-
-Pilosa requires Java 8 or higher and Maven 3 or higher.
-
-##### Create the Environment
-
-Create a project folder:
-```
-mkdir getting-started && cd getting-started
-```
-
-In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
-```
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
-```
-
-We will now create the java directory that will contain our `pom.xml` file and create the `pom.xml` file:
-```
-mkdir startrace && cd startrace
-touch pom.xml
-```
-
-For this specific project, the `pom.xml` file needs to contain:
-```
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>com.pilosa</groupId>
-    <artifactId>getting-started</artifactId>
-    <version>1.0.0</version>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.pilosa</groupId>
-            <artifactId>pilosa-client</artifactId>
-            <version>1.3.1</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.6.1</version>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-
-            <!-- Build an executable JAR -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>3.0.2</version>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <addClasspath>true</addClasspath>
-                            <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>main.java.StarTrace</mainClass>
-                        </manifest>
-                    </archive>
-                </configuration>
-            </plugin>
-
-            <!-- create an uber JAR -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-
-</project>
-```
-
-We will now create the java directory that will contain our `StarTrace.java` file and create the `StarTrace.java` file:
-```
-mkdir -p src/main/java && cd src/main/java
-touch StarTrace.java
-```
-
-This file will be used in the following sections.
-
-##### Create the Schema
-
-Before we can import data or run queries, we need to create our schema. You can see the first six dependencies are imported from the java-pilosa library. Create the schema by creating a client which will communicate our schema to Pilosa, creating a schema which will contain our indexes and fields, and syncing with Pilosa. This is all done in the `StarTrace.java` file:
-```
-package main.java;
-
-import com.pilosa.client.PilosaClient;
-import com.pilosa.client.QueryResponse;
-import com.pilosa.client.exceptions.PilosaException;
-import com.pilosa.client.orm.*;
-import com.pilosa.client.csv.FileRecordIterator; 
-import com.pilosa.client.TimeQuantum;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-
-public class StarTrace {
-    public static void main(String []args) throws IOException {
-        // Create the Schema
-        PilosaClient client = PilosaClient.defaultClient();
-        Schema schema = client.readSchema();
-        // This is were the index will go later
-	// This is were the fields will go later
-        client.syncSchema(schema);
-    }
-}
-```
-
-Next, let's create the `repository` index:
-```
-	Index repository = schema.index("repository");
-```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
-
-Let's create the `stargazer` field which has user IDs of stargazers as its rows:
-```
-	FieldOptions stargazerOptions = FieldOptions.builder()
-            .fieldTime(TimeQuantum.YEAR_MONTH_DAY)
-            .build();
-        Field stargazer = repository.field("stargazer", stargazerOptions);
-```
-Since our data contains time stamps which represent the time users starred repos, we set the field type to `time` using `fieldTime()`. Time quantum is the resolution of the time we want to use, and we set it to `YEAR_MONTH_DAY` for `stargazer`.
-
-Next up is the `language` field, which will contain IDs for programming languages:
-```
-	Field language = repository.field("language");
-```
-The `language` field is a `set` field, but since the default field type is `set`, we don't need to specify it
-
-Your `StarTrace.java` file should look like:
-```
-package main.java;
-
-import com.pilosa.client.PilosaClient;
-import com.pilosa.client.QueryResponse;
-import com.pilosa.client.exceptions.PilosaException;
-import com.pilosa.client.orm.*;
-import com.pilosa.client.csv.FileRecordIterator; 
-import com.pilosa.client.TimeQuantum;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-
-public class StarTrace {
-    public static void main(String []args) throws IOException {
-        // Create the Schema
-        PilosaClient client = PilosaClient.defaultClient();
-        Schema schema = client.readSchema();
-        Index repository = schema.index("repository");
-
-        FieldOptions stargazerOptions = FieldOptions.builder()
-            .fieldTime(TimeQuantum.YEAR_MONTH_DAY)
-            .build();
-        Field stargazer = repository.field("stargazer", stargazerOptions);
-        
-        Field language = repository.field("language");
-        client.syncSchema(schema);
-    }
-}
-```
-
-##### Import Data From CSV Files
-
-Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
-
-First, we will load our data into the `stargazer` field:
-```
-	SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm");
-        FileRecordIterator iterator = FileRecordIterator.fromPath("stargazer.csv", stargazer, timestampFormat);
-        client.importField(stargazer, iterator);
-```
-Due to the time aspect of the `stargazer` csv file, we have to specify the time stamp format in the `fromPath` function. We set the variable `timestampFormat` to the format present in the csv file using the function `SimpleDateFormat()` and pass the variable to the `fromPath` function, which will take the csv file name, the field name, and the time stamp format and translate the csv file into a format Pilosa can read.
-
-Next, we will load our data into the `language` field:
-```
-	iterator = FileRecordIterator.fromPath("language.csv", language);
-        client.importField(language, iterator);
-```
-Since our `language` data doesn't have a time aspect, the time stamp format doesn't need to be specified.
-
-Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
-
-For more information on imports in java-pilosa, please see the java-pilosa [site](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md).
-
-##### Make Some Queries
-
-Now that we have a working schema, we can query it.
-
-Which repositories did user 14 star:
-``` request
-QueryResponse response = client.query(stargazer.row(14));
-System.out.println("User 14 starred: " + response.getResult().getRow().getColumns());
-```
-``` response
-User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
-```
-
-What are the top 5 languages in the sample data:
-``` request
-response = client.query(language.topN(5));
-System.out.println("Top Languages: " + response.getResult().getCountItems());
-```
-``` response
-Top Languages: [CountResultItem(id=5, count=119), CountResultItem(id=1, count=50), CountResultItem(id=4, count=48), CountResultItem(id=9, count=31), CountResultItem(id=13, count=25)]
-```
-
-Which repositories were starred by user 14 and 19:
-``` request
-response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19)));
-System.out.println("Both user 14 and 19 starred: " + response.getResult().getRow().getColumns());
-```
-``` response
-Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486]
-```
-
-Which repositories were starred by user 14 or 19:
-``` request
-response = client.query(repository.union(stargazer.row(14), stargazer.row(19)));
-System.out.println("User 14 or 19 starred: " + response.getResult().getRow().getColumns());
-```
-``` response
-User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514]
-```
-
-Which repositories were starred by user 14 and 19 and also were written in language 1:
-``` request
-response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19), language.row(1)));
-System.out.println("Both user 14 and 19 starred and were written in language 1: " + response.getResult().getRow().getColumns());
-```
-``` response
-Both user 14 and 19 starred and were written in language 1: [2, 362, 416, 461]
-```
-
-Set user 99999 as a stargazer for repository 77777:
-``` request
-client.query(stargazer.set(99999, 77777));
-System.out.println("Set user 99999 as a stargazer for repository 77777");
-```
-``` response
-Set user 99999 as a stargazer for repository 77777
-```
-
-Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. 
-Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
-
-For more information about java-pilosa, please see our Java client library at [java-pilosa](https://github.com/pilosa/java-pilosa) or checkout the java-pilosa [Data Model and Queries](https://github.com/pilosa/java-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
-
-#### Python Users
-
-Pilosa requires Python 2.7 or higher or Python 3.4 or higher.
-
-##### Create the Environment
-
-Create a new project folder:
-```
-mkdir getting-started && cd getting-started
-```
-In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
-```
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
-```
-We will also download two text files. One is the `requirements.txt` that will install python-pilosa later on and the other is `languages.txt` which will provide context to the `language` field.
-```
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/python/requirements.txt
-curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.txt
-```
-We will now create the python environment:
-```
-python3 -m venv startrace
-```
-
-Next, we activate the python environment we created and install the single dependency, python-pilosa:
-```
-source startrace/bin/activate
-pip install -r requirements.txt
-```
-We will also create a file called `startrace.py` as follows:
-```
-touch startrace.py
-```
-This file will be used in the following sections.
-
-##### Create the Schema
-
-Before we can import data or run queries, we need to create our schema. You can see the dependencies dealing with `pilosa` are from the python-pilosa library. Create the schema by creating a client which will communicate our schema to Pilosa, creating a schema which will contain our indexes and fields, and syncing with Pilosa. This is all done in the `startrace.py` file:
-``` 
-from __future__ import print_function
-
-import os
-import sys
-import time
-import pilosa
-
-from pilosa import Client, Index, TimeQuantum
-from pilosa.imports import csv_column_reader, csv_row_id_column_id
-
-try:
-	# Python 2.7 and 3
-	from io import StringIO
-except ImportError:
-	# Python 2.6 and 2.7
-	from StringIO import StringIO
-
-# Create the Schema
-client = pilosa.Client()
-schema = client.schema()
-# This is where the index will go later
-# This is where the fields will go later
-client.sync_schema(schema)
-```
-Next, let's create the `repository` index:
-```
-repository = schema.index("repository")
-```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
-
-Let's create the `stargazer` field which has user IDs of stargazers as its rows:
-```
-stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY)
-```
-Since our data contains time stamps which represent the time users starred repos, we establish the time aspect by using `time_quantum`. Time quantum is the resolution of the time we want to use, and we set it to `YEAR_MONTH_DAY` for `stargazer`.
-
-Next up is the `language` field, which will contain IDs for programming languages:
-```
-language = repository.field("language")
-```
-The `language` field is a `set` field, but since the defualt field is `set`, we didn't need to specify any options.
-
-Your `StarTrace.py` file should look like:
-```
-from __future__ import print_function
-
-import os
-import sys
-import time
-import pilosa
-
-from pilosa import Client, Index, TimeQuantum
-from pilosa.imports import csv_column_reader, csv_row_id_column_id
-
-try:
-        # Python 2.7 and 3
-        from io import StringIO
-except ImportError:
-        # Python 2.6 and 2.7
-        from StringIO import StringIO
-
-# Create the Schema
-client = pilosa.Client()
-schema = client.schema()
-repository = schema.index("repository")
-stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY)
-language = repository.field("language")
-client.sync_schema(schema)
-```
-
-##### Import Data From CSV Files
-
-Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
-
-First, we will load our data into the `stargazer` field:
-```
-time_func = lambda s: int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M")))
-with open("stargazer.csv") as f:
-    stargazer_reader = csv_column_reader(f, timefunc=time_func)
-    client.import_field(stargazer, stargazer_reader)
-```
-Due to the time aspect of the `stargazer` csv file, we have to specify the time stamp format in the `csv_column_reader` function. We set the variable `time_func` to the format present in the csv file and call it in the `csv_column_reader` function, which will take the csv file and the time stamp format and translate the csv file into a format Pilosa can read
-
-Next, we will load our data into the `language` field:
-```
-with open("language.csv") as f:
-    language_reader = csv_column_reader(f, csv_row_id_column_id)
-    client.import_field(language, language_reader)
-```
-
-The `language` is a `set` field, but since the default field type is `set`, we didn't need to specify it.
-
-For more information on imports in python-pilosa, please see the python-pilosa [site](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md).
-
-Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
-
-##### Make Some Queries
-
-Now that we have a working schema, we can query it.
-
-Which repositories did user 14 star:
-``` request
-response = client.query(stargazer.row(14))
-print("User 14 starred: ", response.result.row.columns)
-```
-``` response
-User 14 starred:  [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
-```
-
-What are the top 5 languages in the sample data:
-``` request
-def load_language_names():
-    with open("languages.txt") as f:
-        return [line.strip() for line in f]
-
-def print_topn(items):
-    lines = ["\t{i}. {s[0]}: {s[1]} stars".format(s=s, i=i + 1) for i, s in enumerate(items)]
-    print("\n".join(lines))
-
-language_names = load_language_names()
-top_languages = client.query(language.topn(5)).result.count_items
-language_items = [(language_names[item.id], item.count) for item in top_languages]
-print("Top languages: ")
-print_topn(language_items)
-```
-``` response
-Top languages: 
-        1. Go: 119 stars
-        2. Shell: 50 stars
-        3. Makefile: 48 stars
-        4. HTML: 31 stars
-        5. JavaScript: 25 stars
-```
-
-Which repositories were starred by user 14 and 19:
-``` request
-repsonse = client.query(repository.intersect(stargazer.row(14), stargazer.row(19)))
-print("Both user 14 and 19 starred: ", response.result.row.columns)
-```
-``` resposne
-Both user 14 and 19 starred:  [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
-```
-
-Which repositories were starred by user 14 or 19:
-``` request
-response = client.query(repository.union(stargazer.row(14), stargazer.row(19)))
-print("User 14 or 19 starred: ", response.result.row.columns)
-```
-``` response
-User 14 or 19 starred:  [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514]
-```
-
-Which repositories were starred by user 14 and 19 and also were written in language 1:
-``` request
-response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19), language.row(1)))
-print("Both user 14 and 19 starred and were written in language 1: ", response.result.row.columns)
-```
-``` response
-Both user 14 and 19 starred and were written in language 1:  [2, 362, 416, 461]
-```
-
-Set user 99999 as a stargazer for repository 77777:
-``` request
-client.query(stargazer.set(99999, 77777))
-print("Set user 99999 as a stargazer for repository 77777")
-```
-``` response
-Set user 99999 as a stargazer for repository 77777
-```
-
-Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. 
-Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
-
-For more information about python-pilosa, please see our Python client library at [python-pilosa](https://github.com/pilosa/python-pilosa) or checkout the python-pilosa [Data Model and Queries](https://github.com/pilosa/python-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
-
-### What's Next?
-
-You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).

docs/glossary.md

@@ -1,77 +0,0 @@
-+++
-title = "Glossary"
-weight = 14
-nav = []
-+++
-
-## Glossary
-
-<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
-
-<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
-
-<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column).
-
-<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap).
-
-<strong id="bsi">[BSI](../data-model/#bsi-range-encoding):</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in `int` [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
-
-<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
-
-<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index).
-
-<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
-
-<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of five types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
-
-<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.
-
-<strong id="gossip">[Gossip](https://en.wikipedia.org/wiki/Gossip_protocol):</strong> A protocol used by Pilosa for internal communication.
-
-<strong id="groupby">[GroupBy](../query-language/#group-by):</strong> A [PQL](#pql) query, with functionality similar to a SQL `GROUP BY` clause, that returns the count of the intersection of every combination of rows taking one row each from the specified `Rows` calls. GroupBy can be thought of as a multi-dimensional version of the [TopN](#topn) query.
-
-<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Basic queries cannot operate across multiple indexes.
-
-<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
-
-<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field).
-
-<strong id="maxshard">MaxShard:</strong> The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5.
-
-<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field).
-
-<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
-
-<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
-
-<strong id="pql">[PQL](../query-language/):</strong> Pilosa Query Language.
-
-<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
-
-<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
-
-<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
-
-<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
-
-<strong id="range">[Row (Ranged)](../query-language/#row-range):</strong> A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
-
-<strong id="range-bsi">[Row (BSI)](../query-language/#row-bsi):</strong> A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
-
-<strong id="rows">[Rows](../query-language/#rows):<strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
-
-<strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
-
-<strong id="shard">[Shard](../data-model/#shard):</strong> [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
-
-<strong id="shardwidth">ShardWidth:</strong> This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
-
-<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field).
-
-<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [ranged Row](#range) queries on time [fields](#field).
-
-<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
-
-<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field).
-
-<strong id="view">View:</strong> Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

docs/installation.md

@@ -1,382 +0,0 @@
-+++
-title = "Installation"
-weight = 2
-nav = [
-    "Installing on MacOS",
-    "Installing on Linux",
-]
-+++
-
-
-## Installation
-
-Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#installing-on-linux).
-
-### Installing on MacOS
-
-There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) (recommended), download the binary, build from source, or use [Docker](#docker).
-
-#### Use Homebrew
-
-1. Update your Homebrew formulas:
-    ```
-    brew update
-    ```
-
-2. Install Pilosa
-    ```
-    brew install pilosa
-    ```
-
-3. Make sure Pilosa is installed successfully:
-    ```
-    pilosa
-    ```
-
-    If you see something like:
-    ```
-    Pilosa is a fast index to turbocharge your database.
-
-    This binary contains Pilosa itself, as well as common
-    tools for administering pilosa, importing/exporting data,
-    backing up, and more. Complete documentation is available
-    at https://www.pilosa.com/docs/.
-
-    Version: v1.4.0
-    Build Time: 2018-05-14T22:14:01+0000
-
-    Usage:
-      pilosa [command]
-
-    Available Commands:
-      check           Do a consistency check on a pilosa data file.
-      config          Print the current configuration.
-      export          Export data from pilosa.
-      generate-config Print the default configuration.
-      help            Help about any command
-      import          Bulk load data into pilosa.
-      inspect         Get stats on a pilosa data file.
-      server          Run Pilosa.
-
-    Flags:
-      -c, --config string   Configuration file to read from.
-      -h, --help            help for pilosa
-
-    Use "pilosa [command] --help" for more information about a command.
-    ```
-
-    You're good to go!
-
-#### Download the Binary
-
-1. Download the latest release:
-    ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.4.0/pilosa-v1.4.0-darwin-amd64.tar.gz
-    ```
-
-    Other releases can be downloaded from our Releases page on Github.
-
-2. Extract the binary:
-    ```
-    tar xfz pilosa-v1.4.0-darwin-amd64.tar.gz
-    ```
-
-3. Move the binary into your PATH so you can run `pilosa` from any shell:
-    ```
-    cp -i pilosa-v1.4.0-darwin-amd64/pilosa /usr/local/bin
-    ```
-
-4. Make sure Pilosa is installed successfully:
-    ```
-    pilosa
-    ```
-
-    If you see something like:
-    ```
-    Pilosa is a fast index to turbocharge your database.
-
-    This binary contains Pilosa itself, as well as common
-    tools for administering pilosa, importing/exporting data,
-    backing up, and more. Complete documentation is available
-    at https://www.pilosa.com/docs/.
-
-    Version: v1.4.0
-    Build Time: 2018-05-14T22:14:01+0000
-
-    Usage:
-      pilosa [command]
-
-    Available Commands:
-      check           Do a consistency check on a pilosa data file.
-      config          Print the current configuration.
-      export          Export data from pilosa.
-      generate-config Print the default configuration.
-      help            Help about any command
-      import          Bulk load data into pilosa.
-      inspect         Get stats on a pilosa data file.
-      server          Run Pilosa.
-
-    Flags:
-      -c, --config string   Configuration file to read from.
-      -h, --help            help for pilosa
-
-    Use "pilosa [command] --help" for more information about a command.
-    ```
-
-    You're good to go!
-
-#### Build from Source
-
-<div class="note">
-    <p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
-</div>
-
-1. Install the prerequisites:
-
-    * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
-    * [Git](https://git-scm.com/)
-
-2. Clone the repo:
-    ```
-    mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
-    git clone https://github.com/pilosa/pilosa.git
-    ```
-
-3. Build the Pilosa repo:
-    ```
-    cd $GOPATH/src/github.com/pilosa/pilosa
-    make install-build-deps
-    make install
-    ```
-
-4. Make sure Pilosa is installed successfully:
-    ```
-    pilosa
-    ```
-
-    If you see something like:
-    ```
-    Pilosa is a fast index to turbocharge your database.
-
-    This binary contains Pilosa itself, as well as common
-    tools for administering pilosa, importing/exporting data,
-    backing up, and more. Complete documentation is available
-    at https://www.pilosa.com/docs/.
-
-    Version: v1.4.0
-    Build Time: 2018-05-14T22:14:01+0000
-
-    Usage:
-      pilosa [command]
-
-    Available Commands:
-      check           Do a consistency check on a pilosa data file.
-      config          Print the current configuration.
-      export          Export data from pilosa.
-      generate-config Print the default configuration.
-      help            Help about any command
-      import          Bulk load data into pilosa.
-      inspect         Get stats on a pilosa data file.
-      server          Run Pilosa.
-
-    Flags:
-      -c, --config string   Configuration file to read from.
-      -h, --help            help for pilosa
-
-    Use "pilosa [command] --help" for more information about a command.
-    ```
-
-    You're good to go!
-
-#### What's next?
-
-Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
-
-
-### Installing on Linux
-
-There are three ways to install Pilosa on Linux: download the binary (recommended), build from source, or use [Docker](#docker).
-
-#### Download the Binary
-
-1. To install the latest version of Pilosa, download the latest release:
-    ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.4.0/pilosa-v1.4.0-linux-amd64.tar.gz
-    ```
-
-    Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
-
-2. Extract the binary:
-    ```
-    tar xfz pilosa-v1.4.0-linux-amd64.tar.gz
-    ```
-
-3. Move the binary into your PATH so you can run `pilosa` from any shell:
-    ```
-    cp -i pilosa-v1.4.0-linux-amd64/pilosa /usr/local/bin
-    ```
-
-4. Make sure Pilosa is installed successfully:
-    ```
-    pilosa
-    ```
-
-    If you see something like:
-    ```
-    Pilosa is a fast index to turbocharge your database.
-
-    This binary contains Pilosa itself, as well as common
-    tools for administering pilosa, importing/exporting data,
-    backing up, and more. Complete documentation is available
-    at https://www.pilosa.com/docs/.
-
-    Version: v1.4.0
-    Build Time: 2018-05-14T22:14:01+0000
-
-    Usage:
-      pilosa [command]
-
-    Available Commands:
-      check           Do a consistency check on a pilosa data file.
-      config          Print the current configuration.
-      export          Export data from pilosa.
-      generate-config Print the default configuration.
-      help            Help about any command
-      import          Bulk load data into pilosa.
-      inspect         Get stats on a pilosa data file.
-      server          Run Pilosa.
-
-    Flags:
-      -c, --config string   Configuration file to read from.
-      -h, --help            help for pilosa
-
-    Use "pilosa [command] --help" for more information about a command.
-    ```
-
-    You're good to go!
-
-#### Build from Source
-
-<div class="note">
-    <p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
-</div>
-
-1. Install the prerequisites:
-
-    * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
-    * [Git](https://git-scm.com/)
-
-2. Clone the repo:
-    ```
-    mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
-    git clone https://github.com/pilosa/pilosa.git
-    ```
-
-3. Build the Pilosa repo:
-    ```
-    cd $GOPATH/src/github.com/pilosa/pilosa
-    make install-build-deps
-    make install
-    ```
-
-4. Make sure Pilosa is installed successfully:
-    ```
-    pilosa
-    ```
-
-    If you see something like:
-    ```
-    Pilosa is a fast index to turbocharge your database.
-
-    This binary contains Pilosa itself, as well as common
-    tools for administering pilosa, importing/exporting data,
-    backing up, and more. Complete documentation is available
-    at https://www.pilosa.com/docs/.
-
-    Version: v1.4.0
-    Build Time: 2018-05-14T22:14:01+0000
-
-    Usage:
-      pilosa [command]
-
-    Available Commands:
-      check           Do a consistency check on a pilosa data file.
-      config          Print the current configuration.
-      export          Export data from pilosa.
-      generate-config Print the default configuration.
-      help            Help about any command
-      import          Bulk load data into pilosa.
-      inspect         Get stats on a pilosa data file.
-      server          Run Pilosa.
-
-    Flags:
-      -c, --config string   Configuration file to read from.
-      -h, --help            help for pilosa
-
-    Use "pilosa [command] --help" for more information about a command.
-    ```
-
-    You're good to go!
-
-#### What's next?
-
-Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
-
-
-### Windows
-
-Windows is currently not supported as a target deployment platform for Pilosa, but developing and running Pilosa is made possible by Docker. See the [Docker](#docker) documentation for using Docker for Windows and Docker Toolbox.
-
-Windows Subsystem for Linux is currently not supported.
-
-### Docker
-
-1. Install Docker for your platform. On Linux, Docker is available via your package manager. On MacOS, you can use Docker for Mac or Docker Toolbox. On Windows, you can use Docker for Windows or Docker Toolbox.
-
-2. **This step is necessary only if you are using Docker Toolbox**, otherwise skip to step 3:
-
-    a. Start the Docker support using `docker-machine start` in a terminal. The environment variables of the terminal should be updated accordingly, run `docker-machine env` to display the necessary commands.
-
-    b. Set up port forwarding in the VirtualBox GUI or on the command line. Guest port should be 10101. For the host port, 10101 is recommended. If the `VBoxManage` command is in your `PATH`, you can use the following command (assuming you use the default VM):
-
-    ```
-    VBoxManage modifyvm "default" --natpf1 "pilosa,tcp,,10101,,10101"
-    ```
-
-3. Confirm that the Docker daemon is running in the background:
-    ```
-    docker version
-    ```
-
-    If you are getting a "command not found" or similar, check that `docker` command is in your path. If you don't see the server listed, start the Docker application.
-
-
-4. Pull the official Pilosa image from Docker Hub:
-
-    ```
-    docker pull pilosa/pilosa:latest
-    ```
-
-5. Make sure Pilosa is installed successfully, and make it accessible:
-
-    ```
-    docker run -d --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest server --bind 0.0.0.0:10101
-    ```
-
-6. Check that it is accessible from outside the container.
-
-    Run the following in a separate terminal:
-    ```
-    curl localhost:10101/schema
-    ```
-
-    If that returns `{"indexes":null}` or similar, then Pilosa is accessible from outside the container. Otherwise check that you have correctly typed `-p 10101:10101` when running the Pilosa container and the port mappings in VirtualBox is correct (Docker Toolbox only).
-
-7. When you want to terminate the Pilosa container, you can run the following:
-    ```
-    docker stop pilosa
-    ```
-
-#### What's next?
-
-Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.

docs/introduction.md

@@ -1,19 +0,0 @@
-+++
-title = "Introduction"
-weight = 1
-nav = []
-+++
-
-
-## Introduction
-
-
-Pilosa is an open source, distributed index.
-
-[//]: # (TODO insert a graphic here?)
-
-It is designed primarily for speed and horizontal scalability. If you have data with billions of objects that can have millions of possible attributes, and you want to explore those relationships, Pilosa can help you.
-
-"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
-
-Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.

docs/pdk.md

@@ -1,74 +0,0 @@
-+++
-title = "PDK"
-weight = 11
-nav = [
-    "Examples and Executables",
-    "Library",
-]
-+++
-
-## PDK
-
-The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively.
-
-### Examples and Executables
-Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here.
-
-#### Kafka
-`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
-Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
-record from Kafka is assigned a Pilosa column, and each value in a record is
-assigned a row or field. Pilosa field names are built from the "path" through
-the record to arrive at that field. For example:
-
-```json
-{
-  "name": "jill",
-  "favorite_foods": ["corn chips", "chipotle dip"],
-  "location": {
-    "city": "Austin",
-    "state": "Texas",
-    "latitude": 3754,
-    "longitude": 4526
-  },
-  "active": true,
-  "age": 27
-}
-```
-
-This JSON object would result in the following Pilosa schema:
-
-| Field          | Example Value | Type   | Cache Size |
-|----------------|---------------|--------|------------|
-| name           | "jill"        | ranked |   100000   |
-| favorite_foods | "corn chips"  | ranked |   100000   |
-| default        |               | ranked |   100000   |
-| age            | 27            | int    |            |
-| location       |               | ranked |    1000    |
-| latitude       | 3754          | int    |            |
-| longitude      | 4526          | int    |            |
-| location-city  | "Austin"      | ranked |   100000   |
-| location-state | "Texas"       | ranked |   100000   |
-
-All set fields are created as ranked fields by default, with the cache size 
-listed above. Integer fields are created with a minimum size of zero and a 
-fixed maximum of 2147483647. Field names are a dash-separated concatenation of
-all key values in the path - you can see this with fields like location-city.
-
-
-Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
-kafka topics, kafka group, etc.), but there are a few options that give some
-control over the way data is indexed, and ingestion performance.
-
-* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
-* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state".
-* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
-* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
-* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.
-
-For more information on running `pdk kafka` and how Pilosa interfaces with Kafka, please see the [kafka directory](https://github.com/pilosa/pdk/tree/master/kafka) in the pdk repository.
-
-### Library
-
-For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation.
-

docs/query-language.md

@@ -1,943 +0,0 @@
-+++
-title = "Query Language"
-weight = 6
-nav = [
-    "Conventions",
-    "Arguments and Types",
-    "Write Operations",
-    "Read Operations",
-]
-+++
-
-## Query Language
-
-### Overview
-
-This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
-
-```
-{"results":[...]}
-```
-
-There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists its result type.
-
-#### Conventions
-
-* Angle Brackets `<>` denote required arguments
-* Square Brackets `[]` denote optional arguments
-* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`)
-
-##### Examples
-
-Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index and fields, and to populate them with some data.
-
-The examples just show the PQL quer(ies) needed - to run the query `Set(10, stargazer=1)` against a server using curl, you would:
-``` request
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Set(10, stargazer=1)'
-```
-``` response
-{"results":[true]}
-```
-
-#### Arguments and Types
-
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
-* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
-* `UINT` An unsigned integer (e.g. 42839).
-* `BOOL` A boolean value, `true` or `false`.
-* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`.
-* `ATTR_VALUE` Can be a string, float, integer, or bool.
-* `CALL` Any query.
-* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not`.
-* `ROWS_CALL` A query that returns a `Rows` result (i.e. a list of row IDs). Currently only the `Rows` query.
-* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`).
-
-### Write Operations
-
-#### Set
-
-**Spec:**
-
-```
-Set(<COLUMN>, <FIELD>=<ROW>, [TIMESTAMP])
-```
-
-**Description:**
-
-`Set` assigns a value of 1 to a bit in the binary matrix, thus associating the given row (the `<ROW>` value) in the given field with the given column.
-
-<div class="note">
-    <p>While using "Set" in PQL is a convenient way to get familiar with Pilosa, it's almost always better to use the import functionality in the <a href="https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md">Go</a>, <a href="https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md">Java</a>, and <a href="https://github.com/pilosa/python-pilosa/tree/master/docs/imports.md">Python</a> clients to ingest lots of data. </p>
-</div>
-
-**Result Type:** boolean
-
-A return value of `true` indicates that the bit was changed to 1.
-
-A return value of `false` indicates that the bit was already set to 1 and nothing changed.
-
-
-**Examples:**
-
-Set the bit at row 1, column 10:
-```request
-Set(10, stargazer=1)
-```
-```response
-{"results":[true]}
-```
-
-This sets a bit in the stargazer field, representing that the user with id=1 has starred the repository with id=10.
-
-Set also supports providing a timestamp. To write the date that a user starred a repository:
-```request
-Set(10, stargazer=1, 2016-01-01T00:00)
-```
-```response
-{"results":[true]}
-```
-
-Set multiple bits in a single request:
-```request
-Set(10, stargazer=1) Set(20, stargazer=1) Set(10, stargazer=2) Set(30, stargazer=2)
-```
-```response
-{"results":[false,true,true,true]}
-```
-
-Set the field "pullrequests" to integer value 2 at column 10:
-```request
-Set(10, pullrequests=2)
-```
-```response
-{"results":[true]}
-```
-
-#### SetRowAttrs
-**Spec:**
-
-```
-SetRowAttrs(<FIELD>, <ROW>,
-            <ATTR_NAME=ATTR_VALUE>,
-            [ATTR_NAME=ATTR_VALUE ...])
-```
-
-**Description:**
-
-`SetRowAttrs` associates arbitrary key/value pairs with a row in a field. Setting a value of `null`, without quotes, deletes an attribute.
-
-**Result Type:** null
-
-SetRowAttrs queries always return `null` upon success.
-
-**Examples:**
-
-Set attributes `username` and `active` on row 10:
-```request
-SetRowAttrs(stargazer, 10, username="mrpi", active=true)
-```
-```response
-{"results":[null]}
-```
-
-Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Row](../query-language/#row) query like so `Row(stargazer=10)`.
-
-Delete attribute `username` on row 10:
-```request
-SetRowAttrs(stargazer, 10, username=null)
-```
-```response
-{"results":[null]}
-```
-
-#### SetColumnAttrs
-
-**Spec:**
-
-```
-SetColumnAttrs(<COLUMN>,
-               <ATTR_NAME=ATTR_VALUE>,
-               [ATTR_NAME=ATTR_VALUE ...])
-```
-
-**Description:**
-
-`SetColumnAttrs` associates arbitrary key/value pairs with a column in an index.
-
-**Result Type:** null
-
-SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute.
-
-**Examples:**
-
-Set attributes `stars`, `url`, and `active` on column 10:
-```request
-SetColumnAttrs(10, stars=123, url="http://projects.pilosa.com/10", active=true)
-```
-```response
-{"results":[null]}
-```
-
-Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa.
-
-ColumnAttrs can be requested by adding the URL parameter `columnAttrs=true` to a query. For example:
-```request
-curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Row(stargazer=1) Row(stargazer=2)'
-```
-```response
-{
-  "results":[
-    {"attrs":{},"cols":[10,20]},
-    {"attrs":{},"cols":[10,30]}
-  ],
-  "columnAttrs":[
-    {"id":10,"attrs":{"active":true,"stars":123,"url":"http://projects.pilosa.com/10"}},
-    {"id":20,"attrs":{"active":false,"stars":456,"url":"http://projects.pilosa.com/30"}}
-  ]
-}
-```
-
-In this example, ColumnAttrs have been set on columns 10 and 20, but not column 30. The relevant attributes are all returned in a single columnAttrs list. See the [query index](../api-reference/#query-index) section for more information.
-
-Delete the `url` attribute on column 10:
-```request
-SetColumnAttrs(10, url=null)
-```
-```response
-{"results":[null]}
-```
-
-#### Clear
-
-**Spec:**
-
-```
-Clear(<COLUMN>, <FIELD>=<ROW>)
-```
-
-**Description:**
-
-`Clear` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column.
-
-Note that clearing a column on a time field will remove all data for that column.
-
-**Result Type:** boolean
-
-A return value of `true` indicates that the bit was toggled from 1 to 0.
-
-A return value of `false` indicates that the bit was already set to 0 and nothing changed.
-
-**Examples:**
-
-Clear the bit at row 1 and column 10 in the stargazer field:
-```request
-Clear(10, stargazer=1)
-```
-```response
-{"results":[true]}
-```
-
-This represents removing the relationship between the user with id=1 and the repository with id=10.
-
-#### ClearRow
-
-**Spec:**
-
-```
-ClearRow(<FIELD>=<ROW>)
-```
-
-**Description:**
-
-`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns.
-
-**Result Type:** boolean
-
-A return value of `true` indicates that at least one column was toggled from 1 to 0.
-
-A return value of `false` indicates that all bits in the row were already 0 and nothing changed.
-
-**Examples:**
-
-Clear all bit in row 1 in the stargazer field:
-```request
-ClearRow(stargazer=1)
-```
-```response
-{"results":[true]}
-```
-
-This represents removing the relationship between the user with id=1 and all repositories.
-
-#### Store
-
-**Spec:**
-
-```
-Store(<ROW_CALL>, <FIELD>=<ROW>)
-```
-
-**Description:**
-
-`Store` writes the results of `<ROW_CALL>` to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
-
-**Result Type:** boolean
-
-Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call.
-
-**Examples:**
-
-Store the contents of stargazer row 1 into stargazer row 2:
-```request
-Store(Row(stargazer=1), stargazer=2)
-```
-```response
-{"results":[true]}
-```
-
-Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20.
-```request
-Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20)
-```
-```response
-{"results":[true]}
-```
-
-### Read Operations
-
-#### Row
-
-**Spec:**
-
-```
-Row(<FIELD>=<ROW>)
-```
-
-**Description:**
-
-`Row` retrieves the indices of all the columns in a row. It also retrieves any attributes set on that row.
-
-**Result Type:** object with attrs and columns.
-
-e.g. `{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}`
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1):
-```request
-Row(stargazer=1)
-```
-```response
-{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}
-```
-
-* attrs are the attributes for user 1
-* columns are the repositories which user 1 has starred.
-
-
-#### Row (Range)
-
-**Spec:**
-
-```
-Row(<FIELD>=<ROW>, from=<TIMESTAMP>, to=<TIMESTAMP>)
-```
-
-**Description:**
-
-Similar to `Row`, but only returns bits which were set with timestamps between the given `from` (inclusive) and `to` (exclusive) timestamps. Both `from` and `to` parameters are optional. The default for `to` timestamp is current time + 1 day. If a later end timestamp is required, specify it explicitly.
-
-**Result Type:** object with attrs and bits
-
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
-```request
-Row(stargazer=1, from='2010-01-01T00:00', to='2017-03-02T03:00')
-```
-```response
-{{"attrs":{},"columns":[10]}
-```
-
-This example assumes timestamps have been set on some bits.
-
-* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
-
-
-#### Row (BSI)
-
-**Spec:**
-
-```
-Row([<COMPARISON_VALUE> <COMPARISON_OPERATOR>] <FIELD> <COMPARISON_OPERATOR> <COMPARISON_VALUE>)
-```
-
-**Description:**
-
-The `Row` query is overloaded to work on `integer` values as well as `timestamp` values.
-Returns bits that are true for the comparison operator.
-
-**Result Type:** object with attrs and columns
-
-**Examples:**
-
-In our source data, commitactivity was counted over the last year.
-The following greater-than `Row` query returns all columns with a field value greater than 100 (repositories having more than 100 commits):
-
-```request
-Row(commitactivity > 100)
-```
-```response
-{{"attrs":{},"columns":[10]}
-```
-
-* columns are repositories which had at least 100 commits in the last year.
-
-BSI range queries support the following operators:
-
- Operator | Name                          | Value
-----------|-------------------------------|--------------------
- `>`      | greater-than, GT              | integer
- `<`      | less-than, LT                 | integer
- `<=`     | less-than-or-equal-to, LTE    | integer
- `>=`     | greater-than-or-equal-to, GTE | integer
- `==`     | equal-to, EQ                  | integer
- `!=`     | not-equal-to, NEQ             | integer or `null`
-
-A bounded interval can be specified by chaining the `<` and `<=` operators (but not others). For example:
-
-```request
-Row(50 < commitactivity < 150)
-```
-```response
-{{"attrs":{},"columns":[10]}
-```
-
-As of Pilosa 1.0, the "between" syntax `Row(frame=stats, commitactivity >< [50, 150])` is no longer supported.
-
-#### Union
-
-**Spec:**
-
-```
-Union([ROW_CALL ...])
-```
-
-**Description:**
-
-Union performs a logical OR on the results of all `ROW_CALL` queries passed to it.
-
-**Result Type:** object with attrs and bits
-
-attrs will always be empty
-
-**Examples:**
-
-Query columns with a bit set in either of two rows (repositories that are starred by either of two users):
-```request
-Union(Row(stargazer=1), Row(stargazer=2))
-```
-```response
-{"attrs":{},"columns":[10, 20, 30]}
-```
-
-* columns are repositories that were starred by user 1 OR user 2
-
-#### Intersect
-
-**Spec:**
-
-```
-Intersect(<ROW_CALL>, [ROW_CALL ...])
-```
-
-**Description:**
-
-Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query columns with a bit set in both of two rows (repositories that are starred by both of two users):
-
-```request
-Intersect(Row(stargazer=1), Row(stargazer=2))
-```
-```response
-{"attrs":{},"columns":[10]}
-```
-
-* columns are repositories that were starred by user 1 AND user 2
-
-#### Difference
-
-**Spec:**
-
-```
-Difference(<ROW_CALL>, [ROW_CALL ...])
-```
-
-**Description:**
-
-Difference returns all of the bits from the first `ROW_CALL` argument passed to it, without the bits from each subsequent `ROW_CALL`.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query columns with a bit set in one row and not another (repositories that are starred by one user and not another):
-```request
-Difference(Row(stargazer=1), Row(stargazer=2))
-```
-```response
-{"results":[{"attrs":{},"columns":[20]}]}
-```
-
-* columns are repositories that were starred by user 1 BUT NOT user 2
-
-Query for the opposite difference:
-```request
-Difference(Row(stargazer=2), Row(stargazer=1))
-```
-```response
-{"attrs":{},"columns":[30]}
-```
-
-* columns are repositories that were starred by user 2 BUT NOT user 1
-
-#### Xor
-
-**Spec:**
-
-```
-Xor(<ROW_CALL>, [ROW_CALL ...])
-```
-
-**Description:**
-
-Xor performs a logical XOR on the results of each `ROW_CALL` query passed to it.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query columns with a bit set in exactly one of two rows (repositories that are starred by only one of two users):
-
-```request
-Xor(Row(stargazer=2), Row(stargazer=1))
-```
-```response
-{"results":[{"attrs":{},"columns":[20,30]}]}
-```
-
-* columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
-
-#### Not
-
-**Spec:**
-
-```
-Not(<ROW_CALL>)
-```
-
-**Description:**
-
-Not returns the inverse of all of the bits from the `ROW_CALL` argument. The Not query requires that `trackExistence` has been enabled on the Index.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query existing columns that do not have a bit set in the given row.
-```request
-Not(Row(stargazer=1))
-```
-```response
-{"results":[{"attrs":{},"columns":[30]}]}
-```
-
-* columns are repositories that were not starred by user 1
-
-#### Count
-**Spec:**
-
-```
-Count(<ROW_CALL>)
-```
-
-**Description:**
-
-Returns the number of set bits in the `ROW_CALL` passed in.
-
-**Result Type:** int
-
-**Examples:**
-
-Query the number of bits set in a row (the number of repositories a user has starred):
-```request
-Count(Row(stargazer=1))
-```
-```response
-{"results":[1]}
-```
-
-* Result is the number of repositories that user 1 has starred.
-
-#### Shift
-**Spec:**
-
-```
-Shift(<ROW_CALL>, [n=UINT])
-```
-
-**Description:**
-
-Returns the row specified by `ROW_CALL` shifted by `n` bits.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of the field `stargazer`
-and shift the result by 2:
-```request
-Shift(Row(stargazer=1), n=2)
-```
-```response
-{"attrs":{},"columns":[12, 22]}
-```
-
-* columns are the repositories which user 1 has starred shifted by 2 bits.
-
-#### TopN
-
-**Spec:**
-
-```
-TopN(<FIELD>, [ROW_CALL], [n=UINT],
-     [attrName=<ATTR_NAME>, attrValues=<[]ATTR_VALUE>])
-```
-
-**Description:**
-
-Return the id and count of the top `n` rows (by count of bits) in the field.
-The `attrName` and `attrValues` arguments work together to only return rows which
-have the attribute specified by `attrName` with one of the values specified in
-`attrValues`.
-
-**Result Type:** array of key/count objects
-
-**Caveats:**
-
-* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order.
-* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
-* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
-* Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order.
-* The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
-
-See [field creation](../api-reference/#create-field) for more information about the cache.
-
-**Examples:**
-
-Basic TopN query:
-```request
-TopN(stargazer)
-```
-```response
-{"results":[[{"id":1240,"count":102},{"id":4734,"count":100},{"id":12709,"count":93},...]]}
-```
-
-* `id` is a row ID (user ID)
-* `count` is a count of columns (repositories)
-* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer field. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository.
-
-Limit the number of results:
-```request
-TopN(stargazer, n=2)
-```
-```response
-{"results":[[{"id":1240,"count":102},{"id":4734,"count":100}]]}
-```
-
-* Results are the top two rows (users) sorted by number of bits set (repositories they've starred) in descending order.
-
-Filter based on an existing row:
-```request
-TopN(stargazer, Row(language=1), n=2)
-```
-```response
-{"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]}
-```
-
-* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language field (repositories that they've starred which are written in language 1).
-
-Filter based on attributes:
-```request
-TopN(stargazer, n=2, attrName=active, attrValues=[true])
-```
-```response
-{"results":[[{"id":10,"count":1},{"id":13,"count":1}]]}
-```
-
-* Results are the top two users (rows) which have the "active" attribute set to "true", sorted by the number of bits set (repositories that they've starred).
-
-
-#### Min
-
-**Spec:**
-
-```
-Min([ROW_CALL], field=<FIELD>)
-```
-
-**Description:**
-
-Returns the minimum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
-
-**Result Type:** object with the min and count of columns containing the min value.
-
-**Examples:**
-
-Query the minimum value of a field (minimum size of all repositories):
-```request
-Min(field="diskusage")
-```
-```response
-{"value":4,"count":2}
-```
-
-* Result is the smallest value (repository size in kilobytes, here), plus the count of columns with that value.
-
-#### Max
-
-**Spec:**
-
-```
-Max([ROW_CALL], field=<FIELD>)
-```
-
-**Description:**
-
-Returns the maximum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
-
-**Result Type:** object with the max and count of columns containing the max value.
-
-**Examples:**
-
-Query the maximum value of a field (maximum size of all repositories):
-```request
-Max(field="diskusage")
-```
-```response
-{"value":88,"count":13}
-```
-
-* Result is the largest value (repository size in kilobytes, here), plus the count of columns with that value.
-
-#### Sum
-
-**Spec:**
-
-```
-Sum([ROW_CALL], field=<FIELD>)
-```
-
-**Description:**
-
-Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
-
-**Result Type:** object with the computed sum and count of the values in the integer field.
-
-**Examples:**
-
-Query the size of all repositories.
-```request
-Sum(field="diskusage")
-```
-```response
-{"value":10,"count":3}
-```
-
-* Result is the sum of all values (total size of all repositories in kilobytes, here), plus the count of columns.
-
-### Other Operations
-
-#### Options
-
-**Spec:**
-
-```
-Options(<CALL>, columnAttrs=<BOOL>, excludeColumns=<BOOL>, excludeRowAttrs=<BOOL>, shards=[UINT ...])
-```
-
-**Description:**
-
-Modifies the given query as follows:
-
-* `columnAttrs`: Include column attributes in the result (Default: `false`).
-* `excludeColumns`: Exclude column IDs from the result (Default: `false`).
-* `excludeRowAttrs`: Exclude row attributes from the result (Default: `false`).
-* `shards`: Run the query using only the data from the given shards. By default, the entire data set (i.e. data from all shards) is used.
-
-**Result Type:** Same result type as `<CALL>`.
-
-**Examples:**
-
-Return column attributes:
-```request
-Options(Row(f1=10), columnAttrs=true)
-```
-```response
-{"attrs":{},"columns":[100]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}
-```
-
-Run the query against shards 0 and 2 only:
-```request
-Options(Row(f1=10), shards=[0, 2])
-```
-```response
-{"attrs":{},"columns":[100, 2097152]}
-```
-
-#### Rows
-
-**Spec:**
-
-```
-Rows(<FIELD>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>, from=<TIMESTAMP>, to=<TIMESTAMP>)
-```
-
-**Description:**
-
-Rows returns a list of row IDs in the given field which have at least one bit
-set. The field argument is mandatory, the others are optional.
-
-If `previous` is given, rows prior to and including the specified row ID or
-key will not be returned. If `column` is given, only rows which have a set bit
-in the given column will be returned. `previous` or `column` must be strings if
-and only if the field or index respectively is using key translation. If `limit`
-is given, the number of rowIDs returned will be less than or equal to
-`limit`. The combination of `limit` and `previous` allows for paging over large
-result sets. Results are always ordered, so setting `previous` as the last
-result of the previous request will start from the next available row.
-
-If the field is of type `time`, the `from` and `to` arguments can be provided
-to restrict the result to a specific time span. If `from` and `to` are
-not provided, the full range of existing data will be queried.
-
-**Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
-
-**Examples:**
-
-Without keys:
-```request
-Rows(age)
-```
-```response
-{"rows":[18,22,29]}
-```
-
-With keys:
-```request
-Rows(job)
-```
-```response
-{"rows":null,"keys":["engineer","management","student""]}
-```
-
-#### Group By
-
-**Spec:**
-
-```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
-```
-
-**Description:**
-
-GroupBy returns the count of the intersection of every combination of rows
-taking one row each from the specified `Rows` calls. It returns only those
-combinations for which the count is greater than 0.
-
-The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
- Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
- query.
-
-The optional `limit` argument limits the number of results returned. The results
-are ordered, so as long as the data isn't changing, the same query will return
-the same result set.
-
-Paging through results is supported by passing the `previous` argument to each
-of the `Rows` calls in the GroupBy. Take the last result from your previous
-`GroupBy` query, and pass each row ID in that result as the `previous` argument
-to each of the respective `Rows` queries in your next `GroupBy` query.
-
-**Result Type:** Array of "groups". Each group is an object with a group key and
-a count key. The count is an integer, and the group is an array of objects which
-specify the field and row for each row that was intersected to get that result.
-
-**Examples:**
-
-A single `Rows` query.
-```request
-GroupBy(Rows(age))
-```
-```response
-[{"group":[{"field":"age","rowID":18}],"count":14},
-{"group":[{"field":"age","rowID":22}],"count":22},
-{"group":[{"field":"age","rowID":29}],"count":6}]
-```
-
-With two `Rows` queries - one with IDs and one with keys.
-```request
-GroupBy(Rows(age), Rows(job), limit=7)
-```
-```response
-[{"group":[{"field":"age","rowID":18},{"field":"job","rowKey":"engineer"}],"count":3},
- {"group":[{"field":"age","rowID":18},{"field":"job","rowKey":"management"}],"count":1},
- {"group":[{"field":"age","rowID":18},{"field":"job","rowKey":"student"}],"count":11},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"engineer"}],"count":6},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"management"}],"count":2},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"student"}],"count":4},
- {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"engineer"}],"count":9}]
-```
-
-Getting the rest of the results from the previous example (paging).
-```request
-GroupBy(Rows(age, previous=29), Rows(job, previous="management"), limit=7)
-```
-
-```response
- {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"engineer"}],"count":9}]
-[{"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"management"}],"count":3},
- {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"student"}],"count":1}]
-```
-
-Using the filter argument.
-```request
-GroupBy(Rows(age), Rows(job), limit=7, filter=Row(country=USA))
-```
-
-```response
-[{"group":[{"field":"age","rowID":18},{"field":"job","rowKey":"engineer"}],"count":1},
- {"group":[{"field":"age","rowID":18},{"field":"job","rowKey":"student"}],"count":6},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"engineer"}],"count":3},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"management"}],"count":1},
- {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"student"}],"count":3},
- {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"management"}],"count":7}]
-```

docs/tutorials.md

@@ -1,779 +0,0 @@
-+++
-title = "Tutorials"
-weight = 4
-nav = [
-    "Setting Up a Secure Cluster",
-    "Setting Up a Docker Cluster",
-    "Using Integer Field Values",
-    "Storing Row and Column Attributes",
-]
-+++
-
-## Tutorials
-
-<div class="note">
-<!-- this is html because there is a problem putting a list inside a shortcode -->
-Some of our tutorials work better as standalone repos, since you can <code>git clone</code> the instructions, code, and data all at once. Officially supported tutorials are listed here.<br />
-<br />
-<ul>
-<li><a href="https://github.com/pilosa/cosmosa">Run Pilosa with Microsoft's Azure Cosmos DB</a></li>
-</ul>
-
-</div>
-
-### Setting Up a Secure Cluster
-
-#### Introduction
-
-Pilosa supports encrypting all communication with nodes in a cluster using TLS, including [Mutual TLS Authentication](https://en.wikipedia.org/wiki/Mutual_authentication). In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is beyond the scope of this tutorial.
-
-This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems.
-
-#### Installing Pilosa and Creating the Directory Structure
-
-If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
-
-After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
-``` request
-pilosa --help
-```
-``` response
-Pilosa is a fast index to turbocharge your database.
-
-This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
-backing up, and more. Complete documentation is available
-at https://www.pilosa.com/docs/.
-
-Pilosa v1.4.0
-Build Time: 2019-09-23T14:33:07+0000
-
-Usage:
-  pilosa [command]
-
-Available Commands:
-  check           Do a consistency check on a pilosa data file.
-  config          Print the current configuration.
-  export          Export data from pilosa.
-  generate-config Print the default configuration.
-  help            Help about any command
-  holder          Load Pilosa.
-  import          Bulk load data into pilosa.
-  inspect         Get stats on a pilosa data file.
-  server          Run Pilosa.
-
-Flags:
-  -c, --config string   Configuration file to read from.
-  -h, --help            help for pilosa
-
-Use "pilosa [command] --help" for more information about a command.
-```
-
-First, create a directory in which to put all of the files for this tutorial. Then switch to that directory:
-```
-mkdir $HOME/pilosa-tls-tutorial && cd $_
-```
-
-#### Creating the TLS Certificate and Gossip Key
-
-Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption.
-
-The first step is acquiring the necessary TLS certificates. Operating your own public key infrastructure (PKI) is outside of the scope of this tutorial, but it is easy to get started with [certstrap](https://github.com/square/certstrap) for testing/development purposes. For production, you can use OpenSSL or any other software that provides PKI using X.509 certificates, including [Hashicorp Vault](https://learn.hashicorp.com/vault/secrets-management/sm-pki-engine). It is not recommended to use certstrap in production.
-
-First, create a certificate authority (CA):
-
-```
-$ certstrap init --common-name ca
-Created out/ca.key
-Created out/ca.crt
-Created out/ca.crl
-```
-
-The command above creates three files in the `out/` directory:
-
-* `ca.key` is the CA private key file which must be kept as secret.
-* `ca.crt` is the CA TLS certificate.
-* `ca.crl` is the Certificate Revocation List (CRL).
-
-Next, create and sign a wildcard certificate for pilosa:
-
-```
-$ certstrap request-cert --cn "*.pilosa.local"
-Created out/*.pilosa.local.key
-Created out/*.pilosa.local.csr
-
-$ certstrap sign "*.pilosa.local" --CA ca
-Created out/*.pilosa.local.crt from out/*.pilosa.local.csr signed by out/ca.key
-```
-
-The commands above create three files in the `out/` directory:
-
-* `*.pilosa.local.key` is the private key file which must be kept as secret.
-* `*.pilosa.local.csr` is the certificate signing request (CSR).
-* `*.pilosa.local.crt` is the signed TLS certificate.
-
-You can also create a separate client certificate signed by the same CA to test mutual TLS using curl:
-
-```
-$ certstrap request-cert --cn "curl"
-Created out/curl.key
-Created out/curl.csr
-
-$ certstrap sign "curl" --CA ca
-Created out/curl.crt from out/curl.csr signed by out/ca.key
-```
-
-Having created the TLS certificates, we can now create the gossip encryption key. The gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
-```
-head -c 32 /dev/random > pilosa.local.gossip32
-```
-
-We now have a file called `pilosa.local.gossip32` in the current directory which contains 32 random bytes.
-
-#### Creating the Configuration Files
-
-Pilosa supports passing configuration items using command line options, environment variables, or a configuration file. For this tutorial, we will use three configuration files; one configuration file for each of our three nodes.
-
-One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated.
-
-Create `node1.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node1.config.toml
-
-data-dir = "node1_data"
-bind = "https://01.pilosa.local:10501"
-
-[cluster]
-coordinator = true
-
-[tls]
-ca-certificate = "out/ca.crt"
-certificate = "out/*.pilosa.local.crt"
-key = "out/*.pilosa.local.key"
-enable-client-verification = true
-
-[gossip]
-seeds = ["01.pilosa.local:15000"]
-port = 15000
-key = "pilosa.local.gossip32"
-```
-
-Create `node2.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node2.config.toml
-
-data-dir = "node2_data"
-bind = "https://02.pilosa.local:10502"
-
-[tls]
-ca-certificate = "out/ca.crt"
-certificate = "out/*.pilosa.local.crt"
-key = "out/*.pilosa.local.key"
-enable-client-verification = true
-
-[gossip]
-seeds = ["01.pilosa.local:15000"]
-port = 16000
-key = "pilosa.local.gossip32"
-```
-
-Create `node3.config.toml` in the project directory and paste the following in it:
-
-```toml
-# node3.config.toml
-
-data-dir = "node3_data"
-bind = "https://03.pilosa.local:10503"
-
-[tls]
-ca-certificate = "out/ca.crt"
-certificate = "out/*.pilosa.local.crt"
-key = "out/*.pilosa.local.key"
-enable-client-verification = true
-
-[gossip]
-seeds = ["01.pilosa.local:15000"]
-port = 17000
-key = "pilosa.local.gossip32"
-```
-
-Here is some explanation of the configuration items:
-
-* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
-* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
-* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings.
-* `[tls]` section contains the TLS settings, including the path to the TLS certificate and the corresponding key. The `ca-certificate` setting is optional and will default to your system CAs. You may also disable server-to-server verification by setting `skip-verify` to `true`, which we don't recommend for production.
-* `[gossip]` section contains settings for the gossip protocol. `seeds` contains the list of nodes from which to seed cluster membership. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. If all nodes of the cluster are running on the same computer, the gossip listen address should be different for each node. Otherwise, it can be set to the same value. Finally, the `key` points to the gossip encryption key we created earlier.
-
-#### Final Touches Before Running the Cluster
-
-Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`):
-```
-sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts'
-```
-
-Ensure we can access the hosts in the cluster:
-```
-ping -c 1 01.pilosa.local
-ping -c 1 02.pilosa.local
-ping -c 1 03.pilosa.local
-```
-
-If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname.
-
-#### Running the Cluster
-
-Let's open three terminal windows and run each node in its own window. This will enable us to better observe what's happening on each node.
-
-Switch to the first terminal window, change to the project directory and start the first node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node1.config.toml
-```
-
-Switch to the second terminal window, change to the project directory and start the second node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node2.config.toml
-```
-
-Switch to the third terminal window, change to the project directory and start the third node:
-```
-cd $HOME/pilosa-tls-tutorial
-pilosa server -c node3.config.toml
-```
-
-Let's ensure that all three Pilosa servers are running and they are connected:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://01.pilosa.local:10501/status
-```
-``` response
-{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]}
-```
-
-The `-k` flag is used to tell curl that it shouldn't bother checking the certificate the server provides, and the `--ipv4` flag avoids an issue on MacOS where the curl request takes a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
-
-If everything is set up correctly, the cluster state should be `NORMAL`.
-
-#### Running Queries
-
-Having confirmed that our cluster is running normally, let's perform a few queries. First, we need to create an index and a field:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://01.pilosa.local:10501/index/sample-index \
-     -X POST
-```
-``` response
-{"success":true}
-```
-
-This will create index `sample-index` with default options. Let's create the field now:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://01.pilosa.local:10501/index/sample-index/field/sample-field \
-     -X POST
-```
-``` response
-{"success":true}
-```
-
-We just created field `sample-field` with default options.
-
-Let's run a `Set` query:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://01.pilosa.local:10501/index/sample-index/query \
-     -X POST \
-     -d 'Set(100, sample-field=1)'
-```
-``` response
-{"results":[true]}
-```
-
-Confirm that the value was indeed set:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://01.pilosa.local:10501/index/sample-index/query \
-     -X POST \
-     -d 'Row(sample-field=1)'
-```
-``` response
-{"results":[{"attrs":{},"columns":[100]}]}
-```
-
-The same response should be returned when querying other nodes in the cluster:
-``` request
-curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
-     https://02.pilosa.local:10502/index/sample-index/query \
-     -X POST \
-     -d 'Row(sample-field=1)'
-```
-``` response
-{"results":[{"attrs":{},"columns":[100]}]}
-```
-
-#### What's Next?
-
-Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
-
-### Setting Up a Docker Cluster
-
-In this tutorial, we will be setting up a 2-node Pilosa cluster using Docker containers.
-
-#### Running a Docker Cluster on a Single Server
-
-The instructions below require Docker 1.13 or better.
-
-Let's first be sure that the Pilosa image is up to date:
-```
-docker pull pilosa/pilosa:latest
-```
-
-Then, create a virtual network to attach our containers. We are going to name our network `pilosanet`:
-
-```
-docker network create pilosanet
-```
-
-Let's run the first Pilosa node and attach it to that virtual network. We set the first node as the cluster coordinator and use its address as the gossip seed. And also set the server address to `pilosa1`:
-```
-docker run -it --rm --name pilosa1 -p 10101:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
-```
-
-Let's run the second Pilosa node and attach it to the virtual network as well. Note that we set the address of the gossip seed to the address of the first node:
-```
-docker run -it --rm --name pilosa2 -p 10102:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
-```
-
-Let's test that the nodes in the cluster connected with each other:
-``` request
-curl localhost:10101/status
-```
-``` response
-{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
-```
-
-And similarly for the second node:
-``` request
-curl localhost:10102/status
-```
-``` response
-{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
-```
-The corresponding [Docker Compose](https://docs.docker.com/compose/) file is below:
-
-```yaml
-version: '2'
-services: 
-  pilosa1:
-    image: pilosa/pilosa:latest
-    ports:
-      - "10101:10101"
-    environment:
-      - PILOSA_CLUSTER_COORDINATOR=true
-      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
-    networks:
-      - pilosanet
-    entrypoint:
-      - /pilosa
-      - server
-      - --bind
-      - "pilosa1:10101"
-  pilosa2:
-    image: pilosa/pilosa:latest
-    ports:
-      - "10102:10101"
-    environment:
-      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
-    networks:
-      - pilosanet
-    entrypoint:
-      - /pilosa
-      - server
-      - --bind
-      - "pilosa2:10101"
-networks: 
-  pilosanet:
-```
-
-#### Running a Docker Swarm
-
-It is very easy to run a Pilosa Cluster on different servers using [Docker Swarm mode](https://docs.docker.com/engine/swarm/). All we have to do is create an overlay network instead of a bridge network.
-
-The instructions in this section require Docker 17.06 or newer. Although it is possible to run a Docker swarm on MacOS or Windows, it is easiest to run it on Linux. The following instructions assume you are running on Linux.
-
-We are going to use two servers: the manager node runs in the first server and a worker node in the second server.
-
-Docker nodes require some ports to be accesible from the outside. Before proceeding, make sure the following ports are open on all nodes: TCP/2377, TCP/7946, UDP/7946, UDP/4789.
-
-Let's initialize the swarm first. Run the following on the manager:
-```
-docker swarm init --advertise-addr=IP-ADDRESS
-```
-
-Virtual machines running on the cloud usually have at least two network interfaces: the external interface and the internal interface. Use the IP of the external interface.
-
-The output of the command above should be similar to:
-```
-To add a manager to this swarm, run the following command:
-
-    docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
-```
-
-Let's make the worker node join the manager. Copy/paste the command above in a shell on the worker, replacing the token and IP address with the correct values. You may neeed to add `--advertise-addr=WORKER-EXTERNAL-IP-ADDRESS` parameter if the worker has more than one network interface:
-```
-docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
-```
-
-Run the following on the manager to check that the worker joined to the swarm:
-```
-docker node ls
-```
-
-Which should output:
-
-ID|HOSTNAME|STATUS|AVAILABILITY|MANAGER STATUS|ENGINE VERSION
----|--------|------|------------|--------------|-------------
-MANAGER-ID *|swarm1|Ready|Active|Leader|18.05.0-ce|
-WORKER-ID|swarm2|Ready|Active||18.05.0-ce|
-
-If you have created the `pilosanet` network before, delete it before carrying on, otherwise skip to the next step:
-```
-docker network rm pilosanet
-```
-
-Let's create the `pilosanet` network, but with `overlay` type this time. We should also make this network attachable in order to be able to attach containers to it. Run the following on the manager:
-```
-docker network create -d overlay pilosanet --attachable
-```
-
-We can now create the Pilosa containers. Let's start the coordinator node first. Run the following on one of the servers:
-```
-docker run -it --rm --name pilosa1 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
-```
-
-And the following on the other server:
-```
-docker run -it --rm --name pilosa2 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
-```
-
-These were the same commands we used in the previous section except the port mapping! Let's run another container on the same virtual network to read the status from the coordinator:
-``` request
-docker run -it --rm --network=pilosanet --name shell alpine wget -q -O- pilosa1:10101/status
-```
-``` response
-{"state":"NORMAL","nodes":[{"id":"3e3b0abd-1945-441a-a01f-5a28272972f5","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"71ed27cc-9443-4f41-88fb-1c22f92bf695","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"3e3b0abd-1945-441a-a01f-5a28272972f5"}
-```
-
-You can add additional worker nodes to both the swarm and the Pilosa cluster using the steps above.
-
-#### What's Next?
-
-Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
-
-Refer to the [Docker documentation](https://docs.docker.com) to see your options about running Docker containers. The [Networking with overlay networks](https://docs.docker.com/network/network-tutorial-overlay/) is a detailed overview of the Docket swarm mode and overlay networks.
-
-
-### Using Integer Field Values
-
-#### Introduction
-
-Pilosa can store integer values associated to the columns in an index, and those values are used to support `Row`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
-
-First, create an index called `patients`:
-``` request
-curl localhost:10101/index/patients \
-     -X POST 
-```
-``` response
-{"success":true}
-```
-
-In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `patients` index.
-``` request
-curl localhost:10101/index/patients/field/age \
-     -X POST \
-     -d '{"options":{"type": "int", "min": 0, "max": 120}}'
-```
-``` response
-{"success":true}
-```
-
-``` request
-curl localhost:10101/index/patients/field/weight \
-     -X POST \
-     -d '{"options":{"type": "int", "min": 0, "max": 500}}'
-```
-``` response
-{"success":true}
-```
-
-``` request
-curl localhost:10101/index/patients/field/tcells \
-     -X POST \
-     -d '{"options":{"type": "int", "min": 0, "max": 2000}}'
-```
-``` response
-{"success":true}
-```
-
-Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
-
-The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Set(1, age=34)'
-```
-``` response
-{"results":[true]}
-```
-
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Set(1, weight=128)'
-```
-``` response
-{"results":[true]}
-```
-
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Set(1, tcells=1145)'
-```
-``` response
-{"results":[true]}
-```
-
-In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
-
-Assuming we have a file called `ages.csv` that is structured like this:
-```
-1,34
-2,57
-3,19
-4,40
-5,32
-6,71
-7,28
-8,33
-9,63
-```
-where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command:
-```
-pilosa import -i patients --field age ages.csv
-```
-
-Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
-
-In order to find all patients over the age of 40, then simply run a `Row` query against the `age` field.
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Row(age > 40)'
-```
-``` response
-{"results":[{"attrs":{},"columns":[2,6,9]}]}
-```
-
-You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#row-bsi) documentation.
-
-To find the average age of all patients, run a `Sum` query:
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Sum(field="age")'
-```
-``` response
-{"results":[{"value":377,"count":9}]}
-```
-The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`.
-
-You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Sum(Row(age > 40), field="age")'
-```
-``` response
-{"results":[{"value":191,"count":3}]}
-```
-Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
-
-To find the minimum age of all patients, run a `Min` query:
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Min(field="age")'
-```
-``` response
-{"results":[{"value":19,"count":1}]}
-```
-The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value.
-
-You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Min(Row(age > 40), field="age")'
-```
-``` response
-{"results":[{"value":57,"count":1}]}
-```
-
-To find the maximum age of all patients, run a `Max` query:
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Max(field="age")'
-```
-``` response
-{"results":[{"value":71,"count":1}]}
-```
-The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value.
-
-You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
-``` request
-curl localhost:10101/index/patients/query \
-     -X POST \
-     -d 'Max(Row(age < 40), field="age")'
-```
-``` response
-{"results":[{"value":34,"count":1}]}
-```
-
-### Storing Row and Column Attributes
-
-#### Introduction
-
-Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data.
-
-First, create an index called `books` to use for this tutorial:
-``` request
-curl localhost:10101/index/books \
-     -X POST
-```
-``` response
-{"success":true}
-```
-
-Next, create a field in the `books` index called `members` which will represent library members who have read books.
-``` request
-curl localhost:10101/index/books/field/members \
-     -X POST \
-     -d '{}'
-```
-``` response
-{"success":true}
-```
-
-Now, let's add some books to our index.
-``` request
-curl localhost:10101/index/books/query \
-     -X POST \
-     -d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960)
-         SetColumnAttrs(2, name="No Name in the Street", year=1972)
-         SetColumnAttrs(3, name="The Tipping Point", year=2000)
-         SetColumnAttrs(4, name="Out Stealing Horses", year=2003)
-         SetColumnAttrs(5, name="The Forever War", year=2008)'
-```
-``` response
-{"results":[null,null,null,null,null]}
-```
-
-And add some members.
-``` request
-curl localhost:10101/index/books/query \
-     -X POST \
-     -d 'SetRowAttrs(members, 10001, fullName="John Smith")
-         SetRowAttrs(members, 10002, fullName="Sue Perkins")
-         SetRowAttrs(members, 10003, fullName="Jennifer Hawks")
-         SetRowAttrs(members, 10004, fullName="Pedro Vazquez")
-         SetRowAttrs(members, 10005, fullName="Pat Washington")'
-```
-``` response
-{"results":[null,null,null,null,null]}
-```
-
-At this point we can query one of the `member` records by querying that row.
-``` request
-curl localhost:10101/index/books/query \
-     -X POST \
-     -d 'Row(members=10002)'
-```
-``` response
-{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]}
-```
-
-Now let's add some data to the matrix such that each pair represents a member who has read that book.
-``` request
-curl localhost:10101/index/books/query \
-     -X POST \
-     -d 'Set(3, members=10001)
-         Set(5, members=10001)
-         Set(1, members=10002)
-         Set(2, members=10002)
-         Set(4, members=10002)
-         Set(3, members=10003)
-         Set(4, members=10004)
-         Set(5, members=10004)
-         Set(1, members=10005)
-         Set(2, members=10005)
-         Set(3, members=10005)
-         Set(4, members=10005)
-         Set(5, members=10005)'
-```
-``` response
-{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
-```
-
-Now pull the record for `Sue Perkins` again.
-``` request
-curl localhost:10101/index/books/query \
-     -X POST \
-     -d 'Row(members=10002)'
-```
-``` response
-{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]}
-```
-Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read.
-
-In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
-``` request
-curl localhost:10101/index/books/query?columnAttrs=true \
-     -X POST \
-     -d 'Row(members=10002)'
-```
-``` response
-{
-  "results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}],
-  "columnAttrs":[
-    {"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}},
-    {"id":2,"attrs":{"name":"No Name in the Street","year":1972}},
-    {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
-  ]
-}
-```
-The `book` attributes are included in the result set at the `columnAttrs` attribute.
-
-Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members:
-``` request
-curl localhost:10101/index/books/query?columnAttrs=true \
-     -X POST \
-     -d 'Intersect(Row(members=10002), Row(members=10004))'
-```
-``` response
-{
-  "results":[{"attrs":{},"columns":[4]}],
-  "columnAttrs":[
-    {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
-  ]
-}
-```
-
-Notice that we don't get row attributes on a complex query, but we still get the column attributes—in this case book information.

enterprise/COPYING

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

enterprise/enterprise.go

@@ -1,23 +0,0 @@
-// Copyright (c) 2018 Pilosa Corp. All rights reserved.
-//
-// This file is part of Pilosa Enterprise Edition.
-//
-// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package enterprise is now deprecated, and the functionality under
-// enterprise/b has been copied into roaring/. It existed to inject enterprise
-// implementations of various Pilosa features when Pilosa was built with
-// "ENTERPRISE=1 make install". These features were dual-licensed separately
-// from Pilosa community edition under the AGPL and Pilosa's commercial license.
-package enterprise

etcd/cache.go

@@ -0,0 +1,57 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcd
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/topology"
+)
+
+// EtcdWithCache is a wrapper around the Etcd type which will return a
+// cached value when the number of requests come in below a configured
+// frequency. It also breaks the cache after a configured TTL.
+type EtcdWithCache struct {
+	*Etcd
+
+	peersMu sync.Mutex // peer-list cache updates
+
+	nodes            []*topology.Node // unmarshalled Node data
+	nodesTTL         int              // seconds
+	nodesLastRequest time.Time        // last time requested
+}
+
+// NewEtcdWithCache returns a new instance of Cache.
+func NewEtcdWithCache(opt Options, replicas int) *EtcdWithCache {
+	return &EtcdWithCache{
+		Etcd: NewEtcd(opt, replicas),
+
+		nodesTTL: 6,
+	}
+}
+
+// Nodes caches the result of the underlying implementation's node list.
+func (c *EtcdWithCache) Nodes() []*topology.Node {
+	c.peersMu.Lock()
+	defer c.peersMu.Unlock()
+
+	now := time.Now()
+	if now.Sub(c.nodesLastRequest) > (time.Duration(c.nodesTTL) * time.Second) {
+		c.nodes = c.Etcd.Nodes()
+		c.nodesLastRequest = now
+	}
+	return c.nodes
+}

etcd/leasedkv.go

@@ -0,0 +1,192 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcd
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/disco"
+	"github.com/pkg/errors"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3/clientv3util"
+)
+
+// leasedKV is an etcd key and value attached to a lease. It can be used to detect if a node went down.
+// It will try to renew the lease at any cost after losing it.
+// It will recreate the previous existing value for the key again.
+type leasedKV struct {
+	cli    *clientv3.Client
+	cancel context.CancelFunc
+
+	key        string
+	ttlSeconds int64
+
+	mu      sync.Mutex
+	value   string // protected by mu
+	stopped bool   // protected by mu
+}
+
+func newLeasedKV(cli *clientv3.Client, key string, ttlSeconds int64) *leasedKV {
+	return &leasedKV{
+		cli:        cli,
+		key:        key,
+		ttlSeconds: ttlSeconds,
+	}
+}
+
+// Start creates the key and attaches it to a lease.
+// If the lease cannot be renewed in time, it will try to renew it ad finitum.
+func (l *leasedKV) Start(initValue string) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	kaChann, err := l.create(initValue)
+	if err != nil {
+		return err
+	}
+
+	go l.consumeLease(kaChann)
+	return nil
+}
+
+func (l *leasedKV) create(initValue string) (<-chan *clientv3.LeaseKeepAliveResponse, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	l.cancel = cancel
+
+	leaseResp, err := l.cli.Grant(ctx, l.ttlSeconds)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating a lease")
+	}
+
+	if _, err := l.cli.Txn(ctx).
+		Then(clientv3.OpPut(l.key, initValue, clientv3.WithLease(leaseResp.ID))).
+		Commit(); err != nil {
+		return nil, errors.Wrapf(err, "creating key %s with value [%s]", l.key, initValue)
+	}
+
+	kaChann, err := l.cli.KeepAlive(ctx, leaseResp.ID)
+	if err != nil {
+		return nil, errors.Wrapf(err, "keeping alive the lease for the key %s with value %s", l.key, l.value)
+	}
+
+	l.value = initValue
+
+	return kaChann, nil
+}
+
+func (l *leasedKV) consumeLease(ch <-chan *clientv3.LeaseKeepAliveResponse) {
+	for {
+		_, ok := <-ch
+		if !ok {
+			l.mu.Lock()
+
+			if l.stopped {
+				l.mu.Unlock()
+				return
+			}
+
+			if e := retry("consumeLease", 1*time.Second, func() error {
+				kaChann, err := l.create(l.value)
+				if err != nil {
+					return err
+				}
+
+				go l.consumeLease(kaChann)
+				return nil
+			}); e != nil {
+				log.Printf("lease %q cannot be recreated: %v", l.key, e)
+				l.mu.Unlock()
+				return
+			}
+
+			log.Printf("lease %q recreated after a problem", l.key)
+			l.mu.Unlock()
+			return
+		}
+	}
+}
+
+// Stop will cancel the lease renewal.
+// After calling Stop, this object should be discarded and not used anymore.
+func (l *leasedKV) Stop() {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if l.cancel != nil {
+		l.cancel()
+	}
+
+	l.stopped = true
+}
+
+// Set will change the specific value for this key.
+func (l *leasedKV) Set(ctx context.Context, value string) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if _, err := l.cli.Txn(ctx).
+		Then(clientv3.OpPut(l.key, value, clientv3.WithIgnoreLease())).
+		Commit(); err != nil {
+		return errors.Wrapf(err, "creating key %s with value [%s]", l.key, l.value)
+	}
+
+	l.value = value
+
+	return nil
+}
+
+// Get will obtain the actual value for the key.
+func (l *leasedKV) Get(ctx context.Context) (string, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	getResp, err := l.cli.Txn(ctx).
+		If(clientv3util.KeyExists(l.key)).
+		Then(clientv3.OpGet(l.key, clientv3.WithIgnoreLease())).
+		Commit()
+	if err != nil {
+		return "", errors.Wrapf(err, "getting key %s", l.key)
+	}
+
+	if !getResp.Succeeded || len(getResp.Responses) == 0 {
+		return "", disco.ErrNoResults
+	}
+
+	l.value = string(getResp.Responses[0].GetResponseRange().Kvs[0].Value)
+
+	return l.value, nil
+}
+
+func retry(desc string, sleep time.Duration, f func() error) (err error) {
+	for {
+		lastErr := f()
+		if lastErr == nil {
+			return lastErr
+		}
+		if errors.Is(lastErr, context.DeadlineExceeded) {
+			if err != nil {
+				return err
+			} else {
+				return lastErr
+			}
+		}
+		log.Printf("%s: got error %v, retrying", desc, lastErr)
+		err = lastErr
+		time.Sleep(sleep)
+	}
+}

etcd/leasedkv_test.go

@@ -0,0 +1,105 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/disco"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"go.etcd.io/etcd/embed"
+	"go.etcd.io/etcd/etcdserver/api/v3client"
+)
+
+const initVal = "test"
+const newVal = "newValue"
+
+func TestLeasedKv(t *testing.T) {
+	cfg := embed.NewConfig()
+
+	dir, err := testhook.TempDir(t, "leasedkv-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	cfg.Dir = dir
+	etcd, err := embed.StartEtcd(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cli := v3client.New(etcd.Server)
+	defer func() {
+		etcd.Close()
+		<-etcd.Server.StopNotify()
+		cli.Close()
+	}()
+
+	lkv := newLeasedKV(cli, "/test", 1)
+
+	ctx := context.Background()
+
+	err = lkv.Start(initVal)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	v, err := lkv.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v != initVal {
+		t.Fatal("obtained value is not the same as expected. Obtained:", v, "Expected:", initVal)
+	}
+
+	err = lkv.Set(ctx, "otherValue")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = lkv.Set(ctx, "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = lkv.Set(ctx, newVal)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	v, err = lkv.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v != newVal {
+		t.Fatal("obtained value is not the same as expected. Obtained:", v, "Expected:", newVal)
+	}
+
+	time.Sleep(5 * time.Second)
+
+	lkv.Stop()
+
+	// we need to wait to force the lease expiration
+	time.Sleep(5 * time.Second)
+
+	_, err = lkv.Get(ctx)
+	if err == nil || !errors.Is(err, disco.ErrNoResults) {
+		t.Fatal("expected error:", disco.ErrNoResults, "obtained:", err)
+	}
+}

event.go

@@ -14,6 +14,8 @@
 
 package pilosa
 
+import "github.com/pilosa/pilosa/v2/topology"
+
 // NodeEventType are the types of node events.
 type NodeEventType int
 
@@ -27,5 +29,5 @@ const (
 // NodeEvent is a single event related to node activity in the cluster.
 type NodeEvent struct {
 	Event NodeEventType
-	Node  *Node
+	Node  *topology.Node
 }