Merge pull request #598 from travisturner/backport-bug-fixes

Backport bug fixes

.circleci/config.yml

@@ -1,106 +1,113 @@
-version: 2
+version: 2.1
-defaults: &defaults
+
-  working_directory: /go/src/github.com/pilosa/pilosa
+executors:
+  golang:
+    parameters:
+      version:
+        type: string
+        default: "1.14"
+      resource_class:
+        type: string
+        default: medium
     docker:
-    - image: circleci/golang:1.13
+      - image: circleci/golang:<< parameters.version >>
+    resource_class: << parameters.resource_class >>
+    working_directory: /go/src/github.com/pilosa/pilosa
     environment:
-    GO111MODULE: "on"
+      GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12.
-fast-checkout: &fast-checkout
+
-  attach_workspace:
+commands:
-    at: .
+  add-github-auth:
-jobs:
+    steps:
-  setup:
+      - run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com"
-    <<: *defaults
+  restore-mod-cache:
     steps:
-      - checkout
       - restore_cache:
-          keys:
+          key: mod-cache-{{ checksum "go.sum" }}
-            - mod-cache-{{ checksum "go.sum" }}
+  save-mod-cache:
-      - run: "go mod download"
+    steps:
       - save_cache:
           key: mod-cache-{{ checksum "go.sum" }}
           paths:
             - /go/pkg/mod/
-      - persist_to_workspace:
+  checkout-plus:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
     steps:
-      - *fast-checkout
+      - add-github-auth
+      - checkout
+      - restore-mod-cache
+
+jobs:
+  setup:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod download
+      - save-mod-cache
+  check-license-headers:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
       - run: make golangci-lint
   test-build-arm:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
+  test:
-    <<: *defaults
+    parameters:
+      resource_class:
+        type: string
+        default: medium
+      golang_version:
+        type: string
+        default: "1.14"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
+      resource_class: << parameters.resource_class >>
     steps:
-      - *fast-checkout
+      - checkout-plus
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - setup_remote_docker
       - run: make clustertests-build
   prerelease:
-    <<: *base-test
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
       - run: make prerelease
       - store_artifacts:
           path: build
@@ -108,9 +115,12 @@ jobs:
           root: .
           paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - checkout-plus
+      - attach_workspace:
+          at: .
       - run: make release
       - store_artifacts:
           path: build
@@ -121,25 +131,39 @@ jobs:
     docker:
       - image: circleci/python:2.7-jessie
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - checkout-plus
-      - *fast-checkout
+      - attach_workspace:
+          at: .
       - run: sudo pip install awscli
       - run: make prerelease-upload
-  dockerhub-upload:
+  dockerhub-upload-unstable:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - checkout-plus
-      - *fast-checkout
       - setup_remote_docker
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
+  dockerhub-upload-stable:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - setup_remote_docker
+      - run: make docker
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
+      - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
+
 workflows:
-  version: 2
+  build:
-  test:
     jobs:
-      - setup
+      - setup:
+          context: molecula
+          filters:
+            tags:
+              only: /^v.*/
       - linter:
           requires:
             - setup
@@ -149,22 +173,30 @@ workflows:
       - test-build-arm:
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          matrix:
+            parameters:
+              golang_version: ["1.14", "1.13", "1.12", "1.11"]
           requires:
             - setup
-      - test-golang-1.13-race:
+          filters:
+            tags:
+              only: /^v.*/
+      - test:
+          name: test-race
+          test_make_target: test-race
+          resource_class: xlarge
           requires:
             - setup
-      - test-golang-1.13-386:
+      - test:
+          name: test-386
+          goarch: "386"
           requires:
             - setup
-      - test-golang-1.13:
+      - test:
-          requires:
+          name: test-shardwidth-22
-            - setup
+          shard_width: "22"
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
           requires:
             - setup
       - cluster-tests:
@@ -174,22 +206,20 @@ workflows:
           requires:
             - linter
             - check-license-headers
-            - test-golang-1.13
+            - test-golang-1.14
-      - release:
+      - dockerhub-upload-unstable:
+          context: molecula
           requires:
-            - linter
+            - setup
-            - check-license-headers
+          filters:
-            - test-golang-1.13
+            branches:
+              only: master
+      - dockerhub-upload-stable:
+          context: molecula
+          requires:
+            - setup
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,8 +1,11 @@
 FROM golang:1.13.0 as builder
 
+ARG BUILD_FLAGS
+ARG MAKE_FLAGS
+
 COPY . pilosa
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
 
 FROM alpine:3.9.4
 
@@ -18,5 +21,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
+    && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor"
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,23 +1,31 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
+VARIANT = Molecula
+VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)"
 GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
 RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
+NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+
+PLUGINS=distinct
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export PLUGINS
 
 # Run tests and compile Pilosa
 default: test build
@@ -32,7 +40,11 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)
+
+# Run test suite with race flag
+test-race:
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 30m -v
 
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@@ -53,8 +65,7 @@ build:
 # Create a single release build under the build directory
 release-build:
 	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
+	cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
 	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
 	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
 
@@ -67,11 +78,8 @@ endif
 # Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
 release: check-clean
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
 	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
 
 # try (e.g.) internal/clustertests/docker-compose-replication2.yml
@@ -81,14 +89,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
+clustertests-build: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
@@ -115,14 +123,24 @@ generate-stringer:
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+# dunno if protoc-gen-gofast is actually needed here
+generate-proto-grpc: require-protoc require-protoc-gen-gofast
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+
 # `go generate` all needed packages
 generate: generate-protoc generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
-docker:
+docker: vendor
-	docker build -t "pilosa:$(VERSION)" .
+	docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
 	@echo Created docker image: pilosa:$(VERSION)
 
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
 # Compile Pilosa inside Docker container
 docker-build:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
@@ -133,7 +151,10 @@ docker-test:
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +182,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
 
 ######################
 # Build dependencies #

NOTICE

@@ -14,27 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
 Third-party software licenses
 =============================
 
@@ -115,3 +94,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -63,13 +63,12 @@ There are supported libraries for the following languages:
 - [Java](https://www.pilosa.com/docs/client-libraries/#java)
 - [Python](https://www.pilosa.com/docs/client-libraries/#python)
 
-## Licenses
+## License
 
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
+Pilosa is licensed under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
+
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
+A copy of the license is located in [github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE).
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
+More details about licensing are found in [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE).
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
 
 ## Get Support
 

api.go

@@ -23,7 +23,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -43,6 +45,7 @@ type API struct {
 	holder  *Holder
 	cluster *cluster
 	server  *Server
+	tracker *queryTracker
 
 	importWorkersWG      sync.WaitGroup
 	importWorkerPoolSize int
@@ -93,6 +96,8 @@ func NewAPI(opts ...apiOption) (*API, error) {
 		}()
 	}
 
+	api.tracker = newQueryTracker()
+
 	return api, nil
 }
 
@@ -128,6 +133,7 @@ func (api *API) validate(f apiMethod) error {
 func (api *API) Close() error {
 	close(api.importWork)
 	api.importWorkersWG.Wait()
+	api.tracker.Stop()
 	return nil
 }
 
@@ -144,18 +150,26 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 	if err != nil {
 		return QueryResponse{}, errors.Wrap(err, "parsing")
 	}
+	defer api.tracker.Finish(api.tracker.Start(req.Query))
 	execOpts := &execOptions{
 		Remote:          req.Remote,
+		Profile:         req.Profile,
 		ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
 		ExcludeColumns:  req.ExcludeColumns,  // NOTE: Kept for Pilosa 1.x compat.
 		ColumnAttrs:     req.ColumnAttrs,     // NOTE: Kept for Pilosa 1.x compat.
+		EmbeddedData:    req.EmbeddedData,    // precomputed values that needed to be passed with the request
 	}
 	resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
 	if err != nil {
 		return QueryResponse{}, errors.Wrap(err, "executing")
 	}
 
-	return resp, nil
+	// Check for an error embedded in the response.
+	if resp.Err != nil {
+		err = errors.Wrap(resp.Err, "executing")
+	}
+
+	return resp, err
 }
 
 // CreateIndex makes a new Pilosa index.
@@ -172,16 +186,23 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 	if err != nil {
 		return nil, errors.Wrap(err, "creating index")
 	}
+
+	createdAt := timestamp()
+	index.mu.Lock()
+	index.createdAt = createdAt
+	index.mu.Unlock()
+
 	// Send the create index message to all nodes.
 	err = api.server.SendSync(
 		&CreateIndexMessage{
 			Index:     indexName,
+			CreatedAt: createdAt,
 			Meta:      &options,
 		})
 	if err != nil {
 		return nil, errors.Wrap(err, "sending CreateIndex message")
 	}
-	api.holder.Stats.Count("createIndex", 1, 1.0)
+	api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
 	return index, nil
 }
 
@@ -225,7 +246,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 		api.server.logger.Printf("problem sending DeleteIndex message: %s", err)
 		return errors.Wrap(err, "sending DeleteIndex message")
 	}
-	api.holder.Stats.Count("deleteIndex", 1, 1.0)
+	api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
 	return nil
 }
 
@@ -260,19 +281,23 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	if err != nil {
 		return nil, errors.Wrap(err, "creating field")
 	}
+	createdAt := timestamp()
+	field.mu.Lock()
+	field.createdAt = createdAt
+	field.mu.Unlock()
 
 	// Send the create field message to all nodes.
-	err = api.server.SendSync(
+	err = api.server.SendSync(&CreateFieldMessage{
-		&CreateFieldMessage{
 		Index:     indexName,
 		Field:     fieldName,
+		CreatedAt: createdAt,
 		Meta:      &fo,
 	})
 	if err != nil {
 		api.server.logger.Printf("problem sending CreateField message: %s", err)
 		return nil, errors.Wrap(err, "sending CreateField message")
 	}
-	api.holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return field, nil
 }
 
@@ -315,17 +340,46 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() error {
 			for viewName, viewData := range j.req.Views {
+				// The logic here corresponds to the logic in fragment.cleanViewName().
+				// Unfortunately, the logic in that method is not completely exclusive
+				// (i.e. an "other" view named with format YYYYMMDD would be handled
+				// incorrectly). One way to address this would be to change the logic
+				// overall so there weren't conflicts. For now, we just
+				// rely on the field type to inform the intended view name.
 				if viewName == "" {
 					viewName = viewStandard
-				} else {
+				} else if j.field.Type() == FieldTypeTime {
 					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
 				}
+
+				// TODO: deprecate ImportRoaringRequest.Clear, but
+				// until we do, we need to check its value to provide
+				// backward compatibility.
+				doAction := j.req.Action
+				if doAction == "" {
+					if j.req.Clear {
+						doAction = RequestActionClear
+					} else {
+						doAction = RequestActionSet
+					}
+				}
+
+				var doClear bool
+				switch doAction {
+				case RequestActionOverwrite:
+					if err := j.field.importRoaringOverwrite(j.ctx, viewData, j.shard, viewName, j.req.Block); err != nil {
+						return errors.Wrap(err, "importing roaring as overwrite")
+					}
+				case RequestActionClear:
+					doClear = true
+					fallthrough
+				case RequestActionSet:
 					fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
 					if fileMagic == roaring.MagicNumber { // if pilosa roaring format
-					if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, j.req.Clear); err != nil {
+						if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, doClear); err != nil {
 							return errors.Wrap(err, "importing pilosa roaring")
 						}
 					} else {
@@ -333,11 +387,12 @@ func importWorker(importWork chan importJob) {
 						// field.importRoaring changes the standard roaring run format to pilosa roaring
 						data := make([]byte, len(viewData))
 						copy(data, viewData)
-					if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, j.req.Clear); err != nil {
+						if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, doClear); err != nil {
 							return errors.Wrap(err, "importing standard roaring")
 						}
 					}
 				}
+			}
 			return nil
 		}()
 
@@ -374,20 +429,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 		return errors.Wrap(err, "validating api method")
 	}
 
-	nodes := api.cluster.shardNodes(indexName, shard)
+	index, field, err := api.indexField(indexName, fieldName, shard)
-
+	if index == nil || field == nil {
-	field := api.holder.Field(indexName, fieldName)
-	if field == nil {
 		return newNotFoundError(ErrFieldNotFound)
 	}
 
-	// only set and time fields are supported
+	if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
-	if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime {
+		return newPreconditionFailedError(err)
-		return NewBadRequestError(errors.New("roaring import is only supported for set and time fields"))
 	}
 
+	nodes := api.cluster.shardNodes(indexName, shard)
 	errCh := make(chan error, len(nodes))
-
 	for _, node := range nodes {
 		node := node
 		if node.ID == api.server.nodeID {
@@ -459,7 +511,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Printf("problem sending DeleteField message: %s", err)
 		return errors.Wrap(err, "sending DeleteField message")
 	}
-	api.holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -491,7 +543,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
 		api.server.logger.Printf("problem sending DeleteAvailableShard message: %s", err)
 		return errors.Wrap(err, "sending DeleteAvailableShard message")
 	}
-	api.holder.Stats.CountWithCustomTags("deleteAvailableShard", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -540,8 +592,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		var colStr string
 		var err error
 
-		if field.keys() {
+		if field.Keys() {
-			if rowStr, err = field.translateStore.TranslateID(rowID); err != nil {
+			// TODO: handle case: field.ForeignIndex
+			if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil {
 				return errors.Wrap(err, "translating row")
 			}
 		} else {
@@ -549,7 +602,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		}
 
 		if index.Keys() {
-			if colStr, err = index.translateStore.TranslateID(columnID); err != nil {
+			if store := index.TranslateStore(api.cluster.idPartition(indexName, columnID)); store == nil {
+				return errors.Wrap(err, "partition does not exist")
+			} else if colStr, err = store.TranslateID(columnID); err != nil {
 				return errors.Wrap(err, "translating column")
 			}
 		} else {
@@ -660,6 +715,30 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName
 	return f, nil
 }
 
+// TranslateData returns all translation data in the specified partition.
+func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
+	defer span.Finish()
+
+	if err := api.validate(apiTranslateData); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+
+	// Retrieve index from holder.
+	idx := api.holder.Index(indexName)
+	if idx == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Retrieve translatestore from holder.
+	store := idx.TranslateStore(partition)
+	if store == nil {
+		return nil, ErrTranslateStoreNotFound
+	}
+
+	return store, nil
+}
+
 // Hosts returns a list of the hosts in the cluster including their ID,
 // URL, and which is the coordinator.
 func (api *API) Hosts(ctx context.Context) []*Node {
@@ -714,13 +793,32 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 		return errors.Wrap(err, "deserializing cluster message")
 	}
 
-	// Forward the error message.
+	// Forward the message.
 	if err := api.server.receiveMessage(msg); err != nil {
-		return errors.Wrap(err, "receiving message")
+		return MessageProcessingError{err}
 	}
 	return nil
 }
 
+// MessageProcessingError is an error indicating that a cluster message could not be processed.
+type MessageProcessingError struct {
+	Err error
+}
+
+func (err MessageProcessingError) Error() string {
+	return "processing message: " + err.Err.Error()
+}
+
+// Cause allows the error to be unwrapped.
+func (err MessageProcessingError) Cause() error {
+	return err.Err
+}
+
+// Unwrap allows the error to be unwrapped.
+func (err MessageProcessingError) Unwrap() error {
+	return err.Err
+}
+
 // Schema returns information about each index in Pilosa including which fields
 // they contain.
 func (api *API) Schema(ctx context.Context) []*IndexInfo {
@@ -743,6 +841,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
 		return errors.Wrap(err, "validating api method")
 	}
 
+	// set CreatedAt for indexes and fields (if empty), and then apply schema.
+	for _, index := range s.Indexes {
+		if index.CreatedAt == 0 {
+			index.CreatedAt = timestamp()
+		}
+		for _, field := range index.Fields {
+			if field.CreatedAt == 0 {
+				field.CreatedAt = timestamp()
+			}
+		}
+	}
+
 	if !remote {
 		nodes := api.cluster.Nodes()
 		for i, node := range nodes {
@@ -753,7 +863,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
 		}
 	}
 
-	return api.holder.applySchema(s)
+	return errors.Wrap(api.holder.applySchema(s), "applying schema")
 }
 
 // Views returns the views in the given field.
@@ -889,10 +999,17 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
 	return attrs, nil
 }
 
-// ImportOptions holds the options for the API.Import method.
+// ImportOptions holds the options for the API.Import
+// method.
+//
+// TODO(2.0) we have entirely missed the point of functional options
+// by exporting this structure. If it needs to be exported for some
+// reason, we should consider not using functional options here which
+// just adds complexity.
 type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
+	Presorted      bool
 }
 
 // ImportOption is a functional option type for API.Import.
@@ -916,6 +1033,13 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption {
 	}
 }
 
+func OptImportOptionsPresorted(b bool) ImportOption {
+	return func(o *ImportOptions) error {
+		o.Presorted = b
+		return nil
+	}
+}
+
 // Import bulk imports data into a particular index,field,shard.
 func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
@@ -925,44 +1049,53 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
 		return errors.Wrap(err, "validating api method")
 	}
 
+	index, field, err := api.indexField(req.Index, req.Field, req.Shard)
+	if err != nil {
+		return errors.Wrap(err, "getting index and field")
+	}
+
+	if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
+		return errors.Wrap(err, "validating import value request")
+	}
+
 	// Set up import options.
 	options, err := setUpImportOptions(opts...)
 	if err != nil {
 		return errors.Wrap(err, "setting up import options")
 	}
-
+	span.LogKV(
-	index, field, err := api.indexField(req.Index, req.Field, req.Shard)
+		"index", req.Index,
-	if err != nil {
+		"field", req.Field)
-		return errors.Wrap(err, "getting index and field")
-	}
 
 	// Unless explicitly ignoring key validation (meaning keys have been
 	// translated to ids in a previous step at the coordinator node), then
 	// check to see if keys need translation.
 	if !options.IgnoreKeyCheck {
 		// Translate row keys.
-		if field.keys() {
+		if field.Keys() {
+			span.LogKV("rowKeys", true)
 			if len(req.RowIDs) != 0 {
 				return errors.New("row ids cannot be used because field uses string keys")
 			}
-			if req.RowIDs, err = field.translateStore.TranslateKeys(req.RowKeys); err != nil {
+			if req.RowIDs, err = api.cluster.translateFieldKeys(ctx, field, req.RowKeys...); err != nil {
-				return errors.Wrap(err, "translating rows")
+				return errors.Wrapf(err, "translating field keys")
 			}
 		}
 
 		// Translate column keys.
 		if index.Keys() {
+			span.LogKV("columnKeys", true)
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
 		}
 
 		// For translated data, map the columnIDs to shards. If
 		// this node does not own the shard, forward to the node that does.
-		if index.Keys() || field.keys() {
+		if index.Keys() || field.Keys() {
 			m := make(map[uint64][]Bit)
 
 			for i, colID := range req.ColumnIDs {
@@ -1036,66 +1169,79 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 		return errors.Wrap(err, "validating api method")
 	}
 
+	index, field, err := api.indexField(req.Index, req.Field, req.Shard)
+	if err != nil {
+		return errors.Wrap(err, "getting index and field")
+	}
+
+	if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
+		return errors.Wrap(err, "validating import value request")
+	}
+
 	// Set up import options.
 	options, err := setUpImportOptions(opts...)
 	if err != nil {
 		return errors.Wrap(err, "setting up import options")
 	}
 
-	index, field, err := api.indexField(req.Index, req.Field, req.Shard)
+	index, field, err = api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
 	}
-
+	span.LogKV(
+		"index", req.Index,
+		"field", req.Field)
 	// Unless explicitly ignoring key validation (meaning keys have been
 	// translate to ids in a previous step at the coordinator node), then
 	// check to see if keys need translation.
 	if !options.IgnoreKeyCheck {
 		// Translate column keys.
 		if index.Keys() {
+			span.LogKV("columnKeys", true)
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
-
+			req.Shard = math.MaxUint64
-			// For translated data, map the columnIDs to shards. If
-			// this node does not own the shard, forward to the node that does.
-			m := make(map[uint64][]FieldValue)
-
-			for i, colID := range req.ColumnIDs {
-				shard := colID / ShardWidth
-				if _, ok := m[shard]; !ok {
-					m[shard] = make([]FieldValue, 0)
-				}
-				m[shard] = append(m[shard], FieldValue{
-					Value:    req.Values[i],
-					ColumnID: colID,
-				})
 		}
 
-			// Signal to the receiving nodes to ignore checking for key translation.
+		// Translate values when the field uses keys (for example, when
-			opts = append(opts, OptImportOptionsIgnoreKeyCheck(true))
+		// the field has a ForeignIndex with keys).
-
+		if field.Keys() {
-			var eg errgroup.Group
+			// Perform translation.
-			for shard, vals := range m {
+			span.LogKV("rowKeys", true)
-				// TODO: if local node owns this shard we don't need to go through the client
+			uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues)
-				shard := shard
+			if err != nil {
-				vals := vals
+				return err
-				eg.Go(func() error {
-					return api.server.defaultClient.ImportValue(ctx, req.Index, req.Field, shard, vals, opts...)
-				})
 			}
-			return eg.Wait()
+
+			// Because the BSI field supports negative values, we have to
+			// convert the uint64 keys to a slice of int64.
+			ints := make([]int64, len(uints))
+			for i := range uints {
+				ints[i] = int64(uints[i])
+			}
+			req.Values = ints
 		}
 	}
 
-	// Validate shard ownership.
+	if !options.Presorted {
+		sort.Sort(req)
+	}
+
+	// if we're importing into a specific shard
+	if req.Shard != math.MaxUint64 {
+		// Check that column IDs match the stated shard.
+		if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard {
+			return errors.Errorf("shard %d specified, but import spans shards %d to %d", req.Shard, s1, s2)
+		}
+		// Validate shard ownership. TODO - we should forward to the
+		// correct node rather than barfing here.
 		if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
 			return errors.Wrap(err, "validating shard ownership")
 		}
-
 		// Import columnIDs into existence field.
 		if !options.Clear {
 			if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
@@ -1105,11 +1251,95 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 		}
 
 		// Import into fragment.
+		if len(req.Values) > 0 {
 			err = field.importValue(req.ColumnIDs, req.Values, options)
 			if err != nil {
 				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
 			}
-	return errors.Wrap(err, "importing")
+		} else if len(req.FloatValues) > 0 {
+			err = field.importFloatValue(req.ColumnIDs, req.FloatValues, options)
+			if err != nil {
+				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+			}
+		}
+
+		return errors.Wrap(err, "importing value")
+	}
+
+	options.IgnoreKeyCheck = true
+	start := 0
+	shard := req.ColumnIDs[0] / ShardWidth
+	var eg errgroup.Group // TODO make this a pooled errgroup
+	for i, colID := range req.ColumnIDs {
+		if colID/ShardWidth != shard {
+			subreq := &ImportValueRequest{
+				Index:     req.Index,
+				Field:     req.Field,
+				Shard:     shard,
+				ColumnIDs: req.ColumnIDs[start:i],
+			}
+			if req.Values != nil {
+				subreq.Values = req.Values[start:i]
+			} else if req.FloatValues != nil {
+				subreq.FloatValues = req.FloatValues[start:i]
+			}
+
+			eg.Go(func() error {
+				return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+			})
+			start = i
+			shard = colID / ShardWidth
+		}
+	}
+	subreq := &ImportValueRequest{
+		Index:     req.Index,
+		Field:     req.Field,
+		Shard:     shard,
+		ColumnIDs: req.ColumnIDs[start:],
+	}
+	if req.Values != nil {
+		subreq.Values = req.Values[start:]
+	} else if req.FloatValues != nil {
+		subreq.FloatValues = req.FloatValues[start:]
+	}
+	eg.Go(func() error {
+		// TODO we should elevate the logic for figuring out which
+		// node(s) to send to into API instead of having those details
+		// in the client implementation.
+		return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+	})
+	return eg.Wait()
+
+}
+
+func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.ImportColumnAttrs")
+	defer span.Finish()
+
+	index, err := api.Index(ctx, req.Index)
+	if err != nil {
+		return errors.Wrap(err, "getting index")
+	}
+
+	if err := api.validateShardOwnership(req.Index, uint64(req.Shard)); err != nil {
+		return errors.Wrap(err, "validating shard ownership")
+	}
+
+	if req.IndexCreatedAt != 0 {
+		if index.CreatedAt() != req.IndexCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+
+	bulkAttrs := make(map[uint64]map[string]interface{})
+	for n := 0; n < len(req.ColumnIDs); n++ {
+		bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}
+	}
+	if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil {
+		api.server.logger.Printf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing column attrs")
+	}
+	return nil
 }
 
 func importExistenceColumns(index *Index, columnIDs []uint64) error {
@@ -1285,15 +1515,80 @@ func (api *API) Info() serverInfo {
 	}
 }
 
+func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
+	return api.holder.Inspect(ctx, req)
+}
+
 // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
-func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) {
+func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")
 	defer span.Finish()
-	return api.holder.TranslateEntryReader(ctx, offsets)
+
+	// Ensure all readers are cleaned up if any error.
+	var a []TranslateEntryReader
+	defer func() {
+		if err != nil {
+			for i := range a {
+				a[i].Close()
+			}
+		}
+	}()
+
+	// Fetch all index partition readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for partitionID, offset := range indexMap.Partitions {
+			store := index.TranslateStore(partitionID)
+			if store == nil {
+				return nil, ErrTranslateStoreNotFound
+			}
+
+			r, err := store.EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "index partition translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	// Fetch all field readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for fieldName, offset := range indexMap.Fields {
+			field := index.Field(fieldName)
+			if field == nil {
+				return nil, ErrFieldNotFound
+			}
+
+			r, err := field.TranslateStore().EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "field translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	return NewMultiTranslateEntryReader(ctx, a), nil
+}
+
+func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	return api.cluster.translateIndexKey(ctx, indexName, key)
+}
+
+func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	return api.cluster.translateIndexIDs(ctx, indexName, ids)
 }
 
 // TranslateKeys handles a TranslateKeyRequest.
-func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
+func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) {
 	var req TranslateKeysRequest
 	if buf, err := ioutil.ReadAll(r); err != nil {
 		return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error"))
@@ -1302,13 +1597,22 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	}
 
 	// Lookup store for either index or field and translate keys.
-	store, err := api.holder.TranslateStore(req.Index, req.Field)
+	var ids []uint64
+	if req.Field == "" {
+		if ids, err = api.cluster.translateIndexKeys(ctx, req.Index, req.Keys); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			ids, err = api.cluster.translateIndexKeys(ctx, fi, req.Keys)
 			if err != nil {
 				return nil, err
 			}
-	ids, err := store.TranslateKeys(req.Keys)
+		} else if ids, err = api.cluster.translateFieldKeys(ctx, field, req.Keys...); err != nil {
-	if err != nil {
+			return nil, errors.Wrapf(err, "translating field keys")
-		return nil, err
+		}
 	}
 
 	// Encode response.
@@ -1319,6 +1623,42 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	return buf, nil
 }
 
+// TranslateIDs handles a TranslateIDRequest.
+func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) {
+	var req TranslateIDsRequest
+	if buf, err := ioutil.ReadAll(r); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error"))
+	} else if err := api.Serializer.Unmarshal(buf, &req); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error"))
+	}
+
+	// Lookup store for either index or field and translate ids.
+	var keys []string
+	if req.Field == "" {
+		if keys, err = api.cluster.translateIndexIDs(ctx, req.Index, req.IDs); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			keys, err = api.cluster.translateIndexIDs(ctx, fi, req.IDs)
+			if err != nil {
+				return nil, err
+			}
+		} else if keys, err = field.TranslateStore().TranslateIDs(req.IDs); err != nil {
+			return nil, err
+		}
+	}
+
+	// Encode response.
+	buf, err := api.Serializer.Marshal(&TranslateIDsResponse{Keys: keys})
+	if err != nil {
+		return nil, errors.Wrap(err, "translate ids response encoding error")
+	}
+	return buf, nil
+}
+
 // PrimaryReplicaNodeURL returns the URL of the cluster's primary replica.
 func (api *API) PrimaryReplicaNodeURL() url.URL {
 	node := api.cluster.PrimaryReplicaNode()
@@ -1328,6 +1668,74 @@ func (api *API) PrimaryReplicaNodeURL() url.URL {
 	return node.URI.URL()
 }
 
+func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, remote bool) (*Transaction, error) {
+	if err := api.validate(apiStartTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
+	if exclusive {
+		switch err {
+		case nil:
+			api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
+		case ErrTransactionExclusive:
+			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
+		}
+		if t.Active {
+			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+		}
+	} else {
+		switch err {
+		case nil:
+			api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
+		case ErrTransactionExclusive:
+			api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
+		}
+	}
+	return t, err
+}
+
+func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) {
+	if err := api.validate(apiFinishTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	t, err := api.server.FinishTransaction(ctx, id, remote)
+	if err == nil {
+		if t.Exclusive {
+			api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
+		} else {
+			api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
+		}
+	}
+	return t, err
+}
+
+func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	if err := api.validate(apiTransactions); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.server.Transactions(ctx)
+}
+
+func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) {
+	if err := api.validate(apiGetTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	t, err := api.server.GetTransaction(ctx, id, remote)
+	if err == nil {
+		if t.Exclusive && t.Active {
+			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+		}
+	}
+	return t, err
+}
+
+func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) {
+	if err := api.validate(apiActiveQueries); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.tracker.ActiveQueries(), nil
+}
+
 type serverInfo struct {
 	ShardWidth       uint64 `json:"shardWidth"`
 	Memory           uint64 `json:"memory"`
@@ -1352,6 +1760,7 @@ const (
 	apiFragmentBlockData
 	apiFragmentBlocks
 	apiFragmentData
+	apiTranslateData
 	apiField
 	apiFieldAttrDiff
 	//apiHosts // not implemented
@@ -1374,6 +1783,11 @@ const (
 	//apiVersion // not implemented
 	apiViews
 	apiApplySchema
+	apiStartTransaction
+	apiFinishTransaction
+	apiTransactions
+	apiGetTransaction
+	apiActiveQueries
 )
 
 var methodsCommon = map[apiMethod]struct{}{
@@ -1383,6 +1797,7 @@ var methodsCommon = map[apiMethod]struct{}{
 
 var methodsResizing = map[apiMethod]struct{}{
 	apiFragmentData:  {},
+	apiTranslateData: {},
 	apiResizeAbort:   {},
 }
 
@@ -1408,4 +1823,9 @@ var methodsNormal = map[apiMethod]struct{}{
 	apiShardNodes:           {},
 	apiViews:                {},
 	apiApplySchema:          {},
+	apiStartTransaction:     {},
+	apiFinishTransaction:    {},
+	apiTransactions:         {},
+	apiGetTransaction:       {},
+	apiActiveQueries:        {},
 }

api/client/grpc.go

@@ -0,0 +1,213 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +31,139 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c[0]
+	m1 := c[1]
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		indexName := "i"
+		fieldName := "f"
+		attrKey := "k"
+
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, indexName, fieldName)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs0,
+			AttrVals:       attrVals0,
+			Shard:          0,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:        attrKey,
+			ColumnIDs:      columnIDs1,
+			AttrVals:       attrVals1,
+			Shard:          1,
+			Index:          indexName,
+			IndexCreatedAt: index.CreatedAt(),
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -54,36 +188,46 @@ func TestAPI_Import(t *testing.T) {
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "rick"
+		indexName := "rick"
-		field := "f"
+		fieldName := "f"
 
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if index.CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
+		if field.CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
 
 		rowID := uint64(1)
 		timestamp := int64(0)
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
 		for i := 1; i <= 10; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
-			Index:      index,
+			Index:          indexName,
-			Field:      field,
+			IndexCreatedAt: index.CreatedAt(),
+			Field:          fieldName,
+			FieldCreatedAt: field.CreatedAt(),
 			Shard:          0,
 			RowIDs:         rowIDs,
 			ColumnKeys:     colKeys,
@@ -93,10 +237,10 @@ func TestAPI_Import(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
+		pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 			t.Fatal(err)
 		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 			t.Fatalf("unexpected column keys: %#v", keys)
@@ -104,7 +248,7 @@ func TestAPI_Import(t *testing.T) {
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
 				return err
 			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
 				return fmt.Errorf("unexpected column keys: %#v", keys)
@@ -129,58 +273,6 @@ func TestAPI_Import(t *testing.T) {
 		}
 
 	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
@@ -219,12 +311,13 @@ func TestAPI_ImportValue(t *testing.T) {
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
@@ -258,6 +351,120 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+		if err := m1.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		query := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: %+v", ids)
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		if err := m0.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: %+v", ids)
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.

apimethod_string.go

@@ -19,25 +19,30 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
+	_ = x[apiTranslateData-11]
-	_ = x[apiFieldAttrDiff-12]
+	_ = x[apiField-12]
-	_ = x[apiImport-13]
+	_ = x[apiFieldAttrDiff-13]
-	_ = x[apiImportValue-14]
+	_ = x[apiImport-14]
-	_ = x[apiIndex-15]
+	_ = x[apiImportValue-15]
-	_ = x[apiIndexAttrDiff-16]
+	_ = x[apiIndex-16]
-	_ = x[apiQuery-17]
+	_ = x[apiIndexAttrDiff-17]
-	_ = x[apiRecalculateCaches-18]
+	_ = x[apiQuery-18]
-	_ = x[apiRemoveNode-19]
+	_ = x[apiRecalculateCaches-19]
-	_ = x[apiResizeAbort-20]
+	_ = x[apiRemoveNode-20]
-	_ = x[apiSetCoordinator-21]
+	_ = x[apiResizeAbort-21]
-	_ = x[apiShardNodes-22]
+	_ = x[apiSetCoordinator-22]
-	_ = x[apiViews-23]
+	_ = x[apiShardNodes-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiViews-24]
+	_ = x[apiApplySchema-25]
+	_ = x[apiStartTransaction-26]
+	_ = x[apiFinishTransaction-27]
+	_ = x[apiTransactions-28]
+	_ = x[apiGetTransaction-29]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransaction"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 332, 345, 353, 367, 386, 406, 421, 438}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

boltdb/translate.go

@@ -11,10 +11,13 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
@@ -31,9 +34,14 @@ var (
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
 )
 
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+)
+
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +53,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
 	index       string
 	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +81,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -108,6 +128,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,12 +160,12 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKey converts a string key to an integer ID.
 // If key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	// Find id by key under read lock.
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
+		id, _ = findIDByKey(tx.Bucket([]byte("keys")), key)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -156,13 +181,16 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
 		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
+
+		var boltKey []byte
+		if id, boltKey = findIDByKey(bkt, key); id != 0 {
 			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
+		}
+
+		id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+		if err := bkt.Put(boltKey, u64tob(id)); err != nil {
 			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
+		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
 			return err
 		}
 		written = true
@@ -178,8 +206,8 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	return id, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
-// If key does not have an associated id then one is created.
+// If a key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if len(keys) == 0 {
 		return nil, nil
@@ -193,7 +221,7 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket([]byte("keys"))
 		for i, key := range keys {
-			if id := findIDByKey(bkt, key); id != 0 {
+			if id, _ := findIDByKey(bkt, key); id != 0 {
 				ids[i] = id
 				found++
 			}
@@ -218,13 +246,15 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 				continue
 			}
 
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
+			var boltKey []byte
+			if ids[i], boltKey = findIDByKey(bkt, key); ids[i] != 0 {
 				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
+			}
+
+			ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+			if err := bkt.Put(boltKey, u64tob(ids[i])); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
+			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), boltKey); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
 				return err
 			}
 			written = true
@@ -286,7 +316,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +341,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		max = maxID(tx)
-			max = btou64(key)
-		}
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +349,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -387,13 +470,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
+// emptyKey is a sentinel byte slice which stands for "" as a key.
-	if value := bkt.Get([]byte(key)); value != nil {
+var emptyKey = []byte{
-		return btou64(value)
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
 }
-	return 0
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
+	}
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }

boltdb/translate_test.go

@@ -14,9 +14,12 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"testing"
 	"time"
 
@@ -28,24 +31,23 @@ func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
+	// Ensure initial key translates to first ID for shard
-	if id, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo")
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
 	if id, err := s.TranslateKey("bar"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
 	if id, err := s.TranslateKey("foo"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
@@ -55,31 +57,30 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Ensure initial keys translate to incrementing IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"})
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
 	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
 	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
@@ -89,24 +90,38 @@ func TestTranslateStore_TranslateID(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo")
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("")
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
@@ -114,12 +129,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"})
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -136,7 +152,8 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"})
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +168,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +177,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz")
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -198,20 +216,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo")
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -302,11 +325,76 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
 func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 	s := MustNewTranslateStore()

broadcast.go

@@ -69,6 +69,7 @@ const (
 	messageTypeRecalculateCaches
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -116,6 +117,8 @@ func getMessage(typ byte) Message {
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -155,6 +158,8 @@ func getMessageType(m Message) byte {
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +121,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -136,7 +140,8 @@ var _ cache = &lruCache{}
 type rankCache struct {
 	mu           sync.Mutex
 	entries      map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
 
 	updateN    int
 	updateTime time.Time
@@ -172,6 +177,7 @@ func (c *rankCache) Add(id uint64, n uint64) {
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -185,6 +191,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -209,12 +216,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
+	if len(c.entries) == 0 {
-	for id := range c.entries {
+		return nil
-		a = append(a, id)
 	}
-	sort.Sort(uint64Slice(a))
+	ids := make([]uint64, 0, len(c.entries))
-	return a
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +238,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -238,25 +248,33 @@ func (c *rankCache) invalidate() {
 	if time.Since(c.updateTime).Seconds() < 10 {
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,7 +290,7 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@@ -285,7 +303,13 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +333,61 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
+func (p *bitmapPairs) Len() int           { return len(*p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +463,73 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 

cache_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
+// Ensure cache stays constrained to its configured size.
-func TestCache_Rank(t *testing.T) {
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
 }

client.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -43,7 +44,11 @@ type FieldValue struct {
 // something hasn't been architected correctly.
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
+// Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client.
+// Doing that may obviate the need to refactor this.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
@@ -51,7 +56,6 @@ type InternalClient interface {
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
 	Nodes(ctx context.Context) ([]*Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,6 +63,7 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@@ -68,7 +73,14 @@ type InternalClient interface {
 	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
 	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error)
 	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
 }
 
 //===============
@@ -76,6 +88,8 @@ type InternalClient interface {
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
 	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
 }
 
 type nopInternalQueryClient struct{}
@@ -84,6 +98,14 @@ func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index
 	return nil, nil
 }
 
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
 func newNopInternalQueryClient() *nopInternalQueryClient {
 	return &nopInternalQueryClient{}
 }
@@ -123,15 +145,30 @@ func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest
 func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
+func (n nopInternalClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+func (n nopInternalClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
+	return nil
+}
+
 func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -172,3 +209,19 @@ func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte
 func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
 	return nil, nil
 }
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}

cluster.go

@@ -40,8 +40,8 @@ import (
 )
 
 const (
-	// defaultPartitionN is the default number of partitions in a cluster.
+	// DefaultPartitionN is the default number of partitions in a cluster.
-	defaultPartitionN = 256
+	DefaultPartitionN = 256
 
 	// ClusterState represents the state returned in the /status endpoint.
 	ClusterStateStarting = "STARTING"
@@ -62,15 +62,15 @@ const (
 	resizeJobActionAdd    = "ADD"
 	resizeJobActionRemove = "REMOVE"
 
-	confirmDownRetries = 10
+	defaultConfirmDownRetries = 10
-	confirmDownSleep   = 1
+	defaultConfirmDownSleep   = 1 * time.Second
-	confirmDownTimeout = 2
 )
 
 // Node represents a node in the cluster.
 type Node struct {
 	ID            string `json:"id"`
 	URI           URI    `json:"uri"`
+	GRPCURI       URI    `json:"grpc-uri"`
 	IsCoordinator bool   `json:"isCoordinator"`
 	State         string `json:"state"`
 }
@@ -110,6 +110,17 @@ func (a Nodes) ContainsID(id string) bool {
 	return false
 }
 
+// NodeByID returns the node for an ID. If the ID is not found,
+// it returns nil.
+func (a Nodes) NodeByID(id string) *Node {
+	for _, n := range a {
+		if n.ID == id {
+			return n
+		}
+	}
+	return nil
+}
+
 // Filter returns a new list of nodes with node removed.
 func (a Nodes) Filter(n *Node) []*Node {
 	other := make([]*Node, 0, len(a))
@@ -223,6 +234,9 @@ type cluster struct { // nolint: maligned
 	joined  bool
 
 	abortAntiEntropyCh chan struct{}
+	muAntiEntropy      sync.Mutex
+
+	translationSyncer translationSyncer
 
 	mu         sync.RWMutex
 	jobs       map[int64]*resizeJob
@@ -235,13 +249,16 @@ type cluster struct { // nolint: maligned
 	logger logger.Logger
 
 	InternalClient InternalClient
+
+	confirmDownRetries int
+	confirmDownSleep   time.Duration
 }
 
 // newCluster returns a new instance of Cluster with defaults.
 func newCluster() *cluster {
 	return &cluster{
 		Hasher:     &jmphasher{},
-		partitionN: defaultPartitionN,
+		partitionN: DefaultPartitionN,
 		ReplicaN:   1,
 
 		joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
@@ -249,9 +266,14 @@ func newCluster() *cluster {
 		closing:             make(chan struct{}),
 		joining:             make(chan struct{}),
 
+		translationSyncer: NopTranslationSyncer,
+
 		InternalClient: newNopInternalClient(),
 
 		logger: logger.NopLogger,
+
+		confirmDownRetries: defaultConfirmDownRetries,
+		confirmDownSleep:   defaultConfirmDownSleep,
 	}
 }
 
@@ -390,14 +412,6 @@ func (c *cluster) addNode(node *Node) error {
 		return nil
 	}
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// add to topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -417,14 +431,6 @@ func (c *cluster) removeNode(nodeID string) error {
 	// remove from cluster
 	c.removeNodeBasicSorted(nodeID)
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// remove from topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -477,7 +483,7 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	switch state {
 	case ClusterStateNormal, ClusterStateDegraded:
-		// If state is RESIZING -> NORMAL then run cleanup.
+		// If state is RESIZING -> [NORMAL, DEGRADED] then run cleanup.
 		if c.state == ClusterStateResizing {
 			doCleanup = true
 		}
@@ -485,8 +491,28 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	c.state = state
 
-	if state == ClusterStateResizing {
+	switch state {
-		c.abortAntiEntropy()
+	case ClusterStateNormal:
+		// Because the cluster state is changing to NORMAL,
+		// we [potentially] need to reset the translation sync.
+		// If, for example, the cluster has changed size and is
+		// now settling to NORMAL, the partition ownership may
+		// have changed, and this will force that to be recalculated.
+		//
+		// We can't call Reset() if Server.Open() hasn't run yet,
+		// because that's where we start monitorResetTranslationSync()
+		// which reads the reset channel. If we get here before
+		// Server.Open(), this will deadlock on that channel read.
+		// In order to address this, we call Reset() in a goroutine
+		// so even if it blocks waiting for monitorResetTranslationSync()
+		// to start, it doesn't cause a deadlock, and once Server.Open()
+		// is called, then the sync reset (or in the STARTING case, the
+		// initial sync start) will happen.
+		go func() {
+			if err := c.translationSyncer.Reset(); err != nil {
+				c.logger.Printf("error resetting translation syncer: %s", err)
+			}
+		}()
 	}
 
 	// TODO: consider NOT running cleanup on an active node that has
@@ -575,6 +601,19 @@ func (c *cluster) determineClusterState() (clusterState string) {
 	if c.haveTopologyAgreement() && c.allNodesReady() {
 		return ClusterStateNormal
 	}
+	// TODO:
+	// If the cluster is still STARTING, there's no need to put it into
+	// state DEGRADED. It's possible to force a starting cluster to go
+	// into state DEGRADED by, for example, restarting a 2-node cluster
+	// with replica=3. In that case, the coordinator would come up and
+	// it would immediately trigger this condition. Checking for
+	// state != STARTING here would prevent that. Unfortunately, based
+	// on test TestClusteringNodesReplica2, we expect a DEGRADED cluster
+	// to go back into state STARTING if it loses more replicas than
+	// can support queries. In that case, we might actually want it to
+	// go from STARTING back to DEGRADED. Leaving it as is for now, but
+	// noting that it's a little confusing that a cluster starting up
+	// could possibly go into state DEGRADED.
 	if len(c.Topology.nodeIDs)-len(c.nodeIDs()) < c.ReplicaN && c.allNodesReady() {
 		return ClusterStateDegraded
 	}
@@ -587,6 +626,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus {
 		ClusterID: c.id,
 		State:     c.state,
 		Nodes:     c.nodes,
+		Schema:    &Schema{Indexes: c.holder.Schema()},
 	}
 }
 
@@ -636,6 +676,7 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool {
 			n.State = node.State
 			n.IsCoordinator = node.IsCoordinator
 			n.URI = node.URI
+			n.GRPCURI = node.GRPCURI
 			return true
 		}
 		return false
@@ -867,8 +908,72 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour
 	return m, nil
 }
 
-// partition returns the partition that a shard belongs to.
+// translationNodes returns a list of translationResizeNodes - for each node
-func (c *cluster) partition(index string, shard uint64) int {
+// in the `to` cluster - required to move from cluster `c` to cluster `to`. unprotected.
+// Because the parition scheme for every index is the same, this is used as a template
+// to create index-specific `TranslationResizeSource`s.
+func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResizeNode, error) {
+	m := make(map[string][]*translationResizeNode)
+
+	// Determine if a node is being added or removed.
+	action, diffNodeID, err := c.diff(to)
+	if err != nil {
+		return nil, errors.Wrap(err, "diffing")
+	}
+
+	// Initialize the map with all the nodes in `to`.
+	for _, n := range to.nodes {
+		m[n.ID] = nil
+	}
+
+	for pid := 0; pid < c.partitionN; pid++ {
+		fNodes := c.partitionNodes(pid)
+		tNodes := to.partitionNodes(pid)
+
+		// For `to` cluster, we include all nodes containing a
+		// replica for the partition. The source for each replica
+		// will be the primary in the `from` cluster. For the `from`
+		// cluster, we only need the first node, unless that node is
+		// being removed, then we use the second node. If no second
+		// node exists in that case, then we have to raise an error
+		// indicating that not enough replicas exist to support
+		// the resize.
+		if len(tNodes) > 0 {
+			var foundPrimary bool
+			for i := range fNodes {
+				if action == resizeJobActionRemove && fNodes[i].ID == diffNodeID {
+					continue
+				}
+				// We only need to add the source if the nodes differ;
+				// in other words if the primary partition is on the
+				// same node, it doesn't need to retrieve it.
+				for n := range tNodes {
+					if tNodes[n].ID != fNodes[i].ID {
+						m[tNodes[n].ID] = append(m[tNodes[n].ID],
+							&translationResizeNode{
+								node:        fNodes[i],
+								partitionID: pid,
+							})
+					}
+				}
+				foundPrimary = true
+				break
+			}
+			if !foundPrimary {
+				return nil, ErrResizeNoReplicas
+			}
+		}
+	}
+
+	return m, nil
+}
+
+// shardPartition returns the partition that a shard belongs to.
+func (c *cluster) shardPartition(index string, shard uint64) int {
+	return shardPartition(index, shard, c.partitionN)
+}
+
+func shardPartition(index string, shard uint64, partitionN int) int {
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], shard)
 
@@ -876,7 +981,25 @@ func (c *cluster) partition(index string, shard uint64) int {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(index))
 	_, _ = h.Write(buf[:])
-	return int(h.Sum64() % uint64(c.partitionN))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// keyPartition returns the partition that a key belongs to.
+func (c *cluster) keyPartition(index, key string) int {
+	return keyPartition(index, key, c.partitionN)
+}
+
+func keyPartition(index, key string, partitionN int) int {
+	// Hash the bytes and mod by partition count.
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(index))
+	_, _ = h.Write([]byte(key))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// idPartition returns the partition that an id belongs to.
+func (c *cluster) idPartition(index string, id uint64) int {
+	return shardPartition(index, id/ShardWidth, c.partitionN)
 }
 
 // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use.
@@ -888,7 +1011,19 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
 
 // shardNodes returns a list of nodes that own a fragment. unprotected
 func (c *cluster) shardNodes(index string, shard uint64) []*Node {
-	return c.partitionNodes(c.partition(index, shard))
+	return c.partitionNodes(c.shardPartition(index, shard))
+}
+
+// KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use.
+func (c *cluster) KeyNodes(index, key string) []*Node {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.keyNodes(index, key)
+}
+
+// keyNodes returns a list of nodes that own a key. unprotected
+func (c *cluster) keyNodes(index, key string) []*Node {
+	return c.partitionNodes(c.keyPartition(index, key))
 }
 
 // ownsShard returns true if a host owns a fragment.
@@ -900,33 +1035,73 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
 
 // partitionNodes returns a list of nodes that own a partition. unprotected.
 func (c *cluster) partitionNodes(partitionID int) []*Node {
-
 	// Default replica count to between one and the number of nodes.
 	// The replica count can be zero if there are no nodes.
+
+	// Assume that c.nodes may be missing a node that is part of the cluster but not currently present.
+	// The partition calculation must use the full cluster size in BOTH cases:
+	// - use len(c.Topology.nodeIDs) instead of len(c.nodes),
+	// - collect nodes from c.Topology.nodeIDs rather than from c.nodes,
+	// - when the node is missing, it should be considered, found absent from c.nodes, then omitted from the return slice.
+
+	// Use c.Topology to determine cluster membership when it
+	// exists and contains data. Otherwise, fall back to using
+	// c.nodes. The only time c.Topology should be nil is in
+	// tests.
+	var useTopology bool
+	if c.Topology != nil && len(c.Topology.nodeIDs) > 0 {
+		useTopology = true
+	}
+
 	replicaN := c.ReplicaN
-	if replicaN > len(c.nodes) {
+	var nodeN int
-		replicaN = len(c.nodes)
+	if useTopology {
+		nodeN = len(c.Topology.nodeIDs)
+	} else {
+		nodeN = len(c.nodes)
+	}
+	if replicaN > nodeN {
+		replicaN = nodeN
 	} else if replicaN == 0 {
 		replicaN = 1
 	}
 
 	// Determine primary owner node.
-	nodeIndex := c.Hasher.Hash(uint64(partitionID), len(c.nodes))
+	nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN)
 
 	// Collect nodes around the ring.
-	nodes := make([]*Node, replicaN)
+	nodes := make([]*Node, 0, replicaN)
 	for i := 0; i < replicaN; i++ {
-		nodes[i] = c.nodes[(nodeIndex+i)%len(c.nodes)]
+		if useTopology {
+			maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN]
+			if node := Nodes(c.nodes).NodeByID(maybeNodeID); node != nil {
+				nodes = append(nodes, node)
+			}
+		} else {
+			nodes = append(nodes, c.nodes[(nodeIndex+i)%len(c.nodes)])
+		}
 	}
 
 	return nodes
 }
 
+// ownsPartition returns true if a host owns a partition.
+func (c *cluster) ownsPartition(nodeID string, partition int) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.unprotectedOwnsPartition(nodeID, partition)
+}
+
+// unprotectedOwnsPartition returns true if a host owns a partition.
+func (c *cluster) unprotectedOwnsPartition(nodeID string, partition int) bool {
+	return Nodes(c.partitionNodes(partition)).ContainsID(nodeID)
+}
+
 // containsShards is like OwnsShards, but it includes replicas.
 func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 {
 	var shards []uint64
 	availableShards.ForEach(func(i uint64) {
-		p := c.partition(index, i)
+		p := c.shardPartition(index, i)
 		// Determine the nodes for partition.
 		nodes := c.partitionNodes(p)
 		for _, n := range nodes {
@@ -998,11 +1173,11 @@ func (c *cluster) waitForStarted() error {
 	if !c.isCoordinator() {
 		// In the case where a node has been restarted and memberlist has
 		// not had enough time to determine the node went down/up, then
-		// the coorninator needs to be alerted that this node is back up
+		// the coordinator needs to be alerted that this node is back up
 		// (and now in a state of STARTING) so that it can be put to the correct
 		// cluster state.
 		// TODO: Because the normal code path already sends a NodeJoin event (via
-		// memberlist), this it a bit redundant in most cases. Perhaps determine
+		// memberlist), this is a bit redundant in most cases. Perhaps determine
 		// that the node has been restarted and don't do this step.
 		msg := &NodeEvent{
 			Event: NodeJoin,
@@ -1127,7 +1302,6 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
 	// Broadcast cluster status changes to the cluster.
 	status := c.unprotectedStatus()
 	return c.unprotectedSendSync(status) // TODO fix c.Status
-
 }
 
 func (c *cluster) sendTo(node *Node, m Message) error {
@@ -1149,9 +1323,7 @@ func (c *cluster) listenForJoins() {
 		// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
 		// We use a bool `setNormal` to indicate when at least one node has joined.
 		var setNormal bool
-
 		for {
-
 			// Handle all pending joins before changing state back to NORMAL.
 			select {
 			case nodeAction := <-c.joiningLeavingNodes:
@@ -1234,36 +1406,80 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (*
 		toCluster.addNodeBasicSorted(nodeAction.node)
 	}
 
-	// multiIndex is a map of sources initialized with all the nodes in toCluster.
+	indexes := c.holder.Indexes()
-	multiIndex := make(map[string][]*ResizeSource)
 
+	// fragmentSourcesByNode is a map of Node.ID to sources of fragment data.
+	// It is initialized with all the nodes in toCluster.
+	fragmentSourcesByNode := make(map[string][]*ResizeSource)
 	for _, n := range toCluster.nodes {
-		multiIndex[n.ID] = nil
+		fragmentSourcesByNode[n.ID] = nil
 	}
 
-	// Add to multiIndex the instructions for each index.
+	// Add to fragmentSourcesByNode the instructions for each index.
-	for _, idx := range c.holder.Indexes() {
+	for _, idx := range indexes {
 		fragSources, err := c.fragSources(toCluster, idx)
 		if err != nil {
 			return nil, errors.Wrap(err, "getting sources")
 		}
 
-		for id, sources := range fragSources {
+		for nodeid, sources := range fragSources {
-			multiIndex[id] = append(multiIndex[id], sources...)
+			fragmentSourcesByNode[nodeid] = append(fragmentSourcesByNode[nodeid], sources...)
 		}
 	}
 
-	for id, sources := range multiIndex {
+	// translationSourcesByNode is a map of Node.ID to sources of partitioned
-		// If a host doesn't need to request data, mark it as complete.
+	// key translation data for indexes.
-		if len(sources) == 0 {
+	// It is initialized with all the nodes in toCluster.
-			j.IDs[id] = true
+	translationSourcesByNode := make(map[string][]*TranslationResizeSource)
+	for _, n := range toCluster.nodes {
+		translationSourcesByNode[n.ID] = nil
+	}
+
+	if len(indexes) > 0 {
+		// Add to translationSourcesByNode the instructions for the cluster.
+		translationNodes, err := c.translationNodes(toCluster)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting translation sources")
+		}
+
+		// Create a list of TranslationResizeSource for each index,
+		// using translationNodes as a template.
+		translationSources := make(map[string][]*TranslationResizeSource)
+		for _, idx := range indexes {
+			// Only include indexes with keys.
+			if !idx.Keys() {
 				continue
 			}
+			indexName := idx.Name()
+			for node, resizeNodes := range translationNodes {
+				for i := range resizeNodes {
+					translationSources[node] = append(translationSources[node],
+						&TranslationResizeSource{
+							Node:        resizeNodes[i].node,
+							Index:       indexName,
+							PartitionID: resizeNodes[i].partitionID,
+						})
+				}
+			}
+		}
+
+		for nodeid, sources := range translationSources {
+			translationSourcesByNode[nodeid] = sources
+		}
+	}
+
+	for _, node := range toCluster.nodes {
+		// We may send a resize instruction that has no sources that
+		// the node needs to read from -- for instance, if there's no
+		// data in any fragments it would process. But it still needs
+		// to get the NodeStatus to pick up the schema so it knows
+		// about existing indexes.
 		instr := &ResizeInstruction{
 			JobID:              j.ID,
-			Node:          toCluster.unprotectedNodeByID(id),
+			Node:               toCluster.unprotectedNodeByID(node.ID),
 			Coordinator:        c.unprotectedCoordinatorNode(),
-			Sources:       sources,
+			Sources:            fragmentSourcesByNode[node.ID],
+			TranslationSources: translationSourcesByNode[node.ID],
 			NodeStatus:         c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node.
 			ClusterStatus:      c.unprotectedStatus(),
 		}
@@ -1302,7 +1518,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 		return errors.Wrap(err, "merging cluster status")
 	}
 
-	c.logger.Printf("done MergeClusterStatus, start goroutine")
+	c.logger.Printf("done MergeClusterStatus, start goroutine (%s)", c.Node.ID)
 
 	// The actual resizing runs in a goroutine because we don't want to block
 	// the distribution of other ResizeInstructions to the rest of the cluster.
@@ -1349,9 +1565,8 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 
 			// Request each source file in ResizeSources.
 			for _, src := range instr.Sources {
-				c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
-
 				srcURI := src.Node.URI
+				c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, srcURI)
 
 				// Retrieve field.
 				f := c.holder.Field(src.Index, src.Field)
@@ -1372,7 +1587,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 				}
 
 				// Stream shard from remote node.
-				c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
+				c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, srcURI)
 				rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.View, src.Shard, srcURI)
 				if err != nil {
 					// For now it is an acceptable error if the fragment is not found
@@ -1386,7 +1601,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 					}
 					return errors.Wrap(err, "retrieving shard")
 				} else if rd == nil {
-					return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI)
+					return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, srcURI)
 				}
 
 				// Write to local field and always close reader.
@@ -1398,6 +1613,37 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 					return errors.Wrap(err, "copying remote shard")
 				}
 			}
+
+			// Request each translation source file in TranslationResizeSources.
+			for _, src := range instr.TranslationSources {
+				srcURI := src.Node.URI
+
+				idx := c.holder.Index(src.Index)
+				if idx == nil {
+					return ErrIndexNotFound
+				}
+
+				// Retrieve partition from remote node.
+				c.logger.Printf("retrieve translate partition %d for index %s from host %s", src.PartitionID, src.Index, srcURI)
+				rd, err := c.InternalClient.RetrieveTranslatePartitionFromURI(ctx, src.Index, src.PartitionID, srcURI)
+				if err != nil {
+					return errors.Wrap(err, "retrieving translate partition")
+				} else if rd == nil {
+					return fmt.Errorf("partition %d doesn't exist on host: %s", src.PartitionID, src.Node.URI)
+				}
+
+				// Write to local store and always close reader.
+				if err := func() error {
+					defer rd.Close()
+					// Get the translate store for this index/partition.
+					store := idx.TranslateStore(src.PartitionID)
+					_, err = store.ReadFrom(rd)
+					return errors.Wrap(err, "reading from reader")
+				}(); err != nil {
+					return errors.Wrap(err, "copying remote partition")
+				}
+			}
+
 			return nil
 		}(); err != nil {
 			complete.Error = err.Error()
@@ -1551,6 +1797,7 @@ func (j *resizeJob) distributeResizeInstructions() error {
 		node := &Node{
 			ID:      instr.Node.ID,
 			URI:     instr.Node.URI,
+			GRPCURI: instr.Node.GRPCURI,
 		}
 		j.Logger.Printf("send resize instructions: %v", instr)
 		if err := j.Broadcaster.SendTo(node, instr); err != nil {
@@ -1721,7 +1968,7 @@ func (c *cluster) considerTopology() error {
 // band aid to protect against false nodeLeave events from memberlist
 // the test is the lightest weight endpoint of the node in question /version
 // TODO provide more robust solution to false nodeLeave events
-func confirmNodeDown(uri URI, log logger.Logger) bool {
+func (c *cluster) confirmNodeDown(uri URI) bool {
 	u := url.URL{
 		Scheme: uri.Scheme,
 		Host:   uri.HostPort(),
@@ -1729,11 +1976,11 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
 	}
 	req, err := http.NewRequest("GET", u.String(), nil)
 	if err != nil {
-		log.Printf("bad request:%s %s", u.String(), err)
+		c.logger.Printf("bad request:%s %s", u.String(), err)
 		return false
 	}
-	for i := 0; i < confirmDownRetries; i++ {
+	for i := 0; i < c.confirmDownRetries; i++ {
-		ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second)
+		ctx, cancel := context.WithTimeout(context.Background(), c.confirmDownSleep*2)
 		defer cancel()
 		resp, err := http.DefaultClient.Do(req.WithContext(ctx))
 		var bod []byte
@@ -1744,8 +1991,8 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
 			}
 		}
 
-		log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
+		c.logger.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
-		time.Sleep(confirmDownSleep * time.Second)
+		time.Sleep(c.confirmDownSleep)
 	}
 	return true
 }
@@ -1773,7 +2020,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 			// not already removed by a removeNode request. We treat this as the
 			// host being temporarily unavailable, and expect it to come back
 			// up.
-			if confirmNodeDown(e.Node.URI, c.logger) {
+			if c.confirmNodeDown(e.Node.URI) {
 				if c.removeNodeBasicSorted(e.Node.ID) {
 					c.Topology.nodeStates[e.Node.ID] = nodeStateDown
 					// put the cluster into STARTING if we've lost a number of nodes
@@ -1794,6 +2041,17 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 
 // nodeJoin should only be called by the coordinator.
 func (c *cluster) nodeJoin(node *Node) error {
+	c.abortAntiEntropy()
+	// Technically there is a race condition here which could
+	// allow the anti-entropy process to re-start (and acquire
+	// the lock) before this lock has time to succeed. In that
+	// case, the user would have to wait through an entire
+	// anti-entropy cycle. We decided it wasn't worth the
+	// complexity (of, for example, implementing this with
+	// channels) to avoid that rare case.
+	c.muAntiEntropy.Lock()
+	defer c.muAntiEntropy.Unlock()
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.logger.Printf("node join event on coordinator, node: %s, id: %s", node.URI, node.ID)
@@ -1818,7 +2076,9 @@ func (c *cluster) nodeJoin(node *Node) error {
 			if c.haveTopologyAgreement() {
 				return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
 			}
-			return nil
+			// This lets the remote node to proceed with opening its holder,
+			// instead of waiting in DOWN state because cluster is in STARTING state.
+			return c.sendTo(node, c.unprotectedStatus())
 		} else if err != nil {
 			return errors.Wrap(err, "checking if holder has data")
 		}
@@ -1839,6 +2099,9 @@ func (c *cluster) nodeJoin(node *Node) error {
 			c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI)
 			cnode.URI = node.URI
 		}
+		if cnode.GRPCURI != node.GRPCURI {
+			cnode.GRPCURI = node.GRPCURI
+		}
 		return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
 	}
 
@@ -1864,6 +2127,17 @@ func (c *cluster) nodeJoin(node *Node) error {
 
 // nodeLeave initiates the removal of a node from the cluster.
 func (c *cluster) nodeLeave(nodeID string) error {
+	c.abortAntiEntropy()
+	// Technically there is a race condition here which could
+	// allow the anti-entropy process to re-start (and acquire
+	// the lock) before this lock has time to succeed. In that
+	// case, the user would have to wait through an entire
+	// anti-entropy cycle. We decided it wasn't worth the
+	// complexity (of, for example, implementing this with
+	// channels) to avoid that rare case.
+	c.muAntiEntropy.Lock()
+	defer c.muAntiEntropy.Unlock()
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	// Refuse the request if this is not the coordinator.
@@ -1873,8 +2147,8 @@ func (c *cluster) nodeLeave(nodeID string) error {
 	}
 
 	if c.state != ClusterStateNormal && c.state != ClusterStateDegraded {
-		return fmt.Errorf("cluster must be '%s' to remove a node but is '%s'",
+		return fmt.Errorf("cluster must be '%s' or '%s' to remove a node but is '%s'",
-			ClusterStateNormal, c.state)
+			ClusterStateNormal, ClusterStateDegraded, c.state)
 	}
 
 	// Ensure that node is in the cluster.
@@ -1923,7 +2197,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
 	}
 	var availableShards *roaring.Bitmap
 	for _, idx := range ns.Schema.Indexes {
-		is := &IndexStatus{Name: idx.Name}
+		is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
 		for _, f := range idx.Fields {
 			if field := c.holder.Field(idx.Name, f.Name); field != nil {
 				availableShards = field.AvailableShards()
@@ -1932,6 +2206,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
 			}
 			is.Fields = append(is.Fields, &FieldStatus{
 				Name:            f.Name,
+				CreatedAt:       f.CreatedAt,
 				AvailableShards: availableShards,
 			})
 		}
@@ -1943,7 +2218,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
 func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.logger.Printf("merge cluster status: node=%s cluster=%v", c.Node.ID, cs)
+	c.logger.Printf("merge cluster status: node=%s cluster=%v, topologySize=%v", c.Node.ID, cs, len(c.Topology.nodeIDs))
 	// Ignore status updates from self (coordinator).
 	if c.unprotectedIsCoordinator() {
 		return nil
@@ -2047,12 +2322,187 @@ func (c *cluster) setStatic(hosts []string) error {
 	return nil
 }
 
+// translateFieldKey gets a single key from translateFieldKeys.
+func (c *cluster) translateFieldKey(ctx context.Context, field *Field, key string) (uint64, error) {
+	ids, err := c.translateFieldKeys(ctx, field, key)
+	if err != nil {
+		return 0, err
+	} else if len(ids) == 0 {
+		return 0, errors.New("translating key on coordinator returned empty set")
+	}
+	return ids[0], nil
+}
+
+// translateFieldKeys is basically a wrapper around
+// field.TranslateStore().TranslateKey(key), but in
+// the case where the local node's translate store
+// is read-only (i.e. it's not the primary translate
+// store), then this method will forward the translation
+// request to the coordinator.
+func (c *cluster) translateFieldKeys(ctx context.Context, field *Field, keys ...string) ([]uint64, error) {
+	ids, err := field.TranslateStore().TranslateKeys(keys)
+	// If we get a "read only" error, then forward the request
+	// to the coordinator.
+	if errors.Cause(err) == ErrTranslateStoreReadOnly {
+		coordinatorNode := c.coordinatorNode()
+		if ids, err := c.InternalClient.TranslateKeysNode(ctx, &coordinatorNode.URI, field.Index(), field.Name(), keys); err != nil {
+			return ids, errors.Wrap(err, "translating keys on coordinator")
+		} else {
+			return ids, nil
+		}
+	}
+	return ids, err
+}
+
+func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}})
+	if err != nil {
+		return 0, err
+	}
+	return keyMap[key], nil
+}
+
+func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys []string) ([]uint64, error) {
+	keySet := make(map[string]struct{})
+	for _, key := range keys {
+		keySet[key] = struct{}{}
+	}
+
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, keySet)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make([]uint64, len(keys))
+	for i := range keys {
+		ids[i] = keyMap[keys[i]]
+	}
+	return ids, nil
+}
+
+func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}) (map[string]uint64, error) {
+	keyMap := make(map[string]uint64)
+
+	idx := c.holder.Index(indexName)
+	if idx == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split keys by partition.
+	keysByPartition := make(map[int][]string, c.partitionN)
+	for key := range keySet {
+		partitionID := c.keyPartition(indexName, key)
+		keysByPartition[partitionID] = append(keysByPartition[partitionID], key)
+	}
+
+	// Translate keys by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range keysByPartition {
+		partitionID := partitionID
+		keys := keysByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var ids []uint64
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, indexName, "", keys); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range keys {
+				keyMap[keys[i]] = ids[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return keyMap, nil
+}
+
+func (c *cluster) translateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	idSet := make(map[uint64]struct{})
+	for _, id := range ids {
+		idSet[id] = struct{}{}
+	}
+
+	idMap, err := c.translateIndexIDSet(ctx, indexName, idSet)
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, len(ids))
+	for i := range ids {
+		keys[i] = idMap[ids[i]]
+	}
+	return keys, nil
+}
+
+func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idSet map[uint64]struct{}) (map[uint64]string, error) {
+	idMap := make(map[uint64]string, len(idSet))
+
+	index := c.holder.Index(indexName)
+	if index == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split ids by partition.
+	idsByPartition := make(map[int][]uint64, c.partitionN)
+	for id := range idSet {
+		partitionID := c.idPartition(indexName, id)
+		idsByPartition[partitionID] = append(idsByPartition[partitionID], id)
+	}
+
+	// Translate ids by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range idsByPartition {
+		partitionID := partitionID
+		ids := idsByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var keys []string
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if keys, err = c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, indexName, "", ids); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range ids {
+				idMap[ids[i]] = keys[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return idMap, nil
+}
+
 // ClusterStatus describes the status of the cluster including its
 // state and node topology.
 type ClusterStatus struct {
 	ClusterID string
 	State     string
 	Nodes     []*Node
+	Schema    *Schema
 }
 
 // ResizeInstruction contains the instruction provided to a node
@@ -2062,6 +2512,7 @@ type ResizeInstruction struct {
 	Node               *Node
 	Coordinator        *Node
 	Sources            []*ResizeSource
+	TranslationSources []*TranslationResizeSource
 	NodeStatus         *NodeStatus
 	ClusterStatus      *ClusterStatus
 }
@@ -2076,9 +2527,24 @@ type ResizeSource struct {
 	Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
 }
 
+// TranslationResizeSource is the source of translation data for
+// a node acting on a ResizeInstruction.
+type TranslationResizeSource struct {
+	Node        *Node
+	Index       string
+	PartitionID int
+}
+
+// translateResizeNode holds the node/partition pairs used
+// to create a TranslationResizeSource for each index.
+type translationResizeNode struct {
+	node        *Node
+	partitionID int
+}
+
 // Schema contains information about indexes and their configuration.
 type Schema struct {
-	Indexes []*IndexInfo
+	Indexes []*IndexInfo `json:"indexes"`
 }
 
 func encodeTopology(topology *Topology) *internal.Topology {
@@ -2117,6 +2583,7 @@ type CreateShardMessage struct {
 // CreateIndexMessage is an internal message indicating index creation.
 type CreateIndexMessage struct {
 	Index     string
+	CreatedAt int64
 	Meta      *IndexOptions
 }
 
@@ -2129,6 +2596,7 @@ type DeleteIndexMessage struct {
 type CreateFieldMessage struct {
 	Index     string
 	Field     string
+	CreatedAt int64
 	Meta      *FieldOptions
 }
 
@@ -2193,15 +2661,29 @@ type NodeStatus struct {
 // IndexStatus is an internal message representing the contents of an index.
 type IndexStatus struct {
 	Name      string
+	CreatedAt int64
 	Fields    []*FieldStatus
 }
 
 // FieldStatus is an internal message representing the contents of a field.
 type FieldStatus struct {
 	Name            string
+	CreatedAt       int64
 	AvailableShards *roaring.Bitmap
 }
 
 // RecalculateCaches is an internal message for recalculating all caches
 // within a holder.
 type RecalculateCaches struct{}
+
+// Transaction Actions
+const (
+	TRANSACTION_START    = "start"
+	TRANSACTION_FINISH   = "finish"
+	TRANSACTION_VALIDATE = "validate"
+)
+
+type TransactionMessage struct {
+	Transaction *Transaction
+	Action      string
+}

cluster_internal_test.go

@@ -96,7 +96,7 @@ func newIndexWithTempPath(name string) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	index, err := NewIndex(NewHolder(DefaultPartitionN), path, name)
 	if err != nil {
 		panic(err)
 	}
@@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
+	defer idx.Close()
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
@@ -351,7 +352,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := c.shardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -942,15 +943,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c := newCluster()
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be up")
 	}
 
 }
 func TestCluster_confirmNodeDownTimeout(t *testing.T) {
+	sleep := 50 * time.Millisecond
+	retries := 5
+	if testing.Short() {
+		t.Skip()
+	}
 	r := mux.NewRouter()
 	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
+		time.Sleep(sleep * time.Duration(retries))
 		fmt.Fprintln(w, "ignored")
 	}))
 	server := httptest.NewServer(r)
@@ -969,21 +977,29 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
 		t.Error(err)
 	}
 	uri.Port = uint16(iport)
-
+	c := newCluster()
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	c.confirmDownSleep = sleep
+	c.confirmDownRetries = retries
+	c.logger = logger.NewVerboseLogger(os.Stdout)
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
 }
 
 func TestCluster_confirmNodeDownDown(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	uri := URI{}
 	uri.Scheme = "http"
 	uri.Host = "DoesntMatter"
 	uri.Port = 6666
+	c := newCluster()
+	c.confirmDownSleep = 50 * time.Millisecond
+	c.confirmDownRetries = 5
+	c.logger = logger.NewVerboseLogger(os.Stdout)
 
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
+	if !c.confirmNodeDown(uri) {
 		t.Errorf("expected node to be down")
 	}
-
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
+					Max:       pql.NewDecimal(100, 0),
-					Min:       -10,
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/inspect.go

@@ -45,5 +45,12 @@ Inspects a data file and provides stats.
 			return inspector.Run(context.Background())
 		},
 	}
+	flags := inspectCmd.Flags()
+	flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
+	flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
+	flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
+	flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
+	flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
+	flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
 	return inspectCmd
 }

cmd/root.go

@@ -26,23 +26,18 @@ import (
 )
 
 func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	productName := "Pilosa " + pilosa.Version
-	if pilosa.EnterpriseEnabled {
-		productName = "Pilosa Enterprise " + pilosa.Version
-	}
 	rc := &cobra.Command{
 		Use: "pilosa",
+		// TODO: These short/long descriptions could use some updating.
 		Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.",
-		// TODO - is documentation actually there?
 		Long: `Pilosa is a fast index to turbocharge your database.
 
 This binary contains Pilosa itself, as well as common
-tools for administering pilosa, importing/exporting data,
+tools for administering Pilosa, importing/exporting data,
 backing up, and more. Complete documentation is available
 at https://www.pilosa.com/docs/.
 
-` + productName + `
+` + pilosa.VersionInfo() + "\n",
-Build Time: ` + pilosa.BuildTime + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
 			err := setAllConfig(v, cmd.Flags(), "PILOSA")

cmd/root_test.go

@@ -55,6 +55,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +67,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -53,6 +53,7 @@ func TestServerConfig(t *testing.T) {
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	max-writes-per-request = 3000
 
 	[cluster]
@@ -96,6 +97,7 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		disabled = true
@@ -122,6 +124,7 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		hosts = [

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
 			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
-		}
 
-		// Parse FieldValue.
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
+			}
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
 			value, err := strconv.ParseInt(record[1], 10, 64)
 			if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
 			}
-		val.Value = value
+			req.Values = append(req.Values, value)
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
 		}
-			a = a[:0]
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
@@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
+		if err := test.RetryUntil(2*time.Second, func() error {
 			qry := "Count(Row(f=foo0))"
 			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
 			if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
+				return fmt.Errorf("Querying data for validation: %s", err)
 			}
 
 			// Read body and unmarshal response.
 			exp := `{"results":[100]}` + "\n"
 			if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
+				return fmt.Errorf("reading: %s", err)
 			} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.

ctl/inspect.go

@@ -16,15 +16,23 @@ package ctl
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
+	"hash/fnv"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
-	"unsafe"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -33,6 +41,12 @@ import (
 type InspectCommand struct {
 	// Path to data file
 	Path string
+	// don't list details of objects
+	Quiet bool
+	// list only this many objects
+	Max int
+	// Filters:
+	InspectOpts pilosa.InspectRequest
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -45,8 +59,119 @@ func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectComman
 	}
 }
 
+type pointerContext struct {
+	from, to uintptr
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(cmd.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+		if cmd.Max != 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
+func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(cmd.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+		if cmd.Max > 0 && printed >= cmd.Max {
+			break
+		}
+	}
+	tw.Flush()
+}
+
 // Run executes the inspect command.
-func (cmd *InspectCommand) Run(_ context.Context) error {
+func (cmd *InspectCommand) Run(ctx context.Context) error {
 	// Open file handle.
 	f, err := os.Open(cmd.Path)
 	if err != nil {
@@ -58,7 +183,173 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "statting file")
 	}
+	if fi.IsDir() {
+		total := 0
+		infos, err := f.Readdir(0)
+		if err != nil {
+			return err
+		}
+		if len(infos) == 0 {
+			return errors.New("directory contains no files")
+		}
 
+		names := make([]string, len(infos))
+		nameToInfo := make(map[string]os.FileInfo, len(infos))
+		// find numeric-only names; we'll operate on
+		// either those, or the whole holder if we find
+		// a .topology file.
+		n := 0
+		for _, fi := range infos {
+			name := fi.Name()
+			if name == ".topology" {
+				return cmd.InspectHolder(ctx, cmd.Path)
+			}
+			if _, err := strconv.Atoi(name); err == nil {
+				names[n] = name
+				nameToInfo[name] = fi
+				n++
+			}
+		}
+		if n == 0 {
+			return fmt.Errorf("directory contains no fragments (looking for numeric names)")
+		}
+		names = names[:n]
+		fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
+		for _, name := range names {
+			f2, err := os.Open(filepath.Join(cmd.Path, name))
+			if err != nil {
+				return fmt.Errorf("opening %q: %v", name, err)
+			}
+			fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
+			err = cmd.InspectFile(f2, nameToInfo[name])
+			total++
+			f2.Close()
+			if err != nil {
+				return fmt.Errorf("inspecting %q: %v", name, err)
+			}
+		}
+		return nil
+	}
+	return cmd.InspectFile(f, fi)
+}
+
+// loadTopology is copied almost exactly from pilosa/cluster.go.
+func loadTopology(path string) (topology internal.Topology, myID string, err error) {
+	buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, errors.Wrap(err, "reading file")
+	}
+	if err := proto.Unmarshal(buf, &topology); err != nil {
+		return topology, myID, errors.Wrap(err, "unmarshalling")
+	}
+	sort.Slice(topology.NodeIDs,
+		func(i, j int) bool {
+			return topology.NodeIDs[i] < topology.NodeIDs[j]
+		})
+	buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
+	if os.IsNotExist(err) {
+		return topology, myID, err
+	} else if err != nil {
+		return topology, myID, nil
+	}
+	myID = strings.TrimSpace(string(buf))
+	return topology, myID, nil
+}
+
+var partitions = make(map[string]map[uint64]int)
+
+func findPartition(index string, shard uint64, partitionN int) (partition int) {
+	var shardMap map[uint64]int
+	var ok bool
+	if shardMap, ok = partitions[index]; !ok {
+		shardMap = make(map[uint64]int)
+		partitions[index] = shardMap
+	}
+	if partition, ok = shardMap[shard]; !ok {
+		var buf [8]byte
+		binary.BigEndian.PutUint64(buf[:], shard)
+
+		// Hash the bytes and mod by partition count.
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(index))
+		_, _ = h.Write(buf[:])
+		partition = int(h.Sum64() % uint64(partitionN))
+		shardMap[shard] = partition
+	}
+	return partition
+}
+
+func findPartitionPath(path string, partitionN int) (int, error) {
+	parts := strings.Split(path, "/")
+	shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return findPartition(parts[0], shard, partitionN), nil
+}
+
+func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
+	holder := pilosa.NewHolder(pilosa.DefaultPartitionN)
+	holder.Path = path
+	holder.Opts.Inspect = true
+	holder.Opts.ReadOnly = true
+	err := holder.Open()
+	if err != nil {
+		return fmt.Errorf("%s: holder open: %v", path, err)
+	}
+	holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
+	if err != nil {
+		return fmt.Errorf("%s: inspect: %v", path, err)
+	}
+	myPartition := 0
+	topology, myID, err := loadTopology(path)
+	if err == nil {
+		fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
+		if len(topology.NodeIDs) > 1 {
+			fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
+		} else {
+			fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
+		}
+		found := false
+		for i := range topology.NodeIDs {
+			if topology.NodeIDs[i] == myID {
+				found = true
+				myPartition = i
+				break
+			}
+		}
+		if !found {
+			fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
+		}
+	} else {
+		fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
+	}
+	for _, name := range holderInfo.FragmentNames {
+		partition, err := findPartitionPath(name, len(topology.NodeIDs))
+		if err != nil {
+			fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
+		} else {
+			if partition == myPartition {
+				fmt.Fprintf(cmd.Stdout, "%s:\n", name)
+			} else {
+				fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
+			}
+		}
+		details := holderInfo.FragmentInfo[name]
+		cmd.DisplayInfo(details.BitmapInfo)
+		if details.BlockChecksums != nil {
+			fmt.Fprintf(cmd.Stdout, "  Checksums [%d total]:\n", len(details.BlockChecksums))
+			for _, block := range details.BlockChecksums {
+				fmt.Fprintf(cmd.Stdout, "   %8d: %x\n", block.ID, block.Checksum)
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
 	// Memory map the file.
 	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
@@ -72,39 +363,37 @@ func (cmd *InspectCommand) Run(_ context.Context) error {
 	}()
 	// Attach the mmap file to the bitmap.
 	t := time.Now()
-	fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...")
+	fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
-	bm := roaring.NewBitmap()
+	var info roaring.BitmapInfo
-	if err := bm.UnmarshalBinary(data); err != nil {
+	_, _, err = roaring.InspectBinary(data, true, &info)
-		return errors.Wrap(err, "unmarshalling")
+	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	cmd.DisplayInfo(info)
+	if err != nil {
+		return errors.Wrap(err, "inspecting")
+	}
+	return nil
 }
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
 
-	// Retrieve stats.
+func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
-	t = time.Now()
+	pC := pointerContext{
-	fmt.Fprintf(cmd.Stderr, "calculating stats...")
+		from: info.From,
-	info := bm.Info()
+		to:   info.To,
-	fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
+	}
 
 	// Print top-level info.
-	fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n")
+	fmt.Fprintf(cmd.Stdout, "  Bitmap Info:\n")
-	fmt.Fprintf(cmd.Stdout, "Containers: %d\n", len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Bits: %d\n", info.BitCount)
-	fmt.Fprintf(cmd.Stdout, "Operations: %d\n", info.OpN)
+	fmt.Fprintf(cmd.Stdout, "    Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
+	fmt.Fprintf(cmd.Stdout, "    Operations: %d (%d bits)\n", info.Ops, info.OpN)
 	fmt.Fprintln(cmd.Stdout, "")
 
 	// Print info for each container.
-	fmt.Fprintln(cmd.Stdout, "== Containers ==")
+	if !cmd.Quiet {
-	tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
+		if info.ContainerCount > 0 {
-	fmt.Fprintf(tw, "%s\t%s\t% 8s \t% 8s\t%s\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET")
+			cmd.PrintContainers(info, pC)
-	for _, ci := range info.Containers {
+		}
-		fmt.Fprintf(tw, "%d\t%s\t% 8d \t% 8d \t0x%08x\n",
+		if info.Ops > 0 {
-			ci.Key,
+			cmd.PrintOps(info)
-			ci.Type,
+		}
-			ci.N,
-			ci.Alloc,
-			uintptr(ci.Pointer)-uintptr(unsafe.Pointer(&data[0])),
-		)
 	}
-	tw.Flush()
-
-	return nil
 }

ctl/inspect_test.go

@@ -41,7 +41,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	file.Close()
 	cm.Path = file.Name()
 	err = cm.Run(context.Background())
-	expectedError := "unmarshalling: "
+	expectedError := "inspecting: "
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Fatalf("expected error '%s', got '%v'", expectedError, err)
 	}
@@ -52,7 +52,7 @@ func TestInspectCommand_Run(t *testing.T) {
 	if err != nil {
 		t.Fatalf("copying data: %v", err)
 	}
-	if !strings.Contains(buf.String(), "unmarshalling bitmap...") {
+	if !strings.Contains(buf.String(), "inspecting bitmap...") {
 		t.Fatalf("Inspect doesn't work: %s", err)
 	}
 

ctl/server.go

@@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
@@ -70,14 +71,14 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
+	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
 	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
 	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
 	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
+	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
 	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling

diagnostics.go

@@ -233,7 +233,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {

docs/api-reference.md

@@ -24,22 +24,26 @@ curl -XGET localhost:10101/index/user
 ```
 ``` response
 {
-    "fields": [
-        {
-            "name": "event",
-            "options": {
-                "keys": false,
-                "timeQuantum": "YMD",
-                "type": "time"
-            }
-        }
-    ],
   "name": "user",
+  "createdAt": 1591178953061239000,
   "options": {
     "keys": false,
     "trackExistence": true
+  },
+  "fields": [
+    {
+      "name": "event",
+      "createdAt": 1591178962332452000,
+      "options": {
+        "type": "set",
+        "cacheType": "ranked",
+        "cacheSize": 50000,
+        "keys": false
       }
     }
+  ],
+  "shardWidth": 1048576
+}
 ```
 
 ### Create index
@@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option
 curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"user","createdAt":1591179042178854000}
 ```
 
 ### Remove index
@@ -151,6 +155,17 @@ represents a particular bit to be set. Timestamps are optional, but if they
 exist must also contain the same number of items as rows and columns. The
 column IDs must all be in the shard specified in the request.
 
+Some endpoints and data structures include a `CreatedAt` fields.
+This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
+but to serve as a unique identifier for use in cache invalidation.
+
+The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
+can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
+This is true except in cases where an index or field gets deleted and then recreated,
+or if Pilosa is restored from a backup.
+So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
+and the ingester will know that it needs to drop its cache.
+
 ```
 message ImportRequest {
     string Index = 1;
@@ -158,13 +173,16 @@ message ImportRequest {
     uint64 Shard = 3;
     repeated uint64 RowIDs = 4;
     repeated uint64 ColumnIDs = 5;
+    repeated int64 Timestamps = 6;
     repeated string RowKeys = 7;
     repeated string ColumnKeys = 8;
-	repeated int64 Timestamps = 6;
+    int64 IndexCreatedAt = 9;
+    int64 FieldCreatedAt = 10;
 }
 ```
 
 
+
 ### Create field
 
 `POST /index/<index-name>/field/<field-name>`
@@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \
      -d '{"options": {"type": "int", "min": -1000, "max":2000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"quantity","createdAt":1591180110914425000}
 ```
 
 Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
@@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit,
 curl localhost:10101/index/user/field/language -X POST
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"language","createdAt":1591180128294321000}
 ```
 
 ``` request
 curl localhost:10101/index/repository/field/stats \
     -X POST \
-    -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
+    -d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
 ```
 ``` response
-{"success":true}
+{"success":true,"name":"stats","createdAt":1591180737881627000}
 ```
 
 ### Remove field
@@ -247,30 +265,48 @@ curl -XGET localhost:10101/schema
 {
   "indexes": [
     {
+      "name": "user",
+      "createdAt": 1591178953061239000,
+      "options": {
+        "keys": false,
+        "trackExistence": true
+      },
       "fields": [
         {
           "name": "event",
+          "createdAt": 1591178962332452000,
           "options": {
-                        "keys": false,
+            "type": "set",
-                        "timeQuantum": "YMD",
+            "cacheType": "ranked",
-                        "type": "time"
+            "cacheSize": 50000,
+            "keys": false
           }
         },
         {
           "name": "language",
+          "createdAt": 1591180128294321000,
           "options": {
-                        "cacheSize": 50000,
+            "type": "set",
             "cacheType": "ranked",
+            "cacheSize": 50000,
+            "keys": false
+          }
+        },
+        {
+          "name": "quantity",
+          "createdAt": 1591180110914425000,
+          "options": {
+            "type": "int",
+            "base": 0,
+            "bitDepth": 0,
+            "min": -1000,
+            "max": 2000,
             "keys": false,
-                        "type": "set"
+            "foreignIndex": ""
           }
         }
       ],
-            "name": "user",
+      "shardWidth": 1048576
-            "options": {
-                "keys": false,
-                "trackExistence": true
-            }
     }
   ]
 }
@@ -304,7 +340,7 @@ Returns the version of the Pilosa server.
 curl -XGET localhost:10101/version
 ```
 ``` response
-{"version":"v0.6.0"}
+{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
 ```
 
 ### Get status
@@ -318,19 +354,25 @@ curl -XGET localhost:10101/status
 ```
 ```response
 {
-    "localID": "d3369125-29d8-4305-a351-b4474d14a542",
+  "state": "NORMAL",
   "nodes": [
     {
-            "id": "d3369125-29d8-4305-a351-b4474d14a542",
+      "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
-            "isCoordinator": true,
       "uri": {
+        "scheme": "http",
         "host": "localhost",
-                "port": 10101,
+        "port": 10101
-                "scheme": "http"
+      },
-            }
+      "grpc-uri": {
+        "scheme": "http",
+        "host": "localhost",
+        "port": 20101
+      },
+      "isCoordinator": true,
+      "state": "READY"
     }
   ],
-    "state": "NORMAL"
+  "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
 }
 ```
 

docs/configuration.md

@@ -403,7 +403,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Tracing Sampler Type
 
-* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely.
+* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely. Default is 'off'.
 * Flag: `tracing.sampler-type`
 * Env: `PILOSA_TRACING_SAMPLER_TYPE`
 * Config:

docs/data-model.md

@@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
 #### Set
 
 Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
+Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
+```go
+[]byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+```
+(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
+In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
 
 ``` request
 curl localhost:10101/index/repository/field/info \

docs/getting-started.md

@@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
 ``` response
 {"success":true}
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
@@ -325,7 +325,7 @@ Next, let's create the `repository` index:
 	repository := schema.Index("repository")
 ```
 
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -615,7 +615,7 @@ Next, let's create the `repository` index:
 ```
 	Index repository = schema.index("repository");
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -818,7 +818,7 @@ Next, let's create the `repository` index:
 ```
 repository = schema.index("repository")
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```

docs/query-language.md

@@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
 * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
 * `UINT` An unsigned integer (e.g. 42839).
 * `BOOL` A boolean value, `true` or `false`.
@@ -595,34 +595,6 @@ Count(Row(stargazer=1))
 
 * Result is the number of repositories that user 1 has starred.
 
-#### Shift
-**Spec:**
-
-```
-Shift(<ROW_CALL>, [n=UINT])
-```
-
-**Description:**
-
-Returns the row specified by `ROW_CALL` shifted by `n` bits.
-
-**Result Type:** object with attrs and columns
-
-attrs will always be empty
-
-**Examples:**
-
-Query all columns with a bit set in row 1 of the field `stargazer`
-and shift the result by 2:
-```request
-Shift(Row(stargazer=1), n=2)
-```
-```response
-{"attrs":{},"columns":[12, 22]}
-```
-
-* columns are the repositories which user 1 has starred shifted by 2 bits.
-
 #### TopN
 
 **Spec:**
@@ -864,7 +836,7 @@ Rows(job)
 **Spec:**
 
 ```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
+GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>, aggregate=<CALL>)
 ```
 
 **Description:**
@@ -875,13 +847,17 @@ combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
 Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
+the count. This is analagous to a WHERE clause applied to a relational GROUP BY
 query.
 
 The optional `limit` argument limits the number of results returned. The results
 are ordered, so as long as the data isn't changing, the same query will return
 the same result set.
 
+The optional `aggregate` argument takes a `Sum()` query which will be used to
+calculate the sum & count of each group. This is similar to using a `SUM()` in
+the SELECT clause of a relation GROUP BY query.
+
 Paging through results is supported by passing the `previous` argument to each
 of the `Rows` calls in the GroupBy. Take the last result from your previous
 `GroupBy` query, and pass each row ID in that result as the `previous` argument

enterprise/COPYING

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

enterprise/enterprise.go

@@ -1,23 +0,0 @@
-// Copyright (c) 2018 Pilosa Corp. All rights reserved.
-//
-// This file is part of Pilosa Enterprise Edition.
-//
-// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package enterprise is now deprecated, and the functionality under
-// enterprise/b has been copied into roaring/. It existed to inject enterprise
-// implementations of various Pilosa features when Pilosa was built with
-// "ENTERPRISE=1 make install". These features were dual-licensed separately
-// from Pilosa community edition under the AGPL and Pilosa's commercial license.
-package enterprise

executor_internal_test.go

@@ -15,9 +15,11 @@
 package pilosa
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -25,8 +27,13 @@ import (
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
+	holder := NewHolder(DefaultPartitionN)
+
+	cluster := NewTestCluster(1)
+
 	e := &executor{
-		Holder: NewHolder(),
+		Holder:  holder,
+		Cluster: cluster,
 	}
 	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
 	err := e.Holder.Open()
@@ -46,12 +53,12 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("creating fields %v, %v, %v", erra, errb, errc)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
+	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`)
 	if err != nil {
 		t.Fatalf("parsing query: %v", err)
 	}
 	c := query.Calls[0]
-	err = e.translateGroupByCall("i", idx, c)
+	err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 	if err != nil {
 		t.Fatalf("translating call: %v", err)
 	}
@@ -64,6 +71,19 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		}
 	}
 
+	if having, hok := c.Args["having"].(*pql.Call); !hok {
+		t.Fatal("expected having to be a call")
+	} else if cond, cok := having.Args["count"].(*pql.Condition); !cok {
+		t.Fatal("expected condition to be a count")
+	} else if cond.Op != pql.GT {
+		t.Fatal("expected condition op to be >")
+	} else {
+		val, ok := cond.Uint64Value()
+		if !ok || val != uint64(10) {
+			t.Fatal("expected condition val to be uint64(10)")
+		}
+	}
+
 	errTests := []struct {
 		pql string
 		err string
@@ -102,7 +122,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 				t.Fatalf("parsing query: %v", err)
 			}
 			c := query.Calls[0]
-			err = e.translateGroupByCall("i", idx, c)
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 			if err == nil {
 				t.Fatalf("expected error, but translated call is '%s", c)
 			}
@@ -113,6 +133,62 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 	}
 }
 
+func TestExecutor_TranslateRowsOnBool(t *testing.T) {
+	holder := NewHolder(DefaultPartitionN)
+	defer holder.Close()
+
+	e := &executor{
+		Holder:  holder,
+		Cluster: NewTestCluster(1),
+	}
+	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+	err := e.Holder.Open()
+	if err != nil {
+		t.Fatalf("opening holder: %v", err)
+	}
+
+	idx, err := e.Holder.CreateIndex("i", IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+
+	fb, errb := idx.CreateField("b", OptFieldTypeBool())
+	_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
+	if errb != nil || errbk != nil {
+		t.Fatalf("creating fields %v, %v", errb, errbk)
+	}
+
+	_, err1 := fb.SetBit(1, 1, nil)
+	_, err2 := fb.SetBit(2, 2, nil)
+	_, err3 := fb.SetBit(3, 3, nil)
+	if err1 != nil || err2 != nil || err3 != nil {
+		t.Fatalf("setting bit %v, %v, %v", err1, err2, err3)
+	}
+
+	tests := []struct {
+		pql string
+	}{
+		{pql: "Rows(b)"},
+		{pql: "GroupBy(Rows(b))"},
+		{pql: "Set(4, b=true)"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.pql, func(t *testing.T) {
+			query, err := pql.ParseString(test.pql)
+			if err != nil {
+				t.Fatalf("parsing query: %v", err)
+			}
+
+			c := query.Calls[0]
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
+			if err != nil {
+				t.Fatalf("translating call: %v", err)
+			}
+		})
+	}
+}
+
 func isInt(a interface{}) bool {
 	switch a.(type) {
 	case int, int64, uint, uint64:
@@ -222,3 +298,268 @@ func TestFieldRowMarshalJSON(t *testing.T) {
 		t.Fatalf("unexpected json: %s", b)
 	}
 }
+
+func TestExecutor_GroupCountCondition(t *testing.T) {
+	t.Run("satisfiesCondition", func(t *testing.T) {
+		type condCheck struct {
+			cond string
+			exp  bool
+		}
+		tests := []struct {
+			groupCount GroupCount
+			checks     []condCheck
+		}{
+			{
+				groupCount: GroupCount{Count: 100},
+				checks: []condCheck{
+					{cond: "count == 99", exp: false},
+					{cond: "count != 99", exp: true},
+					{cond: "count < 99", exp: false},
+					{cond: "count <= 99", exp: false},
+					{cond: "count > 99", exp: true},
+					{cond: "count >= 99", exp: true},
+
+					{cond: "count == 100", exp: true},
+					{cond: "count != 100", exp: false},
+					{cond: "count < 100", exp: false},
+					{cond: "count <= 100", exp: true},
+					{cond: "count > 100", exp: false},
+					{cond: "count >= 100", exp: true},
+
+					{cond: "count == 101", exp: false},
+					{cond: "count != 101", exp: true},
+					{cond: "count < 101", exp: true},
+					{cond: "count <= 101", exp: true},
+					{cond: "count > 101", exp: false},
+					{cond: "count >= 101", exp: false},
+
+					{cond: "98 < count < 100", exp: false},
+					{cond: "98 < count <= 100", exp: true},
+					{cond: "98 < count < 101", exp: true},
+					{cond: "100 <= count < 102", exp: true},
+					{cond: "100 < count < 102", exp: false},
+					{cond: "98 <= count <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: 100},
+				checks: []condCheck{
+					{cond: "sum == 99", exp: false},
+					{cond: "sum != 99", exp: true},
+					{cond: "sum < 99", exp: false},
+					{cond: "sum <= 99", exp: false},
+					{cond: "sum > 99", exp: true},
+					{cond: "sum >= 99", exp: true},
+
+					{cond: "sum == 100", exp: true},
+					{cond: "sum != 100", exp: false},
+					{cond: "sum < 100", exp: false},
+					{cond: "sum <= 100", exp: true},
+					{cond: "sum > 100", exp: false},
+					{cond: "sum >= 100", exp: true},
+
+					{cond: "sum == 101", exp: false},
+					{cond: "sum != 101", exp: true},
+					{cond: "sum < 101", exp: true},
+					{cond: "sum <= 101", exp: true},
+					{cond: "sum > 101", exp: false},
+					{cond: "sum >= 101", exp: false},
+
+					{cond: "98 < sum < 100", exp: false},
+					{cond: "98 < sum <= 100", exp: true},
+					{cond: "98 < sum < 101", exp: true},
+					{cond: "100 <= sum < 102", exp: true},
+					{cond: "100 < sum < 102", exp: false},
+					{cond: "98 <= sum <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: -100},
+				checks: []condCheck{
+					{cond: "sum == -99", exp: false},
+					{cond: "sum != -99", exp: true},
+					{cond: "sum < -99", exp: true},
+					{cond: "sum <= -99", exp: true},
+					{cond: "sum > -99", exp: false},
+					{cond: "sum >= -99", exp: false},
+
+					{cond: "sum == -100", exp: true},
+					{cond: "sum != -100", exp: false},
+					{cond: "sum < -100", exp: false},
+					{cond: "sum <= -100", exp: true},
+					{cond: "sum > -100", exp: false},
+					{cond: "sum >= -100", exp: true},
+
+					{cond: "sum == -101", exp: false},
+					{cond: "sum != -101", exp: true},
+					{cond: "sum < -101", exp: false},
+					{cond: "sum <= -101", exp: false},
+					{cond: "sum > -101", exp: true},
+					{cond: "sum >= -101", exp: true},
+
+					{cond: "-100 < sum < -98", exp: false},
+					{cond: "-100 <= sum < -98", exp: true},
+					{cond: "-101 < sum < -98", exp: true},
+					{cond: "-102 < sum <= -100", exp: true},
+					{cond: "-102 < sum < -100", exp: false},
+					{cond: "-102 <= sum <= -98", exp: true},
+				},
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
+				for j, check := range test.checks {
+					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
+
+						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
+						if err != nil {
+							t.Fatalf("parsing query: %v", err)
+						}
+						c := query.Calls[0]
+						having := c.Args["having"].(*pql.Call)
+
+						var got bool
+						for subj, cond := range having.Args {
+							switch subj {
+							case "count", "sum":
+								condition, ok := cond.(*pql.Condition)
+								if !ok {
+									t.Fatalf("not a valid condition")
+								}
+								got = test.groupCount.satisfiesCondition(subj, condition)
+							}
+						}
+
+						if got != check.exp {
+							t.Fatalf("expected: %v, but got: %v", check.exp, got)
+						}
+					})
+				}
+			})
+		}
+	})
+}
+
+func TestValCountComparisons(t *testing.T) {
+	tests := []struct {
+		name       string
+		vc         ValCount
+		other      ValCount
+		expLarger  ValCount
+		expSmaller ValCount
+	}{
+		{
+			name: "zero",
+		},
+		{
+			name:       "ints",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 3, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 1},
+			expSmaller: ValCount{Val: 3, Count: 2},
+		},
+		{
+			name:       "floats",
+			vc:         ValCount{FloatVal: 10.2, Count: 1},
+			other:      ValCount{FloatVal: 3.4, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
+			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
+		},
+		{
+			name:       "intsEquality",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 10, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 3},
+			expSmaller: ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:       "floatsEquality",
+			vc:         ValCount{FloatVal: 10.7, Count: 1},
+			other:      ValCount{FloatVal: 10.7, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
+			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			gotLarger := test.vc.larger(test.other)
+			if gotLarger != test.expLarger {
+				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
+			}
+
+			gotSmaller := test.vc.smaller(test.other)
+			if gotSmaller != test.expSmaller {
+				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
+			}
+		})
+	}
+}
+
+func TestToNegInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1 << 63),
+			i64: int64(-1 << 63),
+		},
+		{
+			u64: uint64(1<<63) - 1,
+			i64: int64(-1<<63) + 1,
+		},
+		{
+			u64:      uint64(1<<63) + 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toNegInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}
+
+func TestToInt64(t *testing.T) {
+	tests := []struct {
+		u64      uint64
+		i64      int64
+		overflow bool
+	}{
+		{
+			u64: uint64(1<<63) - 1,
+			i64: 1<<63 - 1,
+		},
+		{
+			u64: uint64(0),
+			i64: 0,
+		},
+		{
+			u64:      uint64(1 << 63),
+			overflow: true,
+		},
+		{
+			u64:      1<<64 - 1,
+			overflow: true,
+		},
+	}
+
+	for _, tc := range tests {
+		val, err := toInt64(tc.u64)
+		if err != nil && !tc.overflow {
+			t.Fatalf("error: %+v, expected: %+v", err, tc)
+		}
+
+		if val != tc.i64 {
+			t.Fatalf("Expected: %+v, Got: %+v", tc.i64, val)
+		}
+	}
+}

extension.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+
+	"github.com/molecula/ext"
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
+func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
+	return wrappedBitmap{bm}
+}
+
+// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
+// an extension Bitmap.
+type wrappedBitmap struct{ *roaring.Bitmap }
+
+// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
+func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
+	if inner, ok := bm.(wrappedBitmap); ok {
+		if inner.Bitmap != nil {
+			return inner.Bitmap
+		}
+		return roaring.NewFileBitmap()
+	}
+	return roaring.NewFileBitmap()
+}
+
+func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
+	return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
+}
+
+func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
+	shifted, err := b.Bitmap.Shift(n)
+	return wrappedBitmap{shifted}, err
+}
+
+func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Flip(start, last)}
+}
+
+func (b wrappedBitmap) New() ext.Bitmap {
+	return WrapBitmap(roaring.NewFileBitmap())
+}
+
+// ContainerBits tries to get one container's worth of bits.
+func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
+	// it's an error to call this with a non-container-aligned offset
+	if offset&0xFFFF != 0 {
+		return nil
+	}
+	if b.Bitmap == nil {
+		fmt.Printf("ContainerBits on bitmap with no contents\n")
+		return nil
+	}
+	if b.Bitmap.Containers == nil {
+		fmt.Printf("ContainerBits on bitmap with nil Containers\n")
+		return nil
+	}
+	c := b.Bitmap.Containers.Get(offset >> 16)
+	if c == nil {
+		return nil
+	}
+	return c.AsBitmap(target)
+}

extensions/distinct.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2019 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,11 +11,11 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-// +build enterprise
 
-package server
+// +build plugindistinct
+
+package extensions
 
 import (
-	_ "github.com/pilosa/pilosa/v2/enterprise"
+	_ "github.com/molecula/extensions/distinct"
 )

extensions/dummy.go

@@ -0,0 +1,18 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package contains only things which are conditional on build
+// tags.
+
+package extensions

field_internal_test.go

@@ -19,7 +19,10 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -66,12 +69,12 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b0, pql.LT, 5, 105, false},
 			{b0, pql.LT, -8, 92, false},
 			{b0, pql.LT, -108, -8, false},
-			{b0, pql.LT, 1005, 1023, false},
+			{b0, pql.LT, 1005, 1024, false},
 			{b0, pql.LT, 0, 100, false},
 
 			{b1, pql.LT, 5, 5, false},
 			{b1, pql.LT, -8, -8, false},
-			{b1, pql.LT, 1005, 255, false},
+			{b1, pql.LT, 1005, 256, false},
 			{b1, pql.LT, 0, 0, false},
 
 			{b2, pql.LT, 5, -95, false},
@@ -89,6 +92,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, pql.GT, -8, -8, false},
 			{b1, pql.GT, 1005, 0, true},
 			{b1, pql.GT, 0, 0, false},
+			{b1, pql.GT, -300, -256, false},
 
 			{b2, pql.GT, 5, -95, false},
 			{b2, pql.GT, -8, -108, false},
@@ -141,6 +145,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, 5, 20, 5, 20, false},
 			{b1, 20, 1005, 20, 255, false},
 			{b1, 1005, 2000, 0, 0, true},
+			{b1, 0, -1, 0, 0, true},
 
 			{b2, 5, 95, -95, -5, false},
 			{b2, 95, 120, -5, 20, false},
@@ -158,7 +163,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	viewName := viewStandard + "_v"
@@ -195,23 +200,23 @@ type TestField struct {
 }
 
 // NewTestField returns a new instance of TestField d/0.
-func NewTestField(opts FieldOption) *TestField {
+func NewTestField(t *testing.T, opts FieldOption) *TestField {
 	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
-	field, err := NewField(path, "i", "f", opts)
+	field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts)
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
 	return &TestField{Field: field}
 }
 
-// MustOpenField returns a new, opened field at a temporary path. Panic on error.
+// OpenField returns a new, opened field at a temporary path.
-func MustOpenField(opts FieldOption) *TestField {
+func OpenField(t *testing.T, opts FieldOption) *TestField {
-	f := NewTestField(opts)
+	f := NewTestField(t, opts)
 	if err := f.Open(); err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
 	return f
 }
@@ -230,7 +235,7 @@ func (f *TestField) Reopen() error {
 	}
 
 	path, index, name := f.Path(), f.Index(), f.Name()
-	f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
+	f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault())
 	if err != nil {
 		return err
 	}
@@ -258,7 +263,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
 
 // Ensure field can open and retrieve a view.
 func TestField_CreateViewIfNotExists(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	// Create view.
@@ -283,7 +288,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
 }
 
 func TestField_SetTimeQuantum(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	// Set & retrieve time quantum.
@@ -302,7 +307,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
 }
 
 func TestField_RowTime(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
@@ -348,7 +353,8 @@ func TestField_RowTime(t *testing.T) {
 }
 
 func TestField_PersistAvailableShards(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -356,6 +362,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
@@ -366,10 +373,71 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap()
@@ -383,12 +451,14 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
-		t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+		t.Fatalf("unexpected available shards (reopen). expected: %v, \n but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+
 	}
 
 	bm1 := roaring.NewBitmap()
@@ -438,3 +508,314 @@ func TestBSIGroup_BaseDefaultValue(t *testing.T) {
 		}
 	}
 }
+
+func TestField_ApplyOptions(t *testing.T) {
+	for i, tt := range []struct {
+		opts    FieldOptions
+		expOpts FieldOptions
+	}{
+		{
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+		},
+	} {
+
+		fld := &Field{}
+		fld.options = *applyDefaultOptions(&FieldOptions{})
+
+		if err := fld.applyOptions(tt.opts); err != nil {
+			t.Fatal(err)
+		}
+
+		if fld.options.CacheType != tt.expOpts.CacheType {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheType value. expected: %s, but got: %s", i, tt.expOpts.CacheType, fld.options.CacheType)
+		} else if fld.options.CacheSize != tt.expOpts.CacheSize {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheSize value. expected: %d, but got: %d", i, tt.expOpts.CacheSize, fld.options.CacheSize)
+		}
+	}
+}
+
+// Ensure that importValue handles requiredDepth correctly.
+// This test sets the same column value to 1, then 8, then 1.
+// A previous bug was incorrectly determining bitDepth based
+// on the values in the import, and not taking existing values
+// into consideration. This would cause an import of 1/8/1
+// to result in a value of 9 instead of 1.
+func TestBSIGroup_importValue(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+
+		if row, err := f.Range(f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+	}
+}
+
+func TestIntField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []int64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []int64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []int64{10},
+			expMax:    ValCount{Val: 10, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []int64{10, 20},
+			expMax:    ValCount{Val: 20, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []int64{10, 20, 10, 10, 20},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11, 44, 1},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}
+
+// Ensure we get errors when they are expected.
+func TestDecimalField_MinMaxBoundaries(t *testing.T) {
+	for i, test := range []struct {
+		scale  int64
+		min    pql.Decimal
+		max    pql.Decimal
+		expErr bool
+	}{
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 0),
+			max:    pql.NewDecimal(math.MaxInt64, 0),
+			expErr: true,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 3),
+			max:    pql.NewDecimal(math.MaxInt64, 3),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(-44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(1, 0),
+			max:    pql.NewDecimal(2, 0),
+			expErr: true,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(math.MinInt64, 18),
+			max:    pql.NewDecimal(math.MaxInt64, 18),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, 20),
+			max:    pql.NewDecimal(2, 20),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -1),
+			max:    pql.NewDecimal(2, -1),
+			expErr: false,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -19),
+			max:    pql.NewDecimal(2, -19),
+			expErr: true,
+		},
+	} {
+		t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
+			_, err := NewField(NewHolder(DefaultPartitionN), "no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
+			if err != nil && test.expErr {
+				if !strings.Contains(err.Error(), "is not supported") {
+					t.Fatal(err)
+				}
+			} else if err != nil && !test.expErr {
+				t.Fatalf("did not expect error, but got: %s", err)
+			} else if err == nil && test.expErr {
+				t.Fatal("expected error, but got none")
+			}
+		})
+	}
+}
+
+func TestDecimalField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeDecimal(3))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []float64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []float64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []float64{10.1},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []float64{10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "another shard",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11, 44.39, 0.23},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importFloatValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if !reflect.DeepEqual(maxvc, test.expMax) {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if !reflect.DeepEqual(minvc, test.expMin) {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}

field_test.go

@@ -144,7 +144,7 @@ func TestField_NameRestriction(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
+	field, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", ".meta", pilosa.OptFieldTypeDefault())
 	if field != nil {
 		t.Fatalf("unexpected field name %s", err)
 	}
@@ -158,6 +158,7 @@ func TestField_NameValidation(t *testing.T) {
 		"under_score",
 		"abc123",
 		"trailing_",
+		"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
 	}
 	invalidFieldNames := []string{
 		"",
@@ -168,7 +169,7 @@ func TestField_NameValidation(t *testing.T) {
 		"abc def",
 		"camelCase",
 		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
+		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
 	path, err := ioutil.TempDir("", "pilosa-field-")
@@ -176,13 +177,13 @@ func TestField_NameValidation(t *testing.T) {
 		panic(err)
 	}
 	for _, name := range validFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatalf("unexpected field name: %s %s", name, err)
 		}
 	}
 	for _, name := range invalidFieldNames {
-		_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
+		_, err := pilosa.NewField(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i", name, pilosa.OptFieldTypeDefault())
 		if err == nil {
 			t.Fatalf("expected error on field name: %s", name)
 		}
@@ -227,3 +228,44 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(diff)
 	}
 }
+
+func TestField_ClearValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on field.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		if changed, err := f.ClearValue(100); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal(err)
+		}
+
+		// Read value.
+		if _, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if exists {
+			t.Fatal("expected value to not exist")
+		}
+	})
+}

fragment_internal_test.go

@@ -25,6 +25,8 @@ import (
 	"math/rand"
 	"os"
 	"reflect"
+	"runtime"
+	"runtime/debug"
 	"sort"
 	"sync/atomic"
 	"testing"
@@ -109,6 +111,11 @@ func TestFragment_ClearBit(t *testing.T) {
 func TestFragment_RowcacheMap(t *testing.T) {
 	var done int64
 	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+	// Under -race, this test turns out to take a fairly long time
+	// to run with larger OpN, because we write 50,000 bits to
+	// the bitmap, and everything is being race-detected, and we don't
+	// actually need that many to get the result we care about.
+	f.MaxOpN = 2000
 	defer f.Clean(t)
 
 	ch := make(chan struct{})
@@ -377,23 +384,29 @@ func TestFragment_Sum(t *testing.T) {
 	defer f.Clean(t)
 
 	// Set values.
-	if _, err := f.setValue(1000, bitDepth, 382); err != nil {
+	vals := []struct {
-		t.Fatal(err)
+		cid uint64
-	} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
+		val int64
-		t.Fatal(err)
+	}{
-	} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
+		{1000, 382},
-		t.Fatal(err)
+		{2000, 300},
-	} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
+		{2500, -600},
+		{3000, 2818},
+		{4000, 300},
+	}
+	for _, v := range vals {
+		if _, err := f.setValue(v.cid, bitDepth, v.val); err != nil {
 			t.Fatal(err)
 		}
+	}
 
 	t.Run("NoFilter", func(t *testing.T) {
 		if sum, n, err := f.sum(nil, bitDepth); err != nil {
 			t.Fatal(err)
-		} else if n != 4 {
+		} else if n != 5 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != 3800 {
+		} else if got, exp := sum, int64(382+300-600+2818+300); got != exp {
-			t.Fatalf("unexpected sum: %d", sum)
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 
@@ -402,8 +415,8 @@ func TestFragment_Sum(t *testing.T) {
 			t.Fatal(err)
 		} else if n != 2 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != 600 {
+		} else if got, exp := sum, int64(300+300); got != exp {
-			t.Fatalf("unexpected sum: %d", sum)
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 
@@ -414,10 +427,10 @@ func TestFragment_Sum(t *testing.T) {
 	t.Run("ClearValue", func(t *testing.T) {
 		if sum, n, err := f.sum(nil, bitDepth); err != nil {
 			t.Fatal(err)
-		} else if n != 3 {
+		} else if n != 4 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != (3800 - 382) {
+		} else if got, exp := sum, int64(3800-382-600); got != exp {
-			t.Fatalf("unexpected sum: got %d, expecting %d", sum, 3800-382)
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 }
@@ -525,6 +538,30 @@ func TestFragment_Range(t *testing.T) {
 		}
 	})
 
+	t.Run("EQOversizeRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		// Set values.
+		if _, err := f.setValue(1000, 1, 0); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.setValue(2000, 1, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for equality.
+		if b, err := f.rangeOp(pql.EQ, 1, 3); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{}) {
+			t.Fatalf("unexpected columns: %+v", b.Columns())
+		}
+		if b, err := f.rangeOp(pql.EQ, 1, 4); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{}) {
+			t.Fatalf("unexpected columns: %+v", b.Columns())
+		}
+	})
+
 	t.Run("NEQ", func(t *testing.T) {
 		f := mustOpenFragment("i", "f", viewStandard, 0, "")
 		defer f.Clean(t)
@@ -596,6 +633,38 @@ func TestFragment_Range(t *testing.T) {
 		}
 	})
 
+	t.Run("LTRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 1, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeOp(pql.LT, 1, 2); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{1}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
+	t.Run("LTMaxRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 2, 3); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.setValue(2, 2, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeLTUnsigned(NewRow(1, 2), 2, 3, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{2}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
 	t.Run("GT", func(t *testing.T) {
 		f := mustOpenFragment("i", "f", viewStandard, 0, "")
 		defer f.Clean(t)
@@ -644,6 +713,40 @@ func TestFragment_Range(t *testing.T) {
 		}
 	})
 
+	t.Run("GTMinRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 2, 0); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.setValue(2, 2, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 0, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{2}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
+	t.Run("GTOversizeRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 2, 0); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.setValue(2, 2, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 4, false); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
 	t.Run("BETWEEN", func(t *testing.T) {
 		f := mustOpenFragment("i", "f", viewStandard, 0, "")
 		defer f.Clean(t)
@@ -691,6 +794,23 @@ func TestFragment_Range(t *testing.T) {
 			t.Fatalf("unexpected columns: %+v", b.Columns())
 		}
 	})
+
+	t.Run("BetweenCommonBitsRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 64, 0xf0); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.setValue(2, 64, 0xf1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeBetweenUnsigned(NewRow(1, 2), 64, 0xf0, 0xf1); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{1, 2}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
 }
 
 // benchmarkSetValues is a helper function to explore, very roughly, the cost
@@ -1382,6 +1502,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 
 	// Read into another fragment.
 	f1 := mustOpenFragment("i", "f", viewStandard, 0, "")
+	defer f1.Clean(t)
 	if rn, err := f1.ReadFrom(&buf); err != nil {
 		t.Fatal(err)
 	} else if wn != rn {
@@ -1414,7 +1535,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
 	}
 
 	// Open the fragment specified by the path.
-	f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0)
+	f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
@@ -1943,7 +2064,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
 
 	b.ReportAllocs()
 	// Open the fragment specified by the path.
-	f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0)
+	f := newFragment(NewHolder(DefaultPartitionN), *FragmentPath, "i", "f", viewStandard, 0, 0)
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
@@ -2054,7 +2175,11 @@ func BenchmarkImportRoaring(b *testing.B) {
 					b.StartTimer()
 					err := f.importRoaringT(data, false)
 					if err != nil {
-						f.awaitSnapshot()
+						// we don't actually particularly
+						// care whether this succeeds,
+						// but if it's happening we want
+						// it to be done.
+						_ = defaultSnapshotQueue.Await(f)
 						f.Clean(b)
 						b.Fatalf("import error: %v", err)
 					}
@@ -2093,7 +2218,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 							j := j
 							eg.Go(func() error {
 								err := frags[j].importRoaringT(data[j], false)
-								frags[j].awaitSnapshot()
+								// error unimportant if it happened, but we want
+								// any snapshots to have finished.
+								_ = defaultSnapshotQueue.Await(frags[j])
 								return err
 							})
 						}
@@ -2131,11 +2258,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								// is excessive. force storage into snapshotted state, then use import
 								// to generate an op log and/or snapshot.
 								_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
-								frags[j].enqueueSnapshot()
-								frags[j].awaitSnapshot()
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
 								}
+								err = defaultSnapshotQueue.Immediate(frags[j])
+								if err != nil {
+									b.Fatalf("snapshot after import: %v", err)
+								}
 							}
 							eg := errgroup.Group{}
 							b.StartTimer()
@@ -2143,7 +2272,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								j := j
 								eg.Go(func() error {
 									err := frags[j].importRoaringT(updata, false)
-									frags[j].awaitSnapshot()
+									err2 := defaultSnapshotQueue.Await(frags[j])
+									if err == nil {
+										err = err2
+									}
 									return err
 								})
 							}
@@ -2205,20 +2337,38 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						// is excessive. force storage into snapshotted state, then use import
 						// to generate an op log and/or snapshot.
 						_, _, err := f.storage.ImportRoaringBits(data, false, false, 0)
-						f.enqueueSnapshot()
-						f.awaitSnapshot()
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
+						err = defaultSnapshotQueue.Immediate(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StartTimer()
 						err = f.importRoaringT(updata, false)
-						f.awaitSnapshot()
 						if err != nil {
 							f.Clean(b)
 							b.Errorf("import error: %v", err)
 						}
+						err = defaultSnapshotQueue.Await(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StopTimer()
-						stat, _ := f.file.Stat()
+						var stat os.FileInfo
+						var statTarget io.Writer
+						err = f.gen.Transaction(&statTarget, func() error {
+							targetFile, ok := statTarget.(*os.File)
+							if ok {
+								stat, _ = targetFile.Stat()
+							} else {
+								b.Errorf("couldn't stat file")
+							}
+							return nil
+						})
+						if err != nil {
+							b.Errorf("transaction error: %v", err)
+						}
 						fileSize[name] = stat.Size()
 						f.Clean(b)
 					}
@@ -2299,7 +2449,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		}
 		origF.Close()
 		fi.Close()
-		nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0)
+		nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
 		err = nf.Open()
 		if err != nil {
 			b.Fatalf("opening fragment: %v", err)
@@ -2336,7 +2486,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
 		}
 		origF.Close()
 		fi.Close()
-		nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0)
+		nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
 		err = nf.Open()
 		if err != nil {
 			b.Fatalf("opening fragment: %v", err)
@@ -2359,14 +2509,16 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
 	if err != nil {
 		t.Fatalf("importing roaring: %v", err)
 	}
-	if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
+	rows := f.rows(context.Background(), 0)
-		t.Fatalf("unexpected rows: %v", f.rows(0))
+	if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
+		t.Fatalf("unexpected rows: %v", rows)
 	}
 	for i := uint64(1); i < 10; i++ {
 		if f.row(i).Count() >= f.row(i-1).Count() {
 			t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
 		}
 	}
+	f.Clean(t)
 }
 
 // getZipfRowsSliceRoaring generates a random fragment with the given number of
@@ -2512,18 +2664,32 @@ func (f *fragment) sanityCheck(t testing.TB) {
 }
 
 func (f *fragment) Clean(t testing.TB) {
-	f.awaitSnapshot()
+	f.mu.Lock()
+	// we need to ensure that we unlock the mutex before terminating
+	// the clean operation, but we need it held during the sanity
+	// check or else, in some cases, the background snapshot queue
+	// can decide to pick it up.
+	func() {
+		defer f.mu.Unlock()
+		err := defaultSnapshotQueue.Await(f)
+		if err != nil {
+			t.Fatalf("snapshot failed before sanity check: %v", err)
+		}
 		f.sanityCheck(t)
+		if f.storage != nil && f.storage.Source != nil {
+			if f.storage.Source.Dead() {
+				t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+			}
+		}
+	}()
 	errc := f.Close()
+	// prevent double-closes of generation during testing.
+	f.gen = nil
 	errf := os.Remove(f.path)
 	errp := os.Remove(f.cachePath())
 	if errc != nil || errf != nil {
 		t.Fatal("cleaning up fragment: ", errc, errf, errp)
 	}
-	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
-		f.snapshotQueue = nil
-	}
 	// not all fragments have cache files
 	if errp != nil && !os.IsNotExist(errp) {
 		t.Fatalf("cleaning up fragment cache: %v", errp)
@@ -2543,10 +2709,6 @@ func (f *fragment) CleanKeep(t testing.TB) {
 	if errc != nil {
 		t.Fatal("closing fragment: ", errc, errp)
 	}
-	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
-		f.snapshotQueue = nil
-	}
 	// not all fragments have cache files
 	if errp != nil && !os.IsNotExist(errp) {
 		t.Fatalf("cleaning up fragment cache: %v", errp)
@@ -2562,6 +2724,12 @@ func mustOpenBSIFragment(index, field, view string, shard uint64) *fragment {
 	return mustOpenFragmentFlags(index, field, view, shard, "", 1)
 }
 
+var testHolder = NewHolder(DefaultPartitionN)
+
+func init() {
+	testHolder.SnapshotQueue = newSnapshotQueue(1, 1, nil)
+}
+
 // mustOpenFragment returns a new instance of Fragment with a temporary path.
 func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) *fragment {
 	file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-")
@@ -2574,12 +2742,12 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st
 		cacheType = DefaultCacheType
 	}
 
-	f := newFragment(file.Name(), index, field, view, shard, flags)
+	f := newFragment(testHolder, file.Name(), index, field, view, shard, flags)
+
 	f.CacheType = cacheType
 	f.RowAttrStore = &memAttrStore{
 		store: make(map[uint64]map[string]interface{}),
 	}
-	f.snapshotQueue = newSnapshotQueue(1, 1, nil)
 
 	if err := f.Open(); err != nil {
 		panic(err)
@@ -2648,12 +2816,12 @@ func TestFragment_RowsIteration(t *testing.T) {
 			}
 		}
 
-		ids := f.rows(0)
+		ids := f.rows(context.Background(), 0)
 		if !reflect.DeepEqual(expectedAll, ids) {
 			t.Fatalf("Do not match %v %v", expectedAll, ids)
 		}
 
-		ids = f.rows(0, filterColumn(1))
+		ids = f.rows(context.Background(), 0, filterColumn(1))
 		if !reflect.DeepEqual(expectedOdd, ids) {
 			t.Fatalf("Do not match %v %v", expectedOdd, ids)
 		}
@@ -2672,12 +2840,12 @@ func TestFragment_RowsIteration(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		ids := f.rows(0)
+		ids := f.rows(context.Background(), 0)
 		if !reflect.DeepEqual(expected, ids) {
 			t.Fatalf("Do not match %v %v", expected, ids)
 		}
 
-		ids = f.rows(0, filterColumn(66000))
+		ids = f.rows(context.Background(), 0, filterColumn(66000))
 		if !reflect.DeepEqual(expected, ids) {
 			t.Fatalf("Do not match %v %v", expected, ids)
 		}
@@ -2688,18 +2856,18 @@ func TestFragment_RowsIteration(t *testing.T) {
 		defer f.Clean(t)
 
 		expectedRows := make([]uint64, 0)
-		for r := uint64(1); r < uint64(10000); r += 100 {
+		for r := uint64(1); r < uint64(10000); r += 250 {
 			expectedRows = append(expectedRows, r)
-			for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 {
+			for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) {
 				if _, err := f.setBit(r, c); err != nil {
 					t.Fatal(err)
 				}
 
-				ids := f.rows(0)
+				ids := f.rows(context.Background(), 0)
 				if !reflect.DeepEqual(expectedRows, ids) {
 					t.Fatalf("Do not match %v %v", expectedRows, ids)
 				}
-				ids = f.rows(0, filterColumn(c))
+				ids = f.rows(context.Background(), 0, filterColumn(c))
 				if !reflect.DeepEqual(expectedRows, ids) {
 					t.Fatalf("Do not match %v %v", expectedRows, ids)
 				}
@@ -2923,7 +3091,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(false)
 		for i := uint64(0); i < 4; i++ {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i {
 				t.Fatalf("expected row %d but got %d", i, id)
 			}
@@ -2934,7 +3102,7 @@ func TestFragmentRowIterator(t *testing.T) {
 				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
 			}
 		}
-		row, id, wrapped := iter.Next()
+		row, id, _, wrapped := iter.Next()
 		if row != nil {
 			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
 		}
@@ -2956,7 +3124,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(false)
 		for i := uint64(1); i < 8; i += 2 {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i {
 				t.Fatalf("expected row %d but got %d", i, id)
 			}
@@ -2967,7 +3135,7 @@ func TestFragmentRowIterator(t *testing.T) {
 				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
 			}
 		}
-		row, id, wrapped := iter.Next()
+		row, id, _, wrapped := iter.Next()
 		if row != nil {
 			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
 		}
@@ -2989,7 +3157,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(true)
 		for i := uint64(0); i < 5; i++ {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i%4 {
 				t.Fatalf("expected row %d but got %d", i%4, id)
 			}
@@ -3014,7 +3182,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(true)
 		for i := uint64(1); i < 10; i += 2 {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i%8 {
 				t.Errorf("expected row %d but got %d", i%8, id)
 			}
@@ -3032,10 +3200,10 @@ func TestFragmentRowIterator(t *testing.T) {
 
 func TestUnionInPlaceMapped(t *testing.T) {
 	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	// note: clean has to be deferred first, because it has to run with
+	// the lock *not* held, because it is sometimes so it has to grab the
+	// lock...
 	defer f.Clean(t)
-	// I know this doesn't actually matter in our current context, but
-	// strictly speaking, we do say you have to hold the lock while calling
-	// unprotectedWriteToFragment...
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	r0 := rand.New(rand.NewSource(2))
@@ -3068,8 +3236,15 @@ func TestUnionInPlaceMapped(t *testing.T) {
 	f.storage.UnionInPlace(setBM1)
 	countUnion := f.storage.Count()
 	// UnionInPlace produces no ops log, we have to make it snapshot, to
-	// ensure that the on-disk representation is correct.
+	// ensure that the on-disk representation is correct. Note, UIP is
-	f.enqueueSnapshot()
+	// not used for things that are modifying real fragments, usually;
+	// it's used only in computation of things that usually don't go to
+	// disk, which is why we handle this specially in testing and not
+	// generically.
+	err = defaultSnapshotQueue.Immediate(f)
+	if err != nil {
+		t.Fatalf("snapshot after union-in-place: %v", err)
+	}
 
 	if count0 != countF {
 		t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF)
@@ -3178,6 +3353,25 @@ func TestFragmentPositionsForValue(t *testing.T) {
 	}
 }
 
+func TestIntLTRegression(t *testing.T) {
+	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	defer f.Clean(t)
+
+	_, err := f.setValue(1, 6, 33)
+	if err != nil {
+		t.Fatalf("setting value: %v", err)
+	}
+
+	row, err := f.rangeOp(pql.LT, 6, 33)
+	if err != nil {
+		t.Fatalf("doing range of: %v", err)
+	}
+
+	if !row.IsEmpty() {
+		t.Errorf("expected nothing, but got: %v", row.Columns())
+	}
+}
+
 func TestImportClearRestart(t *testing.T) {
 	tests := []struct {
 		rows []uint64
@@ -3200,7 +3394,6 @@ func TestImportClearRestart(t *testing.T) {
 			cols: []uint64{1, 1, 1, 1, 1, 1},
 		},
 	}
-
 	for i, test := range tests {
 		for _, maxOpN := range []int{0, 10000} {
 			t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) {
@@ -3253,11 +3446,11 @@ func TestImportClearRestart(t *testing.T) {
 
 				check(t, f, exp)
 
-				f2 := newFragment(f.path, "i", "f", viewStandard, 0, 0)
+				f2 := newFragment(NewHolder(DefaultPartitionN), f.path, "i", "f", viewStandard, 0, 0)
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
-				err = f.closeStorage(true)
+				err = f.closeStorage()
 				if err != nil {
 					t.Fatalf("closing storage: %v", err)
 				}
@@ -3287,11 +3480,11 @@ func TestImportClearRestart(t *testing.T) {
 
 				check(t, f2, exp)
 
-				f3 := newFragment(f2.path, "i", "f", viewStandard, 0, 0)
+				f3 := newFragment(NewHolder(DefaultPartitionN), f2.path, "i", "f", viewStandard, 0, 0)
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
-				err = f2.closeStorage(true)
+				err = f2.closeStorage()
 				if err != nil {
 					t.Fatalf("f2 closing storage: %v", err)
 				}
@@ -3339,6 +3532,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) {
 
 func TestImportValueConcurrent(t *testing.T) {
 	f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+	defer f.Clean(t)
 	eg := &errgroup.Group{}
 	for i := 0; i < 4; i++ {
 		i := i
@@ -3363,7 +3557,7 @@ func TestImportMultipleValues(t *testing.T) {
 		cols      []uint64
 		vals      []int64
 		checkCols []uint64
-		checkVals []uint64
+		checkVals []int64
 		depth     uint
 	}{
 		{
@@ -3371,7 +3565,7 @@ func TestImportMultipleValues(t *testing.T) {
 			vals:      []int64{97, 100},
 			depth:     7,
 			checkCols: []uint64{0},
-			checkVals: []uint64{100},
+			checkVals: []int64{100},
 		},
 	}
 
@@ -3395,7 +3589,7 @@ func TestImportMultipleValues(t *testing.T) {
 					if !exists {
 						t.Errorf("column %d should exist", cc)
 					}
-					if n != 100 {
+					if n != cv {
 						t.Errorf("wrong value: %d is not %d", n, cv)
 					}
 				}
@@ -3405,6 +3599,66 @@ func TestImportMultipleValues(t *testing.T) {
 	}
 }
 
+func TestImportValueRowCache(t *testing.T) {
+	type testCase struct {
+		cols      []uint64
+		vals      []int64
+		checkCols []uint64
+		depth     uint
+	}
+	tests := []struct {
+		tc1 testCase
+		tc2 testCase
+	}{
+		{
+			tc1: testCase{
+				cols:      []uint64{2},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2},
+			},
+			tc2: testCase{
+				cols:      []uint64{1000},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2, 1000},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		for _, maxOpN := range []int{1, 10000} {
+			t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) {
+				f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+				f.MaxOpN = maxOpN
+				defer f.Clean(t)
+
+				// First import (tc1)
+				if err := f.importValue(test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc1.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns())
+				}
+
+				// Second import (tc2)
+				if err := f.importValue(test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc2.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns())
+				}
+			})
+		}
+	}
+}
+
 func TestFragmentConcurrentReadWrite(t *testing.T) {
 	f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
 	defer f.Clean(t)
@@ -3431,3 +3685,115 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 
 	t.Logf("%d", acc)
 }
+
+func TestRemapCache(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+	// request a panic that doesn't kill the program on fault
+	wouldFault := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(wouldFault)
+		if r := recover(); r != nil {
+			if err, ok := r.(error); ok {
+				// special case: if we caught a page fault, we diagnose that directly. sadly,
+				// we can't see the actual values that were used to generate this, probably.
+				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+					t.Fatalf("segfault trapped during remap test (expected failure mode)")
+				}
+			}
+			t.Fatalf("unexpected panic: %v", r)
+		}
+	}()
+	// create a container
+	_, err := f.storage.Add(65537)
+	if err != nil {
+		t.Fatalf("storage add: %v", err)
+	}
+	// cause the container to be mapped
+	err = f.Snapshot()
+	if err != nil {
+		t.Fatalf("storage snapshot: %v", err)
+	}
+	// freeze the row
+	_ = f.row(0)
+	// add a bit that isn't in that container, so that container doesn't
+	// change
+	_, err = f.storage.Add(2)
+	if err != nil {
+		t.Fatalf("storage add: %v", err)
+	}
+	// make the original container be the most recent, thus cached, container
+	_, err = f.bit(0, 65537)
+	if err != nil {
+		t.Fatalf("storage bit check: %v", err)
+	}
+	// force snapshot, remapping the containers
+	err = f.Snapshot()
+	if err != nil {
+		t.Fatalf("storage snapshot: %v", err)
+	}
+	// get rid of the old mapping
+	runtime.GC()
+	// try to read that container again
+	_, err = f.bit(0, 65537)
+	if err != nil {
+		t.Fatalf("storage bit check: %v", err)
+	}
+}
+
+func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+
+	// byShardWidth is a map of the same roaring (fragment) data generated
+	// with different shard widths.
+	// TODO: a better approach may be to generate this in the test based
+	// on shard width.
+	byShardWidth := make(map[uint64][]byte)
+	// row/col: 1/1
+	byShardWidth[1<<20] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
+	byShardWidth[1<<22] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
+
+	var b []byte
+	if data, ok := byShardWidth[ShardWidth]; ok {
+		b = data
+	}
+
+	defer f.Clean(t)
+	err := f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check the bit
+	res := f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("expecting 1 got: %v", res)
+	}
+	//clear the bit
+	changed, _ := f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+	// import again
+	err = f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check
+	res = f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("again expecting 1 got: %v", res)
+	}
+	changed, _ = f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("again expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+}

gendebug_test.go

@@ -0,0 +1,39 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func examineResults() {
+	results := reportGenerations()
+	if len(results) > 0 {
+		fmt.Printf("generations:\n")
+		for _, res := range results {
+			fmt.Printf("  %s\n", res)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	ret := m.Run()
+	examineResults()
+	os.Exit(ret)
+}

generation.go

@@ -0,0 +1,440 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pkg/errors"
+)
+
+// generation represents one "generation" of opening a data file.
+// This is what determines when it's safe to unmap a data file, if it
+// got mapped, and handles closing/reopening files if we need to
+// manage file handle availability. It's an interface because this
+// lets us write simpler code for specific cases, rather than handling
+// the whole matrix of mapped/unmapped, staying open/being reopened,
+// etcetera.
+//
+// You create a generation by calling newGeneration with a file
+// path. If it succeeds in opening that path, it calls a provided
+// setup function with the data from the generation, and a flag
+// indicating whether the data is mmapped. If the setup function
+// fails, newGeneration cleans things up and closes. Otherwise,
+// it returns a generation.
+//
+// The generation itself uses runtime.SetFinalizer to clean up when
+// the last reference to it goes away. You should store a pointer
+// to the generation in any object which is reliant on the generation.
+//
+// When you anticipate a generation should be done (for instance,
+// opening a new generation), the old one gets marked done, which
+// stashes a timestamp in it. Later operations can check whether
+// the timestamp is a while back, and if so, complain that something
+// might be wrong.
+//
+// In some cases, we don't have enough open file limit to keep every
+// file actually open. To address this, use the `Transaction` function,
+// which ensures that the file is open, stores a reference to it in
+// a provided `*io.Writer`, and then restores the previous value of
+// the io.Writer when it's done. For instance, for a bitmap, this might
+// be used with `&b.OpWriter`.
+//
+// newGeneration takes an optional previous generation; it calls
+// that generation's Done function after running the provided setup,
+// and bumps the generation count.
+type generation interface {
+	// Transaction runs the given transaction with the generation's
+	// file open. If the **os.File parameter is
+	// non-nil, the generation's file will be open, and stored
+	// into that pointer, during the execution of func, after
+	// which the previous contents are restored. Otherwise
+	// the file may or may not be open during the operation.
+	Transaction(*io.Writer, func() error) error
+	// Done() should be called exactly once, to indicate that a
+	// generation is expected not to be in use for long -- for instance,
+	// when a new generation replaces it.
+	Done()
+	// Generation count.
+	Generation() int64
+	// ID indicates the source -- path and generation number -- that
+	// this generation represents.
+	ID() string
+	// Dead indicates whether this generation is Done.
+	Dead() bool
+	// Bytes reports the storage associated with this generation, if any.
+	// DO NOT USE THIS. Except if you're debugging mmap segfaults.
+	Bytes() []byte
+}
+
+type mmapGeneration struct {
+	mu         sync.Mutex // mutex guards modifiers of generation, not of data
+	transMu    sync.Mutex // guards transactions, specifically
+	path       string
+	id         string
+	file       *os.File
+	data       []byte
+	generation int64     // generation counter
+	dead       bool      // we think this generation is dead
+	deadSince  time.Time // when this generation was marked dead
+	retries    int       // for cases where we're retrying
+	logger     logger.Logger
+}
+
+func (m *mmapGeneration) Dead() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.dead
+}
+
+func (m *mmapGeneration) ID() string {
+	return m.id
+}
+
+func (m *mmapGeneration) Generation() int64 {
+	return m.generation
+}
+
+// Transaction runs an exclusive call, ensuring that the file is open if
+// the *io.Writer parameter is present.
+func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
+	m.transMu.Lock()
+	defer m.transMu.Unlock()
+	// HEY LOOK CAREFULLY AT THIS BIT:
+	// We can't just defer this unlock. We specifically want to be
+	// sure to unlock the regular mutex *before* this function is over,
+	// and if we error out trying to open the file, we want to do it
+	// even sooner. If we deferred this, the transaction would block
+	// *everything*, including things like sanity checks against the
+	// generation being Dead(), but also including the deferred
+	// re-close-the-file.
+	m.mu.Lock()
+	// if we've been asked for a file pointer, we need to ensure that
+	// our file is open, and that the file pointer to it is stored in
+	// the requested location, then revert that when we're done.
+	// if we aren't asked for a file pointer, nothing needs the file
+	// open.
+	if m.dead {
+		elapsed := time.Since(m.deadSince)
+		m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
+	}
+	if fileP != nil {
+		if m.file == nil {
+			// we ignore the shouldClose response here; if this
+			// fragment was previously not being kept open, we're
+			// going to stick with that.
+			_, err := m.openFile()
+			if err != nil {
+				m.mu.Unlock()
+				return err
+			}
+			defer func() {
+				// report a close error if we have no other error to report
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				err := m.closeFile()
+				if transactionErr == nil {
+					transactionErr = err
+				}
+			}()
+		}
+		var fileStash io.Writer
+		fileStash, *fileP = *fileP, m.file
+		defer func() {
+			*fileP = fileStash
+		}()
+	}
+	// We are done locking the generation itself for now.
+	m.mu.Unlock()
+	wouldPanic := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(wouldPanic)
+		if r := recover(); r != nil {
+			if err, ok := r.(error); ok {
+				// special case: if we caught a page fault, we diagnose that directly. sadly,
+				// we can't see the actual values that were used to generate this, probably.
+				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+					if transactionErr == nil {
+						transactionErr = errors.New("invalid memory access during transaction")
+					} else {
+						transactionErr = fmt.Errorf("invalid memory access during transaction, previous error %v", transactionErr)
+					}
+					return
+				}
+			}
+			if transactionErr == nil {
+				transactionErr = fmt.Errorf("panic during transaction: %v", r)
+			} else {
+				transactionErr = fmt.Errorf("panic during erroring transaction: panic %v, previous error %v", r, transactionErr)
+			}
+		}
+	}()
+	return fn()
+}
+
+func (m *mmapGeneration) Bytes() []byte {
+	return m.data
+}
+
+// Done marks the generation done, and closes its file, but may not unmap it.
+// It's still conceptually possible to end up doing a Transaction against a
+// done generation, but it's a red flag.
+func (m *mmapGeneration) Done() {
+	if m == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.dead {
+		oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
+			m.id, time.Now(), m.deadSince)
+		panic(oops)
+	}
+	m.dead = true
+	m.deadSince = time.Now()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error closing generation %s: %v", m.id, err)
+	}
+	// If we're not debugging, the finalizer won't have been enabled
+	// previously. Finalizers have non-zero cost, so having them not be
+	// created until they're needed seems rewarding?
+	if !generationDebug {
+		runtime.SetFinalizer(m, generationFinalizer)
+	}
+	endGeneration(m.id)
+	// note, Done() doesn't close the file; only the finalizer actually
+	// does the shutdown.
+}
+
+// Try to close the file if it's currently open.
+func (m *mmapGeneration) closeFile() error {
+	var lastErr error
+	// report the most serious error encountered, but still close
+	// file even if something else failed.
+	if m.file != nil {
+		if err := m.file.Sync(); err != nil {
+			lastErr = fmt.Errorf("sync: %s", err)
+		}
+		if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
+			lastErr = fmt.Errorf("unlock: %s", err)
+		}
+		if err := syswrap.CloseFile(m.file); err != nil {
+			lastErr = fmt.Errorf("close file: %s", err)
+		}
+		m.file = nil
+	}
+	return lastErr
+}
+
+// openFile ensures the file is open and locked, or fails. If it does
+// open the file, it will also report the "you need to close this file
+// when you're done" flag from syswrap.
+func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
+	if m.file != nil {
+		return false, nil
+	}
+	m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return false, err
+	}
+	// do we actually want this in every openFile? I don't know.
+	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		_ = syswrap.CloseFile(m.file)
+		m.file = nil
+		return false, fmt.Errorf("flock: %s", err)
+	}
+	return shouldClose, nil
+}
+
+func generationFinalizer(m *mmapGeneration) {
+	m.mu.Lock()
+	if !m.dead {
+		m.logger.Printf("finalizing generation %s which isn't dead yet\n",
+			m.id)
+	}
+	m.mu.Unlock()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("finalizing generation, closing file: %v\n", err)
+	}
+	if m.data != nil {
+		err := syswrap.Munmap(m.data)
+		if err != nil {
+			m.logger.Printf("finalizing generation, munmap: %v\n", err)
+		}
+		m.data = nil
+	}
+	finalizeGeneration(m.id)
+}
+
+// Cancel closes a generation out entirely. It cancels any finalizer,
+// unmaps any data, ends generation tracking, and closes any files.
+// It does each of these separately whether or not the others need to be done,
+// or succeed. It's used to handle failures from newGeneration; it makes sure
+// the generation isn't holding any resources and doesn't need to be cleaned
+// up otherwise.
+//
+// Mostly a helper function because there's several cases where newGeneration
+// might fail.
+func (m *mmapGeneration) Cancel() {
+	if m.data != nil {
+		_ = syswrap.Munmap(m.data)
+		m.data = nil
+	}
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error cancelling generation %s: %v", m.id, err)
+	}
+	runtime.SetFinalizer(m, nil)
+	m.dead = true
+	m.deadSince = time.Now()
+	cancelGeneration(m.id)
+}
+
+// newGeneration creates a new generation using the given file path. It
+// then calls the provided setup function with the allocated storage, a
+// file handle, the new generation, and a flag indicatting whether the storage
+// is memory-mapped. If the setup function returns a non-nil error, the
+// generation is cleaned up, and newGeneration fails. The setup function
+// also returns a boolean indicating whether it used the mapping; if it
+// didn't, newGeneration discards the mapping and returns a nil generation.
+//
+// If generationDebug is enabled, we track the generation even if no mapping
+// is actually in use, so we can verify that the tracking is working.
+//
+// On failure, newGeneration returns nil values for generation and func,
+// and an error. On success, the func returned is the close func to use
+// when the generation is no longer needed by the caller.
+func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
+	m := mmapGeneration{path: path, logger: logger}
+	if existing != nil {
+		m.generation = existing.Generation() + 1
+		m.retries = existing.(*mmapGeneration).retries
+		// we might keep a previous generation around just for its generation count.
+		if !existing.Dead() {
+			defer existing.Done()
+		}
+	}
+	shouldClose, err := m.openFile()
+	if err != nil {
+		return nil, err
+	}
+	m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
+	// possibly assign new generation ID if this one's been used, which can
+	// happen with reopens, especially during testing.
+	m.id = registerGeneration(m.id)
+	// if debugging, we always want the finalizer on so we notice if a
+	// generation is finalized without being closed. for non-debugging
+	// use, we only need it when the generation is closed.
+	if generationDebug {
+		runtime.SetFinalizer(&m, generationFinalizer)
+	}
+	// Mmap the underlying file so it can be zero copied.
+	var mapped bool
+	var data []byte
+	fi, err := m.file.Stat()
+	if err == nil && fi.Size() > 0 {
+		data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+		if err == syswrap.ErrMaxMapCountReached {
+			// I have no idea where/how to display this message.
+			m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
+		} else if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "mmap failed")
+		} else {
+			mapped = true
+		}
+	}
+	if data == nil && readData {
+		data, err = ioutil.ReadAll(m.file)
+		if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "failure file readall")
+		}
+	}
+	// if we got here, data's the expected data, so let's try to use it
+	mappedAny, err := setup(data, m.file, &m, mapped)
+
+	// if the setup failed, we unmap data if we previously mapped it,
+	// and exit. Note that having no data, or having only trivial
+	// data (like a zero-container Roaring file) isn't "failed".
+	if err != nil {
+		m.Cancel()
+		// Unless, that is, we think the file probably ought to
+		// be truncated: For instance, if a bitmap has a corrupted
+		// ops log, we could truncate that part of it and retry.
+		if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
+			m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
+			// close this generation, then try again. once.
+			m.retries++
+			err := os.Truncate(m.path, err.SuggestedLength())
+			if err != nil {
+				m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
+			}
+			return newGeneration(&m, path, readData, setup, logger)
+		}
+		return nil, err
+	}
+
+	if mapped {
+		// when generationDebug is on, we want to track this even
+		// if it's not being used.
+		if generationDebug || mappedAny {
+			// Advise the kernel that the mmap is accessed randomly.
+			// We don't care much about errors with this.
+			_ = madvise(data, syscall.MADV_RANDOM)
+			// store the data, so we can unmap it when this generation
+			// gets finalized.
+			m.data = data
+		} else {
+			// unmap the data and don't stash the pointer in this
+			// generation. It's not being used. This generation
+			// doesn't need to exist, yay.
+			unmapErr := syswrap.Munmap(data)
+			if unmapErr != nil {
+				m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
+			}
+		}
+	}
+	// shouldClose comes from underlying syswrap.OpenFile, which checks
+	// a count of open files to hint at us when we need to start closing
+	// files to preserve open file descriptor limit.
+	if shouldClose {
+		err := m.closeFile()
+		if err != nil {
+			m.logger.Printf("closing file to preserve open files failed: %v\n", err)
+		}
+	}
+	// It's possible that the generation has no actual data to track,
+	// because nothing's mapped, in which case there won't be any bitmap
+	// sources following this, just the fragment source. (Bitmaps won't
+	// be attached to the source unless they're actually mapped to it,
+	// or generationDebug is true). That's okay. We pay a tiny cost
+	// for the finalizer, but we also get higher confidence that it really
+	// does get cleaned up.
+	return &m, nil
+}

generation_debug.go

@@ -0,0 +1,160 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+const generationDebug = true
+
+type lifespan struct {
+	from, to, finalized time.Time
+}
+
+var knownGenerations map[string]lifespan
+var knownGenerationLock sync.Mutex
+
+var timeZero time.Time
+
+func registerGeneration(id string) string {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	if knownGenerations == nil {
+		knownGenerations = make(map[string]lifespan)
+	}
+	newSpan := lifespan{from: time.Now()}
+	origId := id
+
+	// if you have more than 65k of the same file open, maybe you have bigger
+	// problems than this.
+	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
+		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
+		if span.finalized != timeZero {
+			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
+				id, suffix, span.from, span.to, span.finalized)
+		} else {
+			if span.to != timeZero {
+				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
+			} else {
+				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
+			}
+		}
+		id = origId + suffix
+	}
+	fmt.Printf("new generation %s\n", id)
+	knownGenerations[id] = newSpan
+	return id
+}
+
+func endGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("ending generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero || span.to != timeZero {
+		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
+		panic(oops)
+	}
+	span.to = time.Now()
+	knownGenerations[id] = span
+}
+
+// cancelGeneration marks the generation as finalized. In principle it's
+// only used in cases where we just started a generation but something
+// went wrong. it's not fancier than this because of the weird cases
+// where the same generation shows up again, such as when closing and
+// reopening an index so we don't know about previous instances of the
+// same files.
+func cancelGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if exists {
+		span.finalized = time.Now()
+		span.to = span.finalized
+		knownGenerations[id] = span
+	}
+}
+
+func finalizeGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("finalizing generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero {
+		var oops string
+		if span.to != timeZero {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
+		} else {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
+		}
+		panic(oops)
+	}
+	span.finalized = time.Now()
+	knownGenerations[id] = span
+}
+
+func reportGenerations() []string {
+	runtime.GC()
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	var surviving []string
+	times := make([]int64, 0, len(knownGenerations))
+	for id, span := range knownGenerations {
+		if span.to == timeZero {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
+			} else {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
+			}
+		} else {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
+			} else {
+				times = append(times, int64(span.finalized.Sub(span.to)))
+			}
+		}
+	}
+	if len(times) > 0 {
+		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
+		var total int64
+		for _, d := range times {
+			total += d
+		}
+		var mean, median, p90, p99, worst int64
+		mean = total / int64(len(times))
+		median = times[len(times)/2]
+		p90 = times[(len(times)*9)/10]
+		p99 = times[(len(times)*99)/100]
+		worst = times[len(times)-1]
+		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+	}
+	return surviving
+}

generation_nodebug.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package pilosa
+
+const generationDebug = false
+
+func registerGeneration(id string) string {
+	return id
+}
+
+func endGeneration(id string) {
+}
+
+func cancelGeneration(id string) {
+}
+
+func finalizeGeneration(id string) {
+}
+
+//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
+func reportGenerations() []string { //nolint:unused,deadcode
+	return nil
+}

generation_test.go

@@ -0,0 +1,67 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationparanoia
+
+package pilosa
+
+import (
+	"runtime"
+	"testing"
+	"unsafe"
+)
+
+func TestGenerationPanic(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "none")
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	_ = f.Snapshot()
+	_ = f.row(0)
+	var prevData []byte
+
+	if f.gen.(*mmapGeneration).data == nil {
+		t.Fatalf("generation code didn't create a mapping, apparently?")
+	}
+	prevData = f.gen.(*mmapGeneration).data
+	f.mu.Lock()
+	_ = defaultSnapshotQueue.Immediate(f)
+	f.mu.Unlock()
+	runtime.GC()
+	for i := 0; i < (f.MaxOpN / 2); i++ {
+		_, _ = f.setBit(0, uint64(i*32)+23)
+	}
+	f.mu.Lock()
+	defaultSnapshotQueue.Await(f)
+	f.mu.Unlock()
+	runtime.GC()
+	newData := f.gen.(*mmapGeneration).data
+	if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
+		t.Fatalf("test can't run usefully, didn't get new data pointer")
+	}
+
+	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+		prevData[0] = 0x3c
+		return nil
+	})
+	if err == nil {
+		t.Fatalf("expected a panic to get caught, but nothing happened")
+	}
+	if err.Error() != "invalid memory access during transaction" {
+		t.Fatalf("expected \"invalid memory access during transaction\", got %q", err.Error())
+	}
+}

generator/slice.go

@@ -0,0 +1,51 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package generator
+
+import (
+	"math/rand"
+	"sort"
+)
+
+// Uint64Slice generates between [0, n) random uint64 numbers between min and max.
+func Uint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 {
+	a := make([]uint64, rand.Intn(n))
+	for i := range a {
+		a[i] = min + uint64(rand.Int63n(int64(max-min)))
+	}
+
+	if sorted {
+		sort.Sort(uint64Slice(a))
+	}
+
+	return a
+}
+
+// Uint64SetSlice returns the values in a uint64 set.
+func Uint64SetSlice(m map[uint64]struct{}) []uint64 {
+	a := make([]uint64, 0, len(m))
+	for v := range m {
+		a = append(a, v)
+	}
+	sort.Sort(uint64Slice(a))
+	return a
+}
+
+// uint64Slice represents a sortable slice of uint64 numbers.
+type uint64Slice []uint64
+
+func (u uint64Slice) Swap(i, j int)      { u[i], u[j] = u[j], u[i] }
+func (u uint64Slice) Len() int           { return len(u) }
+func (u uint64Slice) Less(i, j int) bool { return u[i] < u[j] }

go.mod

@@ -3,7 +3,6 @@ module github.com/pilosa/pilosa/v2
 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -13,17 +12,19 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
-	github.com/golang/protobuf v1.3.1
+	github.com/golang/protobuf v1.3.3
 	github.com/google/go-cmp v0.2.0
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
+	github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.18.12+incompatible
@@ -40,8 +41,9 @@ require (
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
 	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/grpc v1.28.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 )
 
-go 1.11
+go 1.13

go.sum

@@ -1,3 +1,4 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
@@ -18,8 +19,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -28,6 +32,9 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -39,10 +46,17 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -80,6 +94,14 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 h1:XOImsA5XhGklFj8Y0TxSm1qWZzEwYxom2JOXiu9GMq0=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@@ -101,6 +123,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -153,18 +177,29 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -180,12 +215,29 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=

gopsutil/systeminfo.go

@@ -15,6 +15,7 @@
 package gopsutil
 
 import (
+	"math"
 	"runtime"
 	"strings"
 
@@ -114,6 +115,13 @@ func (s *systemInfo) collectPlatformInfo() error {
 		}
 		s.cpuModel = infos[0].ModelName
 		s.cpuMHz = computeMHz(s.cpuModel)
+		if s.cpuMHz < 0 {
+			s.cpuMHz = int(math.Round(infos[0].Mhz))
+		}
+		if s.cpuMHz < 0 {
+			// This is supposed to be unsigned.
+			s.cpuMHz = 0
+		}
 
 		// gopsutil reports core and clock speed info inconsistently
 		// by OS

gossip/gossip.go

@@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte {
 		Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
 	}
 	for _, idx := range m.Schema.Indexes {
-		is := &pilosa.IndexStatus{Name: idx.Name}
+		is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
+
 		for _, f := range idx.Fields {
 			availableShards := roaring.NewBitmap()
 			if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
 				availableShards = field.AvailableShards()
 			}
-			is.Fields = append(is.Fields, &pilosa.FieldStatus{
+
+			fs := &pilosa.FieldStatus{
 				Name:            f.Name,
+				CreatedAt:       f.CreatedAt,
 				AvailableShards: availableShards,
-			})
+			}
+			is.Fields = append(is.Fields, fs)
 		}
 		m.Indexes = append(m.Indexes, is)
 	}

handler.go

@@ -16,6 +16,9 @@ package pilosa
 
 import (
 	"encoding/json"
+
+	"github.com/pilosa/pilosa/v2/tracing"
+	"github.com/pkg/errors"
 )
 
 // QueryRequest represent a request to process a query.
@@ -42,12 +45,21 @@ type QueryRequest struct {
 	// If true, indicates that query is part of a larger distributed query.
 	// If false, this request is on the originating node.
 	Remote bool
+
+	// Should we profile this query?
+	Profile bool
+
+	// Additional data associated with the query, in cases where there's
+	// row-style inputs for precomputed values.
+	EmbeddedData []*Row
 }
 
 // QueryResponse represent a response from a processed query.
 type QueryResponse struct {
 	// Result for each top-level query call.
-	// Can be a Bitmap, Pairs, or uint64.
+	// The result type differs depending on the query; types
+	// include: Row, RowIdentifiers, GroupCounts, SignedRow,
+	// ValCount, Pair, Pairs, bool, uint64.
 	Results []interface{}
 
 	// Set of column attribute objects matching IDs returned in Result.
@@ -55,6 +67,9 @@ type QueryResponse struct {
 
 	// Error during parsing or execution.
 	Err error
+
+	// Profiling data, if any
+	Profile *tracing.Profile
 }
 
 // MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
@@ -68,9 +83,11 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(struct {
 		Results        []interface{}    `json:"results"`
 		ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
+		Profile        *tracing.Profile `json:"profile,omitempty"`
 	}{
 		Results:        resp.Results,
 		ColumnAttrSets: resp.ColumnAttrSets,
+		Profile:        resp.Profile,
 	})
 }
 
@@ -98,18 +115,84 @@ var NopHandler Handler = nopHandler{}
 // for a value (BSI) import.
 type ImportValueRequest struct {
 	Index          string
+	IndexCreatedAt int64
 	Field          string
+	FieldCreatedAt int64
+	// if Shard is MaxUint64 (an impossible shard value), this
+	// indicates that the column IDs may come from multiple shards.
 	Shard        uint64
 	ColumnIDs    []uint64
 	ColumnKeys   []string
 	Values       []int64
+	FloatValues  []float64
+	StringValues []string
+}
+
+func (ivr *ImportValueRequest) Len() int           { return len(ivr.ColumnIDs) }
+func (ivr *ImportValueRequest) Less(i, j int) bool { return ivr.ColumnIDs[i] < ivr.ColumnIDs[j] }
+func (ivr *ImportValueRequest) Swap(i, j int) {
+	ivr.ColumnIDs[i], ivr.ColumnIDs[j] = ivr.ColumnIDs[j], ivr.ColumnIDs[i]
+	if len(ivr.Values) > 0 {
+		ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
+	} else if len(ivr.FloatValues) > 0 {
+		ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
+	} else if len(ivr.StringValues) > 0 {
+		ivr.StringValues[i], ivr.StringValues[j] = ivr.StringValues[j], ivr.StringValues[i]
+	}
+}
+
+// Validate ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) Validate() error {
+	return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt)
+}
+
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if ivr.Index == "" || ivr.Field == "" {
+		return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
+	}
+	if len(ivr.ColumnIDs) != 0 && len(ivr.ColumnKeys) != 0 {
+		return errors.Errorf("must pass either column ids or keys, but not both")
+	}
+	var valueSetCount int
+	if len(ivr.Values) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.FloatValues) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.StringValues) != 0 {
+		valueSetCount++
+	}
+	if valueSetCount > 1 {
+		return errors.Errorf("must pass ints, floats, or strings but not multiple")
+	}
+	if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 {
+		if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
+}
+
+// ImportColumnAttrsRequest describes the import request structure
+// for a ColumnAttr import
+type ImportColumnAttrsRequest struct {
+	AttrKey        string
+	ColumnIDs      []uint64
+	AttrVals       []string
+	Shard          int64
+	Index          string
+	IndexCreatedAt int64
 }
 
 // ImportRequest describes the import request structure
 // for an import.
 type ImportRequest struct {
 	Index          string
+	IndexCreatedAt int64
 	Field          string
+	FieldCreatedAt int64
 	Shard          uint64
 	RowIDs         []uint64
 	ColumnIDs      []uint64
@@ -118,13 +201,43 @@ type ImportRequest struct {
 	Timestamps     []int64
 }
 
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
+		if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
+}
+
+const (
+	RequestActionSet       = "set"
+	RequestActionClear     = "clear"
+	RequestActionOverwrite = "overwrite"
+)
+
 // ImportRoaringRequest describes the import request structure
 // for an import containing roaring-encoded data.
 type ImportRoaringRequest struct {
+	IndexCreatedAt int64
+	FieldCreatedAt int64
 	Clear          bool
+	Action         string // [set, clear, overwrite]
+	Block          int
 	Views          map[string][]byte
 }
 
+// ValidateWithTimestamp ensures that the payload of the request is valid.
+func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
+	if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 {
+		if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt {
+			return ErrPreconditionFailed
+		}
+	}
+	return nil
+}
+
 // ImportResponse is the structured response of an import.
 type ImportResponse struct {
 	Err string
@@ -160,3 +273,45 @@ type TranslateKeysRequest struct {
 type TranslateKeysResponse struct {
 	IDs []uint64
 }
+
+// TranslateIDsRequest describes the structure of a request
+// for a batch of id translations.
+type TranslateIDsRequest struct {
+	Index string
+	Field string
+	IDs   []uint64
+}
+
+// TranslateIDsResponse is the structured response of a id
+// translation request.
+type TranslateIDsResponse struct {
+	Keys []string
+}
+
+// InspectRequestParams represents the parts of an InspectRequest that
+// aren't generic holder filtering attributes.
+type InspectRequestParams struct {
+	Containers bool // include container details
+	Checksum   bool // perform checksums
+}
+
+// InspectRequest represents a request for a possibly-partial
+// holder inspection, using a provided holder filter and inspect-specific
+// parameters.
+type InspectRequest struct {
+	HolderFilterParams
+	InspectRequestParams
+}
+
+// InspectResponse contains the structured results for an InspectRequest.
+// It may some day be expanded to include metadata about views or indexes.
+type InspectResponse struct {
+	Fragments []struct {
+		Index string
+		Field string
+		View  string
+		Shard int64
+		Path  string
+		Info  *FragmentInfo
+	}
+}

holder_internal_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Pilosa Corp.
+// Copyright 2020 Pilosa Corp.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,309 +15,163 @@
 package pilosa
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
 	"testing"
-
-	"github.com/pilosa/pilosa/v2/roaring"
 )
 
-type tHolder struct {
+type testHolderOperator struct {
-	*Holder
+	indexSeen, indexProcessed       int
+	fieldSeen, fieldProcessed       int
+	viewSeen, viewProcessed         int
+	fragmentSeen, fragmentProcessed int
+	waitHere                        chan struct{}
 }
 
-// Close closes the holder and removes all underlying data.
+func (t *testHolderOperator) CheckIndex(string) (bool, bool) {
-func (h *tHolder) Close() error {
+	t.indexSeen++
-	defer os.RemoveAll(h.Path)
+	return true, true
-	return h.Holder.Close()
 }
 
-// Reopen instantiates and opens a new holder.
+func (t *testHolderOperator) CheckField(string, string) (bool, bool) {
-// Note that the holder must be Closed first.
+	t.fieldSeen++
-func (h *tHolder) Reopen() error {
+	return true, true
-	path, logger := h.Path, h.Holder.Logger
-	h.Holder = NewHolder()
-	h.Holder.Path = path
-	h.Holder.Logger = logger
-	return h.Holder.Open()
 }
 
-func newHolder() *tHolder {
+func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-")
+	t.viewSeen++
+	return true, true
+}
+
+func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool {
+	t.fragmentSeen++
+	return true
+}
+
+func (t *testHolderOperator) ProcessIndex(*Index) error {
+	t.indexProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessField(*Field) error {
+	t.fieldProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessView(*view) error {
+	t.viewProcessed++
+	return nil
+}
+
+func (t *testHolderOperator) ProcessFragment(*fragment) error {
+	if t.waitHere != nil {
+		<-t.waitHere
+	}
+	t.fragmentProcessed++
+	return nil
+}
+
+func makeHolder() (*Holder, string, error) {
+	path, err := ioutil.TempDir("", "pilosa-")
 	if err != nil {
-		panic(err)
+		return nil, "", err
+	}
+	h := NewHolder(DefaultPartitionN)
+
+	return h, path, nil
 }
 
-	h := &tHolder{Holder: NewHolder()}
+func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
-	h.Path = path
+	idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
-	return h
-}
-
-// MustCreateFieldIfNotExists returns a given field. Panic on error.
-func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
-	f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
 	if err != nil {
-		panic(err)
+		t.Fatalf("creating index: %v", err)
 	}
-	return f
+	f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
-}
-
-// MustCreateIndexIfNotExists returns a given index. Panic on error.
-func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index {
-	idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
 	if err != nil {
-		panic(err)
+		t.Fatalf("setting bit: %v", err)
 	}
-	return idx
+	_, err = f.SetBit(rowID, columnID, nil)
-}
-
-// SetBit clears a bit on the given field.
-func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) {
-	f := h.MustCreateFieldIfNotExists(index, field)
-	_, err := f.SetBit(rowID, columnID, nil)
 	if err != nil {
-		panic(err)
+		t.Fatalf("setting bit: %v", err)
 	}
 }
 
-// Row returns a Row for a given field.
+func TestHolderOperatorProcess(t *testing.T) {
-func (h *tHolder) Row(index, field string, rowID uint64) *Row {
+	h, path, err := makeHolder()
-	f := h.MustCreateFieldIfNotExists(index, field)
-	row, err := f.Row(rowID)
 	if err != nil {
-		panic(err)
+		t.Fatalf("creating holder: %v", err)
 	}
-	return row
+	defer os.RemoveAll(path)
-}
-
-func TestHolder_Optn(t *testing.T) {
-	t.Run("ErrViewPermission", func(t *testing.T) {
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
-		}
-		h := newHolder()
 	defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
+	// Write bits to separate indexes.
-			t.Fatal(err)
+	testSetBit(t, h, "i0", "f", 100, 200)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
+	testSetBit(t, h, "i1", "f", 100, 200)
-			t.Fatal(err)
+	testSetBit(t, h, "i1", "f", 100, 12345678)
-		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
+
-			t.Fatal(err)
+	testOp := testHolderOperator{}
-		} else if err := h.Holder.Close(); err != nil {
+	ctx := context.Background()
-			t.Fatal(err)
+	err = h.Process(ctx, &testOp)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
+	if err != nil {
-			t.Fatal(err)
+		t.Fatalf("processing holder: %v", err)
 	}
-		defer func() {
+	expected := testHolderOperator{
-			// we don't care about a failure here
+		indexSeen: 2, indexProcessed: 2,
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755)
+		fieldSeen: 2, fieldProcessed: 2,
-		}()
+		viewSeen: 2, viewProcessed: 2,
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+		fragmentSeen: 3, fragmentProcessed: 3,
-			t.Fatalf("unexpected error: %s", err)
 	}
-	})
+	if testOp != expected {
-	t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
+		t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
-		if os.Geteuid() == 0 {
-			t.Skip("Skipping permissions test since user is root.")
 	}
-		h := newHolder()
+}
+
+func TestHolderOperatorCancel(t *testing.T) {
+	h, path, err := makeHolder()
+	if err != nil {
+		t.Fatalf("creating holder: %v", err)
+	}
+	defer os.RemoveAll(path)
 	defer h.Close()
 
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
+	// Write bits to separate indexes.
-			t.Fatal(err)
+	testSetBit(t, h, "i0", "f", 100, 200)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
+	testSetBit(t, h, "i1", "f", 100, 200)
-			t.Fatal(err)
+	testSetBit(t, h, "i1", "f", 100, 12345678)
-		} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
+
-			t.Fatal(err)
+	// Here, we want to ensure that the operation gets cancelled
-		} else if err := h.Holder.Close(); err != nil {
+	// successfully. In practice we expect it to process one fragment, then
-			t.Fatal(err)
+	// end up blocked on the waitHere, then get cancelled... But the
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
+	// waitHere blockage isn't really something holder.Process can do
-			t.Fatal(err)
+	// anything about, so we close the channel, so two fragments are
-		}
+	// processed. But in theory you could end up with only one fragment
-		defer func() {
+	// processed if this goroutine managed to cancel before the processor
-			// we don't care about a failure here
+	// gets to the next fragment. Point is, it shouldn't hit all three,
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755)
+	// because the checks against the cancellation should fire before it
+	// gets there.
+	testOp := testHolderOperator{waitHere: make(chan struct{})}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		err = h.Process(ctx, &testOp)
+		close(done)
 	}()
-
+	testOp.waitHere <- struct{}{}
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
+	cancel()
-			t.Fatalf("unexpected error: %s", err)
+	close(testOp.waitHere)
+	<-done
+	if err != context.Canceled {
+		t.Fatalf("processing holder: expected context.Canceled, got %v", err)
 	}
-	})
+	testOp.waitHere = nil
-
+	expected := testHolderOperator{
-	t.Run("ErrFragmentCachePermission", func(t *testing.T) {
+		indexSeen: 2, indexProcessed: 2,
-		if os.Geteuid() == 0 {
+		fieldSeen: 2, fieldProcessed: 2,
-			t.Skip("Skipping permissions test since user is root.")
+		viewSeen: 2, viewProcessed: 2,
+		fragmentSeen: 3, fragmentProcessed: 3,
 	}
-		h := newHolder()
+	if testOp == expected {
-		defer h.Close()
+		t.Fatalf("holder processor did not cancel. expected something other than %#v", expected)
-
-		if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
-			t.Fatal(err)
-		} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
-			t.Fatal(err)
-		} else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
-			t.Fatal(err)
-		} else if _, err := field.SetBit(0, 0, nil); err != nil {
-			t.Fatal(err)
-		} else if err := view.Fragment(0).FlushCache(); err != nil {
-			t.Fatal(err)
-		} else if err := h.Holder.Close(); err != nil {
-			t.Fatal(err)
-		} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
-			t.Fatal(err)
-		}
-		defer func() {
-			_ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644)
-		}()
-		if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	})
-
-}
-
-// Ensure holder can clean up orphaned fragments.
-func TestHolderCleaner_CleanHolder(t *testing.T) {
-	cluster := NewTestCluster(2)
-
-	// Create a local holder.
-	hldr0 := newHolder()
-	defer hldr0.Close()
-
-	// Mock 2-node, fully replicated cluster.
-	cluster.ReplicaN = 2
-
-	cluster.nodes[0].URI = NewTestURIFromHostPort("localhost", 0)
-
-	// Create fields on nodes.
-	for _, hldr := range []*tHolder{hldr0} {
-		hldr.MustCreateFieldIfNotExists("i", "f")
-		hldr.MustCreateFieldIfNotExists("i", "f0")
-		hldr.MustCreateFieldIfNotExists("y", "z")
-	}
-
-	// Set data on the local holder.
-	hldr0.SetBit("i", "f", 0, 10)
-	hldr0.SetBit("i", "f", 0, 4000)
-	hldr0.SetBit("i", "f", 2, 20)
-	hldr0.SetBit("i", "f", 3, 10)
-	hldr0.SetBit("i", "f", 120, 10)
-	hldr0.SetBit("i", "f", 200, 4)
-
-	hldr0.SetBit("i", "f0", 9, ShardWidth+5)
-
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
-	hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
-
-	// Set highest shard.
-	err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1))
-	if err != nil {
-		t.Fatalf("adding remote shards: %v", err)
-	}
-	err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2))
-	if err != nil {
-		t.Fatalf("adding remote shards: %v", err)
-	}
-
-	// Keep replication the same and ensure we get the expected results.
-	cluster.ReplicaN = 2
-
-	// Set up cleaner for replication 2.
-	cleaner2 := holderCleaner{
-		Node:    cluster.nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner2.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*tHolder{hldr0} {
-		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
-			t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
-		}
-
-		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
-	}
-
-	// Change replication factor to ensure we have fragments to remove.
-	cluster.ReplicaN = 1
-
-	// Set up cleaner for replication 1.
-	cleaner1 := holderCleaner{
-		Node:    cluster.nodes[0],
-		Holder:  hldr0.Holder,
-		Cluster: cluster,
-	}
-
-	if err := cleaner1.CleanHolder(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify data is the same on both nodes.
-	for i, hldr := range []*tHolder{hldr0} {
-		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
-			t.Fatalf("unexpected columns(%d/0): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
-			t.Fatalf("unexpected columns(%d/2): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/3): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
-			t.Fatalf("unexpected columns(%d/120): %+v", i, a)
-		} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
-			t.Fatalf("unexpected columns(%d/200): %+v", i, a)
-		}
-
-		f := hldr.fragment("i", "f0", viewStandard, 1)
-		if f != nil {
-			t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
-		}
-
-		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
-			t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
-		}
-	}
-}
-
-// Ensure holder can reopen.
-func TestHolderCleaner_Reopen(t *testing.T) {
-	h := NewHolder()
-	h.Path = "path"
-	err := h.Open()
-	if err != nil {
-		t.Fatalf("couldn't open holder: %v", err)
-	}
-	err = h.Close()
-	if err != nil {
-		t.Fatalf("couldn't close holder: %v", err)
-	}
-	err = h.Open()
-	if err != nil {
-		t.Fatalf("couldn't open holder: %v", err)
-	}
-	err = h.Close()
-	if err != nil {
-		t.Fatalf("couldn't close holder: %v", err)
 	}
 }

holder_test.go

@@ -17,6 +17,7 @@ package pilosa_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,7 +26,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 func TestHolder_Open(t *testing.T) {
@@ -197,7 +200,110 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
+	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		h := test.MustOpenHolder()
+		defer h.Close()
 
+		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := h.Reopen(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ForeignIndex", func(t *testing.T) {
+		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else {
+				_, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("nonexistent"))
+				if err == nil {
+					t.Fatalf("expected error: %s", pilosa.ErrForeignIndexNotFound)
+				} else if errors.Cause(err) != pilosa.ErrForeignIndexNotFound {
+					t.Fatalf("expected error: %s, but got: %s", pilosa.ErrForeignIndexNotFound, err)
+				}
+			}
+		})
+
+		// Foreign index zzz is opened after foo/bar.
+		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("zzz")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Foreign index aaa is opened before foo/bar.
+		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("aaa")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Try to re-create existing index
+		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err = h.CreateIndex("aaa", pilosa.IndexOptions{}); err == nil {
+				t.Fatalf("expected: ConflictError, got: nil")
+			} else if _, ok := err.(pilosa.ConflictError); !ok {
+				t.Fatalf("expected: ConflictError, got: %s", err)
+			}
+
+			idx2, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if idx1 != idx2 {
+				t.Fatalf("expected the same indexes, got: %s and %s", idx1.Name(), idx2.Name())
+			}
+		})
+	})
 }
 
 func TestHolder_HasData(t *testing.T) {
@@ -303,10 +409,14 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	if err != nil {
 		t.Fatalf("creating field f0: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field z in y: %v", err)
 	}
+	_, err = c[0].API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
+	if err != nil {
+		t.Fatalf("creating field b in y: %v", err)
+	}
 
 	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
 	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
@@ -321,6 +431,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 
 	// Set a bit to create the fragment.
 	hldr0.SetBit("y", "z", 0, 0)
+	hldr0.SetBit("y", "b", 0, 0) // rowID = 0 means false
 
 	// Set data on the remote holder.
 	hldr1.SetBit("i", "f", 0, 4000)
@@ -331,6 +442,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
 
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+4) // true
+	hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
+
 	err = c[0].Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
@@ -365,6 +480,119 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
 			t.Errorf("unexpected columns(%d/y/z): %+v", i, a)
 		}
+
+		if a := hldr.Row("y", "b", 0).Columns(); !reflect.DeepEqual(a, []uint64{0, (3 * ShardWidth) + 5}) {
+			t.Errorf("unexpected false columns(%d/y/b): %+v", i, a)
+		}
+		if a := hldr.Row("y", "b", 1).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 7}) {
+			t.Errorf("unexpected true columns(%d/y/b): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder can sync with a remote holder and respects
+// the row boundaries of the block.
+func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c[0].Config.Cluster.ReplicaN = 3
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 3
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	blockEdge := uint64(pilosa.HashBlockSize)
+
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+
+	// Set data on the local holder.
+	hldr0.SetBit("i", "f", blockEdge-1, 10)
+	hldr0.SetBit("i", "f", blockEdge, 20)
+
+	// Set the same data on one of the replicas
+	// so that we have a quorum.
+	hldr1.SetBit("i", "f", blockEdge-1, 10)
+	hldr1.SetBit("i", "f", blockEdge, 20)
+
+	// Leave the third replica empty to force a block merge.
+	//
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", blockEdge-1).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Errorf("unexpected columns(%d/block 0): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", blockEdge).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d/block 1): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder correctly handles clears during block sync.
+func TestHolderSyncer_Clears(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c[0].Config.Cluster.ReplicaN = 3
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 3
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+
+	// Set data on the local holder that should be cleared
+	// because it's the only instance of this value.
+	hldr0.SetBit("i", "f", 0, 30)
+
+	// Set similar data on the replicas, but
+	// different from what's on local. This should end
+	// up being set on all replicas
+	hldr1.SetBit("i", "f", 0, 20)
+	hldr2.SetBit("i", "f", 0, 20)
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d): %+v", i, a)
+		}
 	}
 }
 
@@ -419,3 +647,112 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 		}
 	}
 }
+
+// Ensure holder can sync integer views with a remote holder.
+func TestHolderSyncer_IntField(t *testing.T) {
+	t.Run("BasicSync", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c[0].Config.Cluster.ReplicaN = 2
+		c[0].Config.AntiEntropy.Interval = 0
+		c[1].Config.Cluster.ReplicaN = 2
+		c[1].Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+
+		// Set data on the local holder for node0.
+		hldr0.SetValue("i", "f", 1, 1)
+
+		// Set data on node1.
+		hldr1.SetValue("i", "f", 2, 2)
+
+		err = c[0].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
+				t.Errorf("unexpected value(node%d/0): %d, exists: %v", i, a, exists)
+			}
+			if a, exists := hldr.Value("i", "f", 2); exists {
+				t.Errorf("unexpected value(node%d/1): %d, exists: %v", i, a, exists)
+			}
+		}
+	})
+
+	t.Run("MultiShard", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c[0].Config.Cluster.ReplicaN = 2
+		c[0].Config.AntiEntropy.Interval = 0
+		c[1].Config.Cluster.ReplicaN = 2
+		c[1].Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+
+		// Set data on the local holder for node0.
+		hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
+		hldr0.SetValue("i", "f", 3*pilosa.ShardWidth, 32)
+		hldr0.SetValue("i", "f", 4*pilosa.ShardWidth, math.MinInt32)
+		hldr0.SetValue("i", "f", 7*pilosa.ShardWidth, math.MinInt32)
+
+		// Set data on node1.
+		hldr1.SetValue("i", "f", 0*pilosa.ShardWidth, 2)
+		hldr1.SetValue("i", "f", 2*pilosa.ShardWidth, 22)
+		hldr1.SetValue("i", "f", 4*pilosa.ShardWidth, math.MaxInt32)
+		hldr1.SetValue("i", "f", 7*pilosa.ShardWidth, math.MaxInt32)
+
+		// Primary for shards (for index "i"):
+		// node0: [0,3,7]
+		// node1: [1,2,4]
+
+		err = c[0].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+		err = c[1].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 1: %v", err)
+		}
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a := hldr.Range("i", "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+			if a := hldr.Range("i", "f", pql.LT, 0); !reflect.DeepEqual(a.Columns(), []uint64{7 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+		}
+	})
+}

http/client.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/encoding/proto"
@@ -290,6 +291,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -392,6 +394,13 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits
 
 	// Get the coordinator node; all bits are sent to the
 	// primary translate store (i.e. coordinator).
+	// TODO... is that right^^?
+	// RESPONSE: It looks like in ctl/import.go, we could change the
+	// logic in ImportCommand.importBits() to only use ImportK
+	// when useRowKeys = true. It's no longer necessary to
+	// send column key translations to the coordinator (although
+	// it should still work). As far as I know, the only thing
+	// that uses ImportK is the pilosa import sub-command.
 	nodes, err := c.Nodes(ctx)
 	if err != nil {
 		return fmt.Errorf("getting nodes: %s", err)
@@ -487,6 +496,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -553,6 +563,35 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 	return nil
 }
 
+// ImportValue2 is a simplified ImportValue method which just uses the
+// ImportValueRequest instead of splitting up ImportValue and
+// ImportValueK... it also supports importing float values. The idea
+// being that (assuming it works) this will become the default (and be
+// renamed) for 2.0, and we can deprecate the other methods.
+func (c *InternalClient) ImportValue2(ctx context.Context, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.NewImportValue")
+	defer span.Finish()
+
+	buf, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Errorf("marshal import request: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, req.Index, req.Shard)
+	if err != nil {
+		return errors.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, req.Index, req.Field, buf, options); err != nil {
+			return errors.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+	return nil
+}
+
 // ImportValueK bulk imports keyed field values to a host.
 func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
@@ -645,6 +684,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	}
 	httpReq.Header.Set("Content-Type", "application/x-protobuf")
 	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("X-Pilosa-Row", "roaring")
 	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	// Execute request against the host.
@@ -667,6 +707,57 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	return nil
 }
 
+// ImportColumnAttrs does bulk import of column attrs
+func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	url := fmt.Sprintf("%s/index/%s/import-column-attrs", uri, index)
+
+	// Marshal data to protobuf.
+	data, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Wrap(err, "marshal import-column-attrs request")
+	}
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("Content-Type", "application/x-protobuf")
+	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("X-Pilosa-Row", "roaring")
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	dec := json.NewDecoder(resp.Body)
+	rbody := &pilosa.ImportResponse{}
+	err = dec.Decode(rbody)
+	// Decode can return EOF when no error occurred. helpful!
+	if err != nil && err != io.EOF {
+		return errors.Wrap(err, "decoding response body")
+	}
+	if rbody.Err != "" {
+		return errors.Wrap(errors.New(rbody.Err), "importing roaring")
+	}
+
+	return nil
+}
+
 // ExportCSV bulk exports data for a single shard from a host to CSV format.
 func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
@@ -791,18 +882,28 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
 	}
 
 	// convert pilosa.FieldOptions to fieldOptions
+	//
+	// TODO this kind of sucks because it's one more place that needs
+	// changes when we change anything with field options (and there
+	// are a lot of places already). It's not clear to me that this is
+	// providing a lot of value, but I think this kind of validation
+	// should probably happen in the field anyway??
 	fieldOpt := fieldOptions{
 		Type: opt.Type,
 		Keys: &opt.Keys,
 	}
-	if fieldOpt.Type == "set" {
+	if fieldOpt.Type == pilosa.FieldTypeSet {
 		fieldOpt.CacheType = &opt.CacheType
 		fieldOpt.CacheSize = &opt.CacheSize
-	} else if fieldOpt.Type == "int" {
+	} else if fieldOpt.Type == pilosa.FieldTypeInt {
 		fieldOpt.Min = &opt.Min
 		fieldOpt.Max = &opt.Max
-	} else if fieldOpt.Type == "time" {
+	} else if fieldOpt.Type == pilosa.FieldTypeTime {
 		fieldOpt.TimeQuantum = &opt.TimeQuantum
+	} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
+		fieldOpt.Min = &opt.Min
+		fieldOpt.Max = &opt.Max
+		fieldOpt.Scale = &opt.Scale
 	}
 
 	// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
@@ -909,6 +1010,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
 	req.Header.Set("Content-Type", "application/protobuf")
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Accept", "application/protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 
 	resp, err := c.executeRequest(req.WithContext(ctx))
@@ -1026,21 +1128,286 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Connection", "keep-alive")
 
 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
 		return errors.Wrap(err, "executing request")
 	}
-	return errors.Wrap(resp.Body.Close(), "closing response body")
+	defer resp.Body.Close()
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	return errors.Wrap(err, "draining SendMessage response body")
+}
+
+// TranslateKeysNode sends a key translation request to a specific node.
+func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string) ([]uint64, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{
+		Index: index,
+		Field: field,
+		Keys:  keys,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateKeysRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/keys")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateKeysResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.IDs, nil
+}
+
+// TranslateIDsNode sends an id translation request to a specific node.
+func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{
+		Index: index,
+		Field: field,
+		IDs:   ids,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateIDsRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/ids")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("X-Pilosa-Row", "roaring")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateIDsResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.Keys, nil
+}
+
+func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transactions")
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating transactions request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	trnsMap := make(map[string]*pilosa.Transaction)
+	err = json.NewDecoder(resp.Body).Decode(&trnsMap)
+	return trnsMap, errors.Wrap(err, "json decoding")
+}
+
+func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.StartTransaction")
+	defer span.Finish()
+	buf, err := json.Marshal(&pilosa.Transaction{
+		ID:        id,
+		Timeout:   timeout,
+		Exclusive: exclusive,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshalling payload")
+	}
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating post transaction request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+	if resp.StatusCode == 409 {
+		err = pilosa.ErrTransactionExclusive
+	} else if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FinishTransaction")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id+"/finish")
+	req, err := http.NewRequest("POST", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating finish transaction request")
+	}
+
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetTransaction")
+	defer span.Finish()
+
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating get transaction request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+type executeOpts struct {
+	// giveRawResponse instructs executeRequest not to process the
+	// respStatusCode and try to extract errors or whatever.
+	giveRawResponse bool
+}
+
+type executeRequestOption func(*executeOpts)
+
+func giveRawResponse(b bool) executeRequestOption {
+	return func(eo *executeOpts) {
+		eo.giveRawResponse = b
+	}
 }
 
 // executeRequest executes the given request and checks the Response. For
 // responses with non-2XX status, the body is read and closed, and an error is
 // returned. If the error is nil, the caller must ensure that the response body
 // is closed.
-func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) {
+func (c *InternalClient) executeRequest(req *http.Request, opts ...executeRequestOption) (*http.Response, error) {
+	eo := &executeOpts{}
+	for _, opt := range opts {
+		opt(eo)
+	}
+
 	tracing.GlobalTracer.InjectHTTPHeaders(req)
+	req.Close = false
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		if resp != nil {
@@ -1048,6 +1415,9 @@ func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, erro
 		}
 		return nil, errors.Wrap(err, "getting response")
 	}
+	if eo.giveRawResponse {
+		return resp, nil
+	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		defer resp.Body.Close()
 		buf, err := ioutil.ReadAll(resp.Body)
@@ -1278,3 +1648,40 @@ func nodePathToURL(node *pilosa.Node, path string) url.URL {
 		Path:   path,
 	}
 }
+
+// RetrieveTranslatePartitionFromURI returns a ReadCloser which contains the data of the
+// specified translate partition from the specified node. Caller *must* close the returned
+// ReadCloser or risk leaking goroutines/tcp connections.
+func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pilosa.URI) (io.ReadCloser, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveTranslatePartitionFromURI")
+	defer span.Finish()
+
+	node := &pilosa.Node{
+		URI: uri,
+	}
+
+	u := nodePathToURL(node, "/internal/translate/data")
+	u.RawQuery = url.Values{
+		"index":     {index},
+		"partition": {strconv.FormatInt(int64(partition), 10)},
+	}.Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			return nil, pilosa.ErrFragmentNotFound
+		}
+		return nil, err
+	}
+
+	return resp.Body, nil
+}

http/client_test.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	gohttp "net/http"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -31,6 +33,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 // Test distributed TopN Row count across 3 nodes.
@@ -116,15 +119,15 @@ func TestClient_MultiNode(t *testing.T) {
 	// Rebuild the RankCache.
 	// We have to do this to avoid the 10-second cache invalidation delay
 	// built into cache.Invalidate()
-	err = c[0].RecalculateCaches()
+	err = c[0].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[1].RecalculateCaches()
+	err = c[1].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[2].RecalculateCaches()
+	err = c[2].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
@@ -147,7 +150,8 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	// Test must return exactly N results.
-	if len(result.Results[0].([]pilosa.Pair)) != topN {
+	pairsField := result.Results[0].(*pilosa.PairsField)
+	if len(pairsField.Pairs) != topN {
 		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
 	}
 	p := []pilosa.Pair{
@@ -157,7 +161,7 @@ func TestClient_MultiNode(t *testing.T) {
 		{ID: 99, Count: 7}}
 
 	// Valdidate the Top 4 result counts.
-	if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
+	if !reflect.DeepEqual(pairsField.Pairs, p) {
 		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
 	}
 
@@ -292,22 +296,29 @@ func TestClient_Export(t *testing.T) {
 		buf := bytes.NewBuffer(nil)
 		bw := bufio.NewWriter(buf)
 
-		// Send export request.
+		// Send export request for every partition.
-		if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", 0, bw); err != nil {
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil {
 				t.Fatal(err)
 			}
+		}
 
 		got := buf.String()
 
-		// Expected output.
+		// Expected output is not sorted because of key sharding.
-		exp := ""
+		exp := "" +
-		for _, bit := range data {
+			"2,col200\n" +
-			exp += fmt.Sprintf("%d,%s\n", bit.RowID, bit.ColumnKey)
+			"2,col201\n" +
-		}
+			"2,col202\n" +
+			"2,col203\n" +
+			"1,col103\n" +
+			"1,col102\n" +
+			"1,col101\n" +
+			"1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 
@@ -327,21 +338,28 @@ func TestClient_Export(t *testing.T) {
 		bw := bufio.NewWriter(buf)
 
 		// Send export request.
-		if err := c.ExportCSV(context.Background(), "keyed", "keyedf", 0, bw); err != nil {
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil {
 				t.Fatal(err)
 			}
+		}
 
 		got := buf.String()
 
-		// Expected output.
+		// Expected output is unsorted because of key sharding.
-		exp := ""
+		exp := "" +
-		for _, bit := range data {
+			"row2,col200\n" +
-			exp += fmt.Sprintf("%s,%s\n", bit.RowKey, bit.ColumnKey)
+			"row2,col201\n" +
-		}
+			"row2,col202\n" +
+			"row2,col203\n" +
+			"row1,col103\n" +
+			"row1,col102\n" +
+			"row1,col101\n" +
+			"row1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 }
@@ -394,6 +412,60 @@ func TestClient_Import(t *testing.T) {
 	}
 }
 
+// Ensure client can bulk import column attrs.
+func TestClient_ImportColumnAttrs(t *testing.T) {
+	cluster := test.MustNewCluster(t, 2)
+	for _, c := range cluster {
+		c.Config.Cluster.ReplicaN = 2
+	}
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer cluster.Close()
+
+	ctx := context.Background()
+	_, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+	_, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+
+	attrKey := "k"
+	// Send import request.
+	host := cluster[0].URL()
+	c := MustNewClient(host, http.GetHTTPClient(nil))
+	colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
+	if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data.
+	pql := "Options(Row(f=0), columnAttrs=true)"
+	res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res.ColumnAttrSets) != 5 {
+		t.Fatal("incorrect number of column attrs set")
+	}
+
+	for _, v := range res.ColumnAttrSets {
+		attrVal := attrFun(v.ID)
+		if attrVal != v.Attrs[attrKey] {
+			t.Fatal(err)
+		}
+	}
+
+}
+
 // Ensure client can bulk import data.
 func TestClient_ImportRoaring(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
@@ -546,18 +618,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -573,18 +645,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(unkeyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{ID: 1, Count: 3},
 				{ID: 2, Count: 2},
 				{ID: 3, Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -600,18 +672,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "unkeyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 	})
@@ -645,18 +717,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd0.MustRecalculateCaches(t)
-			resp := cmd0.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd0.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf0)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -677,18 +749,18 @@ func TestClient_ImportKeys(t *testing.T) {
 			time.Sleep(500 * time.Millisecond)
 
 			cmd1.MustRecalculateCaches(t)
-			resp := cmd1.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd1.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf1)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %#v", pairs)
+				t.Fatalf("unexpected topn result: %#v", pairs.Pairs)
 			}
 		})
 	})
@@ -705,7 +777,7 @@ func TestClient_ImportKeys(t *testing.T) {
 
 		// Load bitmap into cache to ensure cache gets updated.
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
-		field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
+		_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -720,15 +792,6 @@ func TestClient_ImportKeys(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Verify Sum.
-		sum, cnt, err := field.Sum(nil, fldName)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if sum != 50 || cnt != 3 {
-			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
-		}
-
 		// Verify range.
 		queryRequest := &pilosa.QueryRequest{
 			Query:  fmt.Sprintf(`Row(%s>10)`, fldName),
@@ -751,15 +814,6 @@ func TestClient_ImportKeys(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Verify Sum.
-		sum, cnt, err = field.Sum(nil, fldName)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if sum != 30 || cnt != 2 {
-			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=30, cnt=2", sum, cnt)
-		}
-
 		// Verify Range.
 		queryRequest = &pilosa.QueryRequest{
 			Query:  fmt.Sprintf(`Row(%s>10)`, fldName),
@@ -777,6 +831,72 @@ func TestClient_ImportKeys(t *testing.T) {
 	})
 }
 
+func TestClient_ImportIDs(t *testing.T) {
+	// Ensure that running a query between two imports does
+	// not affect the result set. It turns out, this is caused
+	// by the fragment.rowCache failing to be cleared after an
+	// importValue. This ensures that the rowCache is cleared
+	// after an import.
+	t.Run("ImportRangeImport", func(t *testing.T) {
+		cluster := test.MustRunCluster(t, 1)
+		defer cluster.Close()
+		cmd := cluster[0]
+		host := cmd.URL()
+		holder := cmd.Server.Holder()
+		hldr := test.Holder{Holder: holder}
+
+		idxName := "i"
+		fldName := "f"
+
+		// Load bitmap into cache to ensure cache gets updated.
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{Keys: false})
+		_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-10000, 10000))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Send import request.
+		c := MustNewClient(host, http.GetHTTPClient(nil))
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 2, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		queryRequest := &pilosa.QueryRequest{
+			Query:  fmt.Sprintf(`Row(%s>0)`, fldName),
+			Remote: false,
+		}
+
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+
+		// Send import request.
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 1000, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2, 1000}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+	})
+}
+
 // Ensure client can bulk import value data.
 func TestClient_ImportValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
@@ -790,7 +910,7 @@ func TestClient_ImportValue(t *testing.T) {
 
 	// Load bitmap into cache to ensure cache gets updated.
 	index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
-	field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
+	_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -806,43 +926,21 @@ func TestClient_ImportValue(t *testing.T) {
 	}
 
 	// Verify Sum.
-	sum, cnt, err := field.Sum(nil, fldName)
+	if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
-	if err != nil {
 		t.Fatal(err)
-	}
+	} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
-	if sum != 50 || cnt != 3 {
+		t.Fatalf("expected ValCount; got %T", resp.Results[0])
-		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
+	} else if vc.Val != 50 || vc.Count != 3 {
-	}
+		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", vc.Val, vc.Count)
-
-	// Verify Min.
-	min, cnt, err := field.Min(nil, fldName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if min != -10 || cnt != 1 {
-		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
-	}
-
-	// Verify Min with Filter.
-	filter, err := field.Range(fldName, pql.GT, 40)
-	if err != nil {
-		t.Fatal(err)
-	}
-	min, cnt, err = field.Min(filter, fldName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if min != 0 || cnt != 0 {
-		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt)
 	}
 
 	// Verify Max.
-	max, cnt, err := field.Max(nil, fldName)
+	if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
-	if err != nil {
 		t.Fatal(err)
-	}
+	} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
-	if max != 40 || cnt != 1 {
+		t.Fatalf("expected ValCount; got %T", resp.Results[0])
-		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
+	} else if vc.Val != 40 || vc.Count != 1 {
+		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", vc.Val, vc.Count)
 	}
 
 	// Send import request.
@@ -854,34 +952,21 @@ func TestClient_ImportValue(t *testing.T) {
 	}
 
 	// Verify Sum.
-	sum, cnt, err = field.Sum(nil, fldName)
+	if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Sum(field=f)`}); err != nil {
-	if err != nil {
 		t.Fatal(err)
-	}
+	} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
-	if sum != 20 || cnt != 1 {
+		t.Fatalf("expected ValCount; got %T", resp.Results[0])
-		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=20, cnt=1", sum, cnt)
+	} else if vc.Val != 20 || vc.Count != 1 {
-	}
+		t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=20, cnt=1", vc.Val, vc.Count)
-
-	// Verify Min with Filter.
-	filter, err = field.Range(fldName, pql.GT, 40)
-	if err != nil {
-		t.Fatal(err)
-	}
-	min, cnt, err = field.Min(filter, fldName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if min != 0 || cnt != 0 {
-		t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt)
 	}
 
 	// Verify Max.
-	max, cnt, err = field.Max(nil, fldName)
+	if resp, err := c.Query(context.Background(), "i", &pilosa.QueryRequest{Query: `Max(field=f)`}); err != nil {
-	if err != nil {
 		t.Fatal(err)
-	}
+	} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
-	if max != 20 || cnt != 1 {
+		t.Fatalf("expected ValCount; got %T", resp.Results[0])
-		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=20, cnt=1", max, cnt)
+	} else if vc.Val != 20 || vc.Count != 1 {
+		t.Fatalf("unexpected values: got max=%v, count=%v; expected max=20, cnt=1", vc.Val, vc.Count)
 	}
 }
 
@@ -933,7 +1018,7 @@ func TestClient_ImportExistence(t *testing.T) {
 		fldName := "fint"
 
 		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true})
-		field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
+		_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -949,12 +1034,12 @@ func TestClient_ImportExistence(t *testing.T) {
 		}
 
 		// Verify Sum.
-		sum, cnt, err := field.Sum(nil, fldName)
+		if resp, err := c.Query(context.Background(), idxName, &pilosa.QueryRequest{Query: fmt.Sprintf(`Sum(field=%s)`, fldName)}); err != nil {
-		if err != nil {
 			t.Fatal(err)
-		}
+		} else if vc, ok := resp.Results[0].(pilosa.ValCount); !ok {
-		if sum != 50 || cnt != 3 {
+			t.Fatalf("expected ValCount; got %T", resp.Results[0])
-			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
+		} else if vc.Val != 50 || vc.Count != 3 {
+			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", vc.Val, vc.Count)
 		}
 
 		// Verify existence.
@@ -998,6 +1083,271 @@ func TestClient_FragmentBlocks(t *testing.T) {
 	}
 }
 
+func TestClient_CreateDecimalField(t *testing.T) {
+	cluster := test.MustRunCluster(t, 1)
+	defer cluster.Close()
+	cmd := cluster[0]
+
+	c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
+
+	index := "cdf"
+	err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	field := "dfield"
+	err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeDecimal, Scale: 1, Min: pql.NewDecimal(-1000, 0), Max: pql.NewDecimal(1000, 0)})
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	fld, err := cmd.API.Field(context.Background(), index, field)
+	if err != nil {
+		t.Fatalf("getting field: %v", err)
+	}
+	if fld.Options().Scale != 1 {
+		t.Fatalf("expected Scale 1, got: %+v", fld.Options())
+	}
+
+	err = c.ImportValue2(context.Background(), &pilosa.ImportValueRequest{Index: index, Field: field, ColumnIDs: []uint64{1, 2, 3}, Shard: 0, FloatValues: []float64{1.1, 2.2, 3.3}}, &pilosa.ImportOptions{})
+	if err != nil {
+		t.Fatalf("importing float values: %v", err)
+	}
+
+	// Integer predicate.
+	resp, err := c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicate.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.1)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Integer predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1<dfield<3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<=dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<=3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>=2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+}
+
+func TestClientTransactions(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	client0 := MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
+	client1 := MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
+
+	// can create, list, get, and finish a transaction
+	var expDeadline time.Time
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	if trnsMap, err := client0.Transactions(context.Background()); err != nil {
+		t.Errorf("listing transactions: %v", err)
+	} else {
+		if len(trnsMap) != 1 {
+			t.Errorf("unexpected trnsMap: %+v", trnsMap)
+		}
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trnsMap["blah"])
+	}
+
+	if trns, err := client0.GetTransaction(context.Background(), "blah"); err != nil {
+		t.Fatalf("error getting transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	if trns, err := client0.FinishTransaction(context.Background(), "blah"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// can create exclusive transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blahe", time.Minute, true); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// cannot start new transaction - correct error and exclusive transaction are returned
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); errors.Cause(err) != pilosa.ErrTransactionExclusive {
+		t.Fatalf("shouldn't be able to start transaction while an exclusive is running, but got: %+v, %v", trns, err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish exclusive transaction
+	if trns, err := client0.FinishTransaction(context.Background(), "blahe"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// start new transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// try to start same transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionExists.Error()) {
+		t.Fatalf("expected ErrTransactionExists, but got: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// start an exclusive transaction which can't go active
+	if trns, err := client0.StartTransaction(context.Background(), "blahe", time.Minute, true); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: false, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish exclusive transaction that never went active
+	if trns, err := client0.FinishTransaction(context.Background(), "blahe"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: false, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish non-existent transaction
+	if trns, err := client0.FinishTransaction(context.Background(), "zzz"); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionNotFound.Error()) {
+		t.Fatalf("unexpected error finishing nonexistent transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// get non-existent transaction
+	if trns, err := client0.GetTransaction(context.Background(), "xxx"); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionNotFound.Error()) {
+		t.Fatalf("unexpected error getting nonexistent transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// non-coordinator
+	if trns, err := client1.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrNodeNotCoordinator.Error()) {
+		t.Fatalf("unexpected error starting on non-coordinator: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// start transaction with blank id
+	if trns, err := client0.StartTransaction(context.Background(), "", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		if len(trns.ID) != 36 {
+			t.Errorf("expected generated UUID, but got '%s'", trns.ID)
+		}
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: trns.ID, Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+}
+
 // Client represents a test wrapper for pilosa.Client.
 type Client struct {
 	*http.InternalClient
@@ -1021,3 +1371,23 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring
 		},
 	}
 }
+
+func attrFun(id uint64) string {
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pilosa.ImportColumnAttrsRequest {
+	colIDs := make([]uint64, 0, 5)
+	attrVals := make([]string, 0, 5)
+	for n := uint64(0); n < 5; n++ {
+		colIDs = append(colIDs, n)
+		attrVals = append(attrVals, attrFun(n))
+	}
+	return &pilosa.ImportColumnAttrsRequest{
+		Index:     index,
+		Shard:     shard,
+		AttrKey:   attrKey,
+		ColumnIDs: colIDs,
+		AttrVals:  attrVals,
+	}
+}

http/handler_internal_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 // Test custom UnmarshalJSON for postIndexRequest object
@@ -99,8 +100,8 @@ func stringPtr(s string) *string {
 	return &s
 }
 
-func int64Ptr(i int64) *int64 {
+func decimalPtr(d pql.Decimal) *pql.Decimal {
-	return &i
+	return &d
 }
 
 // Test fieldOption validation.
@@ -135,10 +136,10 @@ func TestFieldOptionValidation(t *testing.T) {
 		// FieldType: Int
 		{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
 		{json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"},
-		{json: `{"options": {"type": "int", "min": 0, "max": 1000}}`, expected: postFieldRequest{Options: fieldOptions{
+		{json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{
 			Type: pilosa.FieldTypeInt,
-			Min:  int64Ptr(0),
+			Min:  decimalPtr(pql.NewDecimal(0, 0)),
-			Max:  int64Ptr(1000),
+			Max:  decimalPtr(pql.NewDecimal(1001, 0)),
 		}}},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"},

http/handler_test.go

@@ -15,11 +15,13 @@
 package http_test
 
 import (
+	"encoding/json"
 	"net"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/test"
 )
 
 func TestHandlerOptions(t *testing.T) {
@@ -40,3 +42,39 @@ func TestHandlerOptions(t *testing.T) {
 		t.Fatalf("expected error making handler without options, got nil")
 	}
 }
+
+func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
+	tests := []struct {
+		name string
+		tr   *http.TransactionResponse
+	}{
+		{
+			name: "nil transaction",
+			tr:   &http.TransactionResponse{},
+		},
+		{
+			name: "empty transaction",
+			tr:   &http.TransactionResponse{Transaction: &pilosa.Transaction{}},
+		},
+	}
+
+	for _, tst := range tests {
+		t.Run(tst.name, func(t *testing.T) {
+			data, err := json.Marshal(tst.tr)
+			if err != nil {
+				t.Fatalf("marshaling: %v", err)
+			}
+
+			mytr := &http.TransactionResponse{}
+			err = json.Unmarshal(data, mytr)
+			if err != nil {
+				t.Fatalf("unmarshalling: %v", err)
+			}
+
+			if mytr.Error != tst.tr.Error {
+				t.Errorf("errors mismatch:exp/got \n%v\n%v", tst.tr.Error, mytr.Error)
+			}
+			test.CompareTransactions(t, tst.tr.Transaction, mytr.Transaction)
+		})
+	}
+}

index.go

@@ -21,12 +21,12 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2/internal"
-	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pkg/errors"
@@ -36,6 +36,7 @@ import (
 // Index represents a container for fields.
 type Index struct {
 	mu        sync.RWMutex
+	createdAt int64
 	path      string
 	name      string
 	keys      bool // use string keys
@@ -52,23 +53,23 @@ type Index struct {
 	// Column attribute storage and cache.
 	columnAttrs AttrStore
 
-	translateStore TranslateStore
-
 	broadcaster broadcaster
 	Stats       stats.StatsClient
 
-	logger        logger.Logger
+	// Passed to field for foreign-index lookup.
-	snapshotQueue chan *fragment
-
-	// Used for notifying holder when a field is added.
 	holder *Holder
 
-	// Instantiates new translation stores for fields.
+	// Per-partition translation stores
+	translateStores map[int]TranslateStore
+
+	translationSyncer translationSyncer
+
+	// Instantiates new translation stores
 	OpenTranslateStore OpenTranslateStoreFunc
 }
 
 // NewIndex returns a new instance of Index.
-func NewIndex(path, name string) (*Index, error) {
+func NewIndex(holder *Holder, path, name string) (*Index, error) {
 	err := validateName(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "validating name")
@@ -84,28 +85,46 @@ func NewIndex(path, name string) (*Index, error) {
 
 		broadcaster:    NopBroadcaster,
 		Stats:          stats.NopStatsClient,
-		logger:         logger.NopLogger,
+		holder:         holder,
 		trackExistence: true,
 
+		translateStores: make(map[int]TranslateStore),
+
+		translationSyncer: NopTranslationSyncer,
+
 		OpenTranslateStore: OpenInMemTranslateStore,
 	}, nil
 }
 
+// CreatedAt is an timestamp for a specific version of an index.
+func (i *Index) CreatedAt() int64 {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	return i.createdAt
+}
+
 // Name returns name of the index.
 func (i *Index) Name() string { return i.name }
 
 // Path returns the path the index was initialized with.
 func (i *Index) Path() string { return i.path }
 
+// TranslateStorePath returns the translation database path for a partition.
+func (i *Index) TranslateStorePath(partitionID int) string {
+	return filepath.Join(i.path, translateStoreDir, strconv.Itoa(partitionID))
+}
+
+// TranslateStore returns the translation store for a given partition.
+func (i *Index) TranslateStore(partitionID int) TranslateStore {
+	return i.translateStores[partitionID]
+}
+
 // Keys returns true if the index uses string keys.
 func (i *Index) Keys() bool { return i.keys }
 
 // ColumnAttrStore returns the storage for column attributes.
 func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs }
 
-// TranslateStore returns the underlying translation store for the index.
-func (i *Index) TranslateStore() TranslateStore { return i.translateStore }
-
 // Options returns all options for this index.
 func (i *Index) Options() IndexOptions {
 	i.mu.RLock()
@@ -121,21 +140,26 @@ func (i *Index) options() IndexOptions {
 }
 
 // Open opens and initializes the index.
-func (i *Index) Open() (err error) {
+func (i *Index) Open() error { return i.open(false) }
+
+// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
+func (i *Index) OpenWithTimestamp() error { return i.open(true) }
+
+func (i *Index) open(withTimestamp bool) (err error) {
 	// Ensure the path exists.
-	i.logger.Debugf("ensure index path exists: %s", i.path)
+	i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
 	if err := os.MkdirAll(i.path, 0777); err != nil {
 		return errors.Wrap(err, "creating directory")
 	}
 
 	// Read meta file.
-	i.logger.Debugf("load meta file for index: %s", i.name)
+	i.holder.Logger.Debugf("load meta file for index: %s", i.name)
 	if err := i.loadMeta(); err != nil {
 		return errors.Wrap(err, "loading meta file")
 	}
 
-	i.logger.Debugf("open fields for index: %s", i.name)
+	i.holder.Logger.Debugf("open fields for index: %s", i.name)
-	if err := i.openFields(); err != nil {
+	if err := i.openFields(withTimestamp); err != nil {
 		return errors.Wrap(err, "opening fields")
 	}
 
@@ -149,9 +173,27 @@ func (i *Index) Open() (err error) {
 		return errors.Wrap(err, "opening attrstore")
 	}
 
-	// Instantiate & open translation store.
+	i.holder.Logger.Debugf("open translate store for index: %s", i.name)
-	if i.translateStore, err = i.OpenTranslateStore(filepath.Join(i.path, "keys"), i.name, ""); err != nil {
+
-		return errors.Wrap(err, "opening translate store")
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := 0; partitionID < i.holder.partitionN; partitionID++ {
+		partitionID := partitionID
+
+		g.Go(func() error {
+			store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN)
+			if err != nil {
+				return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			i.translateStores[partitionID] = store
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return err
 	}
 
 	return nil
@@ -160,7 +202,7 @@ func (i *Index) Open() (err error) {
 var indexQueue = make(chan struct{}, 8)
 
 // openFields opens and initializes the fields inside the index.
-func (i *Index) openFields() error {
+func (i *Index) openFields(withTimestamp bool) error {
 	f, err := os.Open(i.path)
 	if err != nil {
 		return errors.Wrap(err, "opening directory")
@@ -189,18 +231,25 @@ fileLoop:
 				defer func() {
 					<-indexQueue
 				}()
-				i.logger.Debugf("open field: %s", fi.Name())
+				i.holder.Logger.Debugf("open field: %s", fi.Name())
 				mu.Lock()
 				fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
+				if withTimestamp {
+					fld.createdAt = timestamp()
+				}
 				mu.Unlock()
 				if err != nil {
 					return errors.Wrapf(ErrName, "'%s'", fi.Name())
 				}
 
+				// Pass holder through to the field for use in looking
+				// up a foreign index.
+				fld.holder = i.holder
+
 				if err := fld.Open(); err != nil {
 					return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
 				}
-				i.logger.Debugf("add field to index.fields: %s", fi.Name())
+				i.holder.Logger.Debugf("add field to index.fields: %s", fi.Name())
 				mu.Lock()
 				i.fields[fld.Name()] = fld
 				mu.Unlock()
@@ -213,7 +262,7 @@ fileLoop:
 
 // openExistenceField gets or creates the existence field and associates it to the index.
 func (i *Index) openExistenceField() error {
-	f, err := i.createFieldIfNotExists(existenceFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
+	f, err := i.createFieldIfNotExists(existenceFieldName, &FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
 	if err != nil {
 		return errors.Wrap(err, "creating existence field")
 	}
@@ -223,7 +272,8 @@ func (i *Index) openExistenceField() error {
 
 // loadMeta reads meta data for the index, if any.
 func (i *Index) loadMeta() error {
-	var pb internal.IndexMeta
+	// TrackExistence is by default true
+	pb := &internal.IndexMeta{TrackExistence: true}
 
 	// Read data from meta file.
 	buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
@@ -232,14 +282,18 @@ func (i *Index) loadMeta() error {
 	} else if err != nil {
 		return errors.Wrap(err, "reading")
 	} else {
-		if err := proto.Unmarshal(buf, &pb); err != nil {
+		if err := proto.Unmarshal(buf, pb); err != nil {
 			return errors.Wrap(err, "unmarshalling")
 		}
 	}
 
 	// Copy metadata fields.
-	i.keys = pb.Keys
+	if pb == nil {
+		i.trackExistence = true
+	} else {
 		i.trackExistence = pb.TrackExistence
+	}
+	i.keys = pb.GetKeys()
 
 	return nil
 }
@@ -271,6 +325,14 @@ func (i *Index) Close() error {
 	// Close the attribute store.
 	i.columnAttrs.Close()
 
+	// Close partitioned translation stores.
+	for _, store := range i.translateStores {
+		if err := store.Close(); err != nil {
+			return errors.Wrap(err, "closing translation store")
+		}
+	}
+	i.translateStores = make(map[int]TranslateStore)
+
 	// Close all fields.
 	for _, f := range i.fields {
 		if err := f.Close(); err != nil {
@@ -279,12 +341,6 @@ func (i *Index) Close() error {
 	}
 	i.fields = make(map[string]*Field)
 
-	if i.translateStore != nil {
-		if err := i.translateStore.Close(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -299,10 +355,10 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
 
 	b := roaring.NewBitmap()
 	for _, f := range i.fields {
-		b = b.Union(f.AvailableShards())
+		b.UnionInPlace(f.AvailableShards())
 	}
 
-	i.Stats.Gauge("maxShard", float64(b.Max()), 1.0)
+	i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0)
 	return b
 }
 
@@ -362,14 +418,11 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 		return nil, newConflictError(ErrFieldExists)
 	}
 
-	// Apply functional options.
+	// Apply and validate functional options.
-	fo := FieldOptions{}
+	fo, err := newFieldOptions(opts...)
-	for _, opt := range opts {
-		err := opt(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
 	}
-	}
 
 	return i.createField(name, fo)
 }
@@ -389,19 +442,16 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field
 		return f, nil
 	}
 
-	// Apply functional options.
+	// Apply and validate functional options.
-	fo := FieldOptions{}
+	fo, err := newFieldOptions(opts...)
-	for _, opt := range opts {
-		err := opt(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
 	}
-	}
 
 	return i.createField(name, fo)
 }
 
-func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createFieldIfNotExists(name string, opt *FieldOptions) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -413,7 +463,7 @@ func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, e
 	return i.createField(name, opt)
 }
 
-func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
 	if name == "" {
 		return nil, errors.New("field name required")
 	} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
@@ -426,17 +476,17 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 		return nil, errors.Wrap(err, "initializing")
 	}
 
+	// Pass holder through to the field for use in looking
+	// up a foreign index.
+	f.holder = i.holder
+
+	f.setOptions(opt)
+
 	// Open field.
 	if err := f.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 
-	// Apply field options.
-	if err := f.applyOptions(opt); err != nil {
-		f.Close()
-		return nil, errors.Wrap(err, "applying options")
-	}
-
 	if err := f.saveMeta(); err != nil {
 		f.Close()
 		return nil, errors.Wrap(err, "saving meta")
@@ -445,24 +495,22 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 	// Add to index's field lookup.
 	i.fields[name] = f
 
-	// Update replication, if needed.
+	// Kick off the field's translation sync process.
-	if i.holder != nil {
+	if err := i.translationSyncer.Reset(); err != nil {
-		go i.holder.refreshTranslateStoreReplicator()
+		return nil, errors.Wrap(err, "resetting translation syncer")
 	}
 
 	return f, nil
 }
 
 func (i *Index) newField(path, name string) (*Field, error) {
-	f, err := newField(path, i.name, name, OptFieldTypeDefault())
+	f, err := newField(i.holder, path, i.name, name, OptFieldTypeDefault())
 	if err != nil {
 		return nil, err
 	}
-	f.logger = i.logger
 	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
-	f.snapshotQueue = i.snapshotQueue
 	f.OpenTranslateStore = i.OpenTranslateStore
 	return f, nil
 }
@@ -503,7 +551,7 @@ func (i *Index) DeleteField(name string) error {
 	// Remove reference.
 	delete(i.fields, name)
 
-	return nil
+	return i.translationSyncer.Reset()
 }
 
 type indexSlice []*Index
@@ -515,6 +563,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
 // IndexInfo represents schema information for an index.
 type IndexInfo struct {
 	Name       string       `json:"name"`
+	CreatedAt  int64        `json:"createdAt,omitempty"`
 	Options    IndexOptions `json:"options"`
 	Fields     []*FieldInfo `json:"fields"`
 	ShardWidth uint64       `json:"shardWidth"`

index_internal_test.go

@@ -25,7 +25,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, "i")
+	index, err := NewIndex(NewHolder(1), path, "i")
 	if err != nil {
 		panic(err)
 	}

index_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
 	"github.com/pkg/errors"
 )
@@ -185,6 +186,30 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 		*/
 	})
+
+	t.Run("WithKeys", func(t *testing.T) {
+		// Don't allow an int field to be created with keys=true
+		t.Run("IntField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrIntFieldWithKeys {
+				t.Fatal("int field cannot be created with keys=true")
+			}
+		})
+
+		// Don't allow a decimal field to be created with keys=true
+		t.Run("DecimalField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -1}, pql.Decimal{Value: 1}), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrDecimalFieldWithKeys {
+				t.Fatal("decimal field cannot be created with keys=true")
+			}
+		})
+	})
 }
 
 // Ensure index can delete a field.
@@ -217,7 +242,7 @@ func TestIndex_InvalidName(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(path, "ABC")
+	index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "ABC")
 	if err == nil {
 		t.Fatalf("should have gotten an error on index name with caps")
 	}

internal/clustertests/docker-compose.yml

@@ -51,6 +51,6 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command:
-      - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
+      - "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
 networks: 
   pilosanet:

internal/private.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package internal;
 
+import "public.proto";
+
 message IndexMeta {
 	bool Keys = 3;
 	bool TrackExistence = 4;
@@ -12,12 +14,16 @@ message FieldOptions {
 	string CacheType = 3;
 	uint32 CacheSize = 4;
 	string TimeQuantum = 5;
-	int64 Min = 9;
+	int64 OldMin = 9;
-	int64 Max = 10;
+	int64 OldMax = 10;
 	bool Keys = 11;
 	bool NoStandardView = 12;
 	int64 Base = 13;
 	uint64 BitDepth = 14;
+	int64 Scale = 15;
+	string ForeignIndex = 16;
+	Decimal Min = 17;
+	Decimal Max = 18;
 }
 
 message ImportResponse {
@@ -58,12 +64,14 @@ message DeleteIndexMessage {
 message CreateIndexMessage {
 	string Index = 1;
 	IndexMeta Meta = 2;
+	int64 CreatedAt  = 3;
 }
 
 message CreateFieldMessage {
 	string Index = 1;
 	string Field = 2;
 	FieldOptions Meta = 3;
+	int64 CreatedAt  = 4;
 }
 
 message DeleteFieldMessage {
@@ -81,6 +89,7 @@ message Field {
 	string Name = 1;
 	FieldOptions Meta = 2;
 	repeated string Views = 3;
+	int64 CreatedAt = 4;
 }
 
 message Schema {
@@ -89,6 +98,8 @@ message Schema {
 
 message Index {
 	string Name = 1;
+	int64 CreatedAt = 2;
+	IndexMeta Options = 5;
 	repeated Field Fields = 4;
 }
 
@@ -103,6 +114,7 @@ message Node {
 	URI URI = 2;
 	bool IsCoordinator = 3;
 	string State = 4;
+	URI GRPCURI = 5;
 }
 
 message NodeStateMessage {
@@ -124,17 +136,20 @@ message NodeStatus {
 message IndexStatus {
 	string Name = 1;
 	repeated FieldStatus Fields = 2;
+	int64 CreatedAt = 3;
 }
 
 message FieldStatus {
 	string Name = 1;
 	repeated uint64 AvailableShards = 2;
+	int64 CreatedAt = 3;
 }
 
 message ClusterStatus {
 	string ClusterID = 1;
 	string State = 2;
 	repeated Node Nodes = 3;
+	Schema Schema = 4;
 }
 
 message BSIGroup {
@@ -161,6 +176,7 @@ message ResizeInstruction {
 	Node Node = 2;
 	Node Coordinator = 3;
 	repeated ResizeSource Sources = 4;
+	repeated TranslationResizeSource TranslationSources = 8;
 	NodeStatus NodeStatus = 7;
 	ClusterStatus ClusterStatus = 6;
 }
@@ -173,6 +189,12 @@ message ResizeSource {
 	uint64 Shard = 5;
 }
 
+message TranslationResizeSource {
+	Node Node = 1;
+	string Index = 2;
+	int32 PartitionID = 3;
+}
+
 message ResizeInstructionComplete {
 	int64 JobID = 1;
 	Node Node = 2;
@@ -193,3 +215,19 @@ message Topology {
 }
 
 message RecalculateCaches {}
+
+message TransactionMessage {
+	string Action = 1;
+	Transaction Transaction = 2;
+}
+
+message Transaction {
+	string ID = 1;
+	bool Active = 2;
+	bool Exclusive = 3;
+	int64 Timeout = 4;
+	int64 Deadline = 5;
+	TransactionStats Stats = 6;
+}
+
+message TransactionStats {}

internal/public.proto

@@ -6,6 +6,12 @@ message Row {
 	repeated uint64 Columns = 1;
 	repeated string Keys = 3;
 	repeated Attr Attrs = 2;
+	bytes Roaring = 4;
+}
+
+message SignedRow {
+	Row Pos = 1;
+	Row Neg = 2;
 }
 
 message RowIdentifiers {
@@ -19,20 +25,43 @@ message Pair {
 	uint64 Count = 2;
 }
 
+message PairField {
+	Pair Pair = 1;
+	string Field = 2;
+}
+
+message PairsField {
+	repeated Pair Pairs = 1;
+	string Field = 2;
+}
+
+message Int64 {
+	int64 Value = 1;
+}
+
 message FieldRow {
 	string Field = 1;
 	uint64 RowID = 2;
 	string RowKey = 3;
+	Int64 Value = 4;
 }
 
 message GroupCount{
 	repeated FieldRow Group = 1;
 	uint64 Count = 2;
+	int64 Sum = 3;
 }
 
 message ValCount {
 	int64 Val = 1;
 	int64 Count = 2;
+	double FloatVal = 3;
+	Decimal DecimalVal = 4;
+}
+
+message Decimal {
+	int64 Value = 1;
+	int64 Scale = 2;
 }
 
 message ColumnAttrSet {
@@ -61,6 +90,7 @@ message QueryRequest {
 	bool Remote = 5;
 	bool ExcludeRowAttrs = 6;
 	bool ExcludeColumns = 7;
+	repeated Row EmbeddedData = 8;
 }
 
 message QueryResponse {
@@ -79,10 +109,14 @@ message QueryResult {
 	repeated uint64 RowIDs = 7;
 	repeated GroupCount GroupCounts = 8;
 	RowIdentifiers RowIdentifiers = 9;
+	SignedRow SignedRow = 10;
+	PairsField PairsField = 11;
+	PairField PairField = 12;
 }
 
 message ImportRequest {
 	string Index = 1;
+
 	string Field = 2;
 	uint64 Shard = 3;
 	repeated uint64 RowIDs = 4;
@@ -90,6 +124,8 @@ message ImportRequest {
 	repeated string RowKeys = 7;
 	repeated string ColumnKeys = 8;
 	repeated int64 Timestamps = 6;
+	int64 IndexCreatedAt = 9;
+	int64 FieldCreatedAt = 10;
 }
 
 message ImportValueRequest {
@@ -99,6 +135,10 @@ message ImportValueRequest {
 	repeated uint64 ColumnIDs = 5;
 	repeated string ColumnKeys = 7;
 	repeated int64 Values = 6;
+	repeated double FloatValues = 8;
+	repeated string StringValues = 9;
+	int64 IndexCreatedAt = 10;
+	int64 FieldCreatedAt = 11;
 }
 
 message TranslateKeysRequest {
@@ -111,6 +151,16 @@ message TranslateKeysResponse {
 	repeated uint64 IDs = 3;
 }
 
+message TranslateIDsRequest {
+	string Index = 1;
+	string Field = 2;
+	repeated uint64 IDs = 3;
+}
+
+message TranslateIDsResponse {
+	repeated string Keys = 3;
+}
+
 message ImportRoaringRequestView {
 	string Name = 1;
 	bytes Data = 2;
@@ -119,4 +169,18 @@ message ImportRoaringRequestView {
 message ImportRoaringRequest {
 	bool Clear = 1;
 	repeated ImportRoaringRequestView views = 2;
+	string Action = 3;
+	uint64 Block = 4;
+	int64 IndexCreatedAt = 5;
+	int64 FieldCreatedAt = 6;
+
+}
+
+message ImportColumnAttrsRequest {
+	string Index = 1;
+	int64 Shard = 2;
+	string AttrKey = 3;
+	repeated string AttrVals = 4;
+	repeated uint64 ColumnIDs = 5;
+	int64 IndexCreatedAt = 6;
 }

license.exceptions

@@ -0,0 +1,11 @@
+# names of files which we do not expect to have our license header
+./apimethod_string.go
+./pql/pql.peg.go
+./internal/private.pb.go
+./internal/public.pb.go
+./lru/lru.go
+./roaring/btree.go
+./roaring/btree_test.go
+./proto/pilosa.pb.go
+./logger/filewriter.go
+./logger/filewriter_test.go

logger/filewriter.go

@@ -0,0 +1,108 @@
+// This file is a modified redistribution of reopen (github.com/client9/reopen),
+// which is governed by the following license notice:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2015 Nick Galbreath
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package logger
+
+import (
+	"os"
+	"sync"
+)
+
+// FileWriter that can also be reopened
+type FileWriter struct {
+	mu   sync.Mutex // ensures close / reopen / write are not called at the same time, protects f
+	f    *os.File
+	mode os.FileMode
+	name string
+}
+
+// Close calls the underlyding File.Close()
+func (f *FileWriter) Close() error {
+	f.mu.Lock()
+	err := f.f.Close()
+	f.mu.Unlock()
+	return err
+}
+
+// mutex free version
+func (f *FileWriter) reopen() error {
+	if f.f != nil {
+		f.f.Close()
+		f.f = nil
+	}
+	newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode)
+	if err != nil {
+		f.f = nil
+		return err
+	}
+	f.f = newf
+
+	return nil
+}
+
+// Reopen the file
+func (f *FileWriter) Reopen() error {
+	f.mu.Lock()
+	err := f.reopen()
+	f.mu.Unlock()
+	return err
+}
+
+// Write implements the stander io.Writer interface
+func (f *FileWriter) Write(p []byte) (int, error) {
+	f.mu.Lock()
+	n, err := f.f.Write(p)
+	f.mu.Unlock()
+	return n, err
+}
+
+// Fd returns the file descriptor of the underlying file.
+func (f *FileWriter) Fd() uintptr {
+	f.mu.Lock()
+	n := f.f.Fd()
+	f.mu.Unlock()
+	return n
+}
+
+// NewFileWriter opens a file for appending and writing and can be reopened.
+// it is a ReopenWriteCloser...
+func NewFileWriter(name string) (*FileWriter, error) {
+	// Standard default mode
+	return NewFileWriterMode(name, 0666)
+}
+
+// NewFileWriterMode opens a Reopener file with a specific permission
+func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) {
+	writer := FileWriter{
+		f:    nil,
+		name: name,
+		mode: mode,
+	}
+	err := writer.reopen()
+	if err != nil {
+		return nil, err
+	}
+	return &writer, nil
+}

logger/filewriter_test.go

@@ -0,0 +1,166 @@
+// This file is a modified redistribution of reopen (github.com/client9/reopen),
+// which is governed by the following license notice:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2015 Nick Galbreath
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package logger
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+// TestReopenAppend -- make sure we always append to an existing file
+//
+// 1. Create a sample file using normal means
+// 2. Open a ioreopen.File
+//    write line 1
+// 3. call Reopen
+//    write line 2
+// 4. close file
+// 5. read file, make sure it contains line0,line1,line2
+//
+func TestReopenAppend(t *testing.T) {
+	// TODO fix
+	// (travis) I have no idea what this TODO is asking for.
+	// Perhaps use `ioutil.TempFile()`?
+	var fname = "/tmp/foo"
+
+	// Step 1 -- Create a sample file using normal means
+	forig, err := os.Create(fname)
+	if err != nil {
+		t.Fatalf("Unable to create initial file %s: %s", fname, err)
+	}
+	_, err = forig.Write([]byte("line0\n"))
+	if err != nil {
+		t.Fatalf("Unable to write initial line %s: %s", fname, err)
+	}
+	err = forig.Close()
+	if err != nil {
+		t.Fatalf("Unable to close initial file: %s", err)
+	}
+
+	// Test that making a new File appends
+	f, err := NewFileWriter(fname)
+	if err != nil {
+		t.Fatalf("Unable to create %s", fname)
+	}
+	_, err = f.Write([]byte("line1\n"))
+	if err != nil {
+		t.Errorf("Got write error1: %s", err)
+	}
+
+	// Test that reopen always appends
+	err = f.Reopen()
+	if err != nil {
+		t.Errorf("Got reopen error %s: %s", fname, err)
+	}
+	_, err = f.Write([]byte("line2\n"))
+	if err != nil {
+		t.Errorf("Got write error2 on %s: %s", fname, err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Errorf("Got closing error for %s: %s", fname, err)
+	}
+
+	out, err := ioutil.ReadFile(fname)
+	if err != nil {
+		t.Fatalf("Unable read in final file %s: %s", fname, err)
+	}
+
+	outstr := string(out)
+	if outstr != "line0\nline1\nline2\n" {
+		t.Errorf("Result was %s", outstr)
+	}
+}
+
+// Test that reopen works when Inode is swapped out
+// 1. Create a sample file using normal means
+// 2. Open a ioreopen.File
+//    write line 1
+// 3. call Reopen
+//    write line 2
+// 4. close file
+// 5. read file, make sure it contains line0,line1,line2
+//
+func TestChangeInode(t *testing.T) {
+	// TODO fix
+	// (travis) I have no idea what this TODO is asking for.
+	// Perhaps use `ioutil.TempFile()`?
+	var fname = "/tmp/foo"
+
+	// Step 1 -- Create a empty sample file
+	forig, err := os.Create(fname)
+	if err != nil {
+		t.Fatalf("Unable to create initial file %s: %s", fname, err)
+	}
+	err = forig.Close()
+	if err != nil {
+		t.Fatalf("Unable to close initial file: %s", err)
+	}
+
+	// Test that making a new File appends
+	f, err := NewFileWriter(fname)
+	if err != nil {
+		t.Fatalf("Unable to create %s", fname)
+	}
+	_, err = f.Write([]byte("line1\n"))
+	if err != nil {
+		t.Errorf("Got write error1: %s", err)
+	}
+
+	// Now move file
+	err = os.Rename(fname, fname+".orig")
+	if err != nil {
+		t.Errorf("Renaming error: %s", err)
+	}
+	_, err = f.Write([]byte("after1\n"))
+	if err != nil {
+		t.Errorf("Write error: %s", err)
+	}
+
+	// Test that reopen always appends
+	err = f.Reopen()
+	if err != nil {
+		t.Errorf("Got reopen error %s: %s", fname, err)
+	}
+	_, err = f.Write([]byte("line2\n"))
+	if err != nil {
+		t.Errorf("Got write error2 on %s: %s", fname, err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Errorf("Got closing error for %s: %s", fname, err)
+	}
+
+	out, err := ioutil.ReadFile(fname)
+	if err != nil {
+		t.Fatalf("Unable read in final file %s: %s", fname, err)
+	}
+	outstr := string(out)
+	if outstr != "line2\n" {
+		t.Errorf("Result was %s", outstr)
+	}
+}

logger/logger.go

@@ -105,3 +105,27 @@ func (cl *CaptureLogger) Printf(format string, v ...interface{}) {
 func (cl *CaptureLogger) Debugf(format string, v ...interface{}) {
 	cl.Debugs = append(cl.Debugs, fmt.Sprintf(format, v...))
 }
+
+// Logfer is a thing that has only a Logf() method, like for instance,
+// testing.T or testing.B.
+type Logfer interface {
+	Logf(format string, v ...interface{})
+}
+
+// LogfLogger is a logger that wraps something that has a Logf interface
+// and makes it act like our logger.
+type LogfLogger struct {
+	wrapped Logfer
+}
+
+func (ll *LogfLogger) Printf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func (ll *LogfLogger) Debugf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func NewLogfLogger(l Logfer) *LogfLogger {
+	return &LogfLogger{wrapped: l}
+}

metrics.go

@@ -0,0 +1,68 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+const (
+	MetricCreateIndex                     = "create_index_total"
+	MetricDeleteIndex                     = "delete_index_total"
+	MetricCreateField                     = "create_field_total"
+	MetricDeleteField                     = "delete_field_total"
+	MetricDeleteAvailableShard            = "delete_available_shard_total"
+	MetricRecalculateCache                = "recalculate_cache_total"
+	MetricInvalidateCache                 = "invalidate_cache_total"
+	MetricRankCacheLength                 = "rank_cache_length"
+	MetricCacheThresholdReached           = "cache_threshold_reached_total"
+	MetricRow                             = "query_row_total"
+	MetricRowBSI                          = "query_row_bsi_total"
+	MetricSetRowAttrs                     = "query_setrowattrs_total"
+	MetricSetColumnAttrs                  = "query_setcolumnattrs_total"
+	MetricSetBit                          = "set_bit_total"
+	MetricClearBit                        = "clear_bit_total"
+	MetricImportingN                      = "importing_total"
+	MetricImportedN                       = "imported_total"
+	MetricClearingN                       = "clearing_total"
+	MetricClearedN                        = "cleared_total"
+	MetricSnapshotDurationSeconds         = "snapshot_duration_seconds"
+	MetricBlockRepair                     = "block_repair_total"
+	MetricSyncFieldDurationSeconds        = "sync_field_duration_seconds"
+	MetricSyncIndexDurationSeconds        = "sync_index_duration_seconds"
+	MetricColumnAttrStoreBlocks           = "column_attr_store_blocks_total"
+	MetricColumnAttrDiff                  = "column_attr_diff_total"
+	MetricRowAttrStoreBlocks              = "row_attr_store_blocks_total"
+	MetricRowAttrDiff                     = "row_attr_diff_total"
+	MetricHTTPRequest                     = "http_request_duration_seconds"
+	MetricGRPCUnaryQueryDurationSeconds   = "grpc_request_pql_unary_query_duration_seconds"
+	MetricGRPCUnaryFormatDurationSeconds  = "grpc_request_pql_unary_format_duration_seconds"
+	MetricGRPCStreamQueryDurationSeconds  = "grpc_request_pql_stream_query_duration_seconds"
+	MetricGRPCStreamFormatDurationSeconds = "grpc_request_pql_stream_format_duration_seconds"
+	MetricMaxShard                        = "maximum_shard"
+	MetricAntiEntropy                     = "antientropy_total"
+	MetricAntiEntropyDurationSeconds      = "antientropy_duration_seconds"
+	MetricGarbageCollection               = "garbage_collection_total"
+	MetricGoroutines                      = "goroutines"
+	MetricOpenFiles                       = "open_files"
+	MetricHeapAlloc                       = "heap_alloc"
+	MetricHeapInuse                       = "heap_inuse"
+	MetricStackInuse                      = "stack_inuse"
+	MetricMallocs                         = "mallocs"
+	MetricFrees                           = "frees"
+	MetricTransactionStart                = "transaction_start"
+	MetricTransactionEnd                  = "trasaction_end"
+	MetricTransactionBlocked              = "transaction_blocked"
+	MetricExclusiveTransactionRequest     = "transaction_exclusive_request"
+	MetricExclusiveTransactionActive      = "transaction_exclusive_active"
+	MetricExclusiveTransactionEnd         = "trasaction_exclusive_end"
+	MetricExclusiveTransactionBlocked     = "transaction_exclusive_blocked"
+)

mmap_test.go

@@ -0,0 +1,102 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/syswrap"
+)
+
+type cv struct {
+	cols []uint64
+	vals []int64
+}
+
+func forceSnapshotsCheckMapping(t *testing.T) {
+	depth := uint(6)
+	f := mustOpenBSIFragment("i", "f", viewStandard, 0)
+	f.Logger = logger.NewLogfLogger(t)
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	err := f.Snapshot()
+	if err != nil {
+		t.Fatalf("initial snapshot error: %v", err)
+	}
+
+	values := make([]cv, 1024)
+	for i := range values {
+		cols := make([]uint64, 128)
+		vals := make([]int64, 128)
+		for j := range cols {
+			// pick values in the first 16 cols of each of the 16
+			// shards in a default shardwidth, so each set will
+			// probably change some values from the previous one.
+			cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16))
+			vals[j] = int64(rand.Int63n(1 << depth))
+		}
+		values[i] = cv{cols, vals}
+	}
+
+	// modify the original bitmap, until it causes a snapshot, which
+	// then invalidates the other map...
+	for i := 0; i < 32; i++ {
+		cv := values[i%len(values)]
+		// periodically force gc, so if we have a small pool of maps
+		// we'll go in and out of mapping mode
+		if i%5 == 0 {
+			runtime.GC()
+		}
+		err := f.importValue(cv.cols, cv.vals, depth, (i%3 == 1))
+		if err != nil {
+			t.Fatalf("importValue[%d]: %v", i, err)
+		}
+		err = f.Snapshot()
+		if err != nil {
+			t.Fatalf("snapshot[%d]: %v", i, err)
+		}
+	}
+}
+
+// This test should basically never fail, but it might if you were running
+// out of available mmaps. Which you can fake up by adding '&& false' to the test
+// in newGeneration in generation.go. So this is probably useless but it's
+// a failure mode we've been bitten by once...
+func TestMmapBehavior(t *testing.T) {
+	var changed bool
+	var original uint64
+	defer func() {
+		syswrap.SetMaxMapCount(original)
+	}()
+
+	for _, mmapMaxVal := range []uint64{0, 3} {
+		prev := syswrap.SetMaxMapCount(mmapMaxVal)
+		if !changed {
+			original = prev
+			changed = true
+		}
+		t.Run(fmt.Sprintf("maps%d", mmapMaxVal), func(t *testing.T) {
+			forceSnapshotsCheckMapping(t)
+		})
+	}
+}

mock/translator.go

@@ -16,6 +16,7 @@ package mock
 
 import (
 	"context"
+	"io"
 
 	"github.com/pilosa/pilosa/v2"
 )
@@ -25,6 +26,7 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil)
 type TranslateStore struct {
 	CloseFunc         func() error
 	MaxIDFunc         func() (uint64, error)
+	PartitionIDFunc   func() int
 	ReadOnlyFunc      func() bool
 	SetReadOnlyFunc   func(v bool)
 	TranslateKeyFunc  func(key string) (uint64, error)
@@ -43,6 +45,10 @@ func (s *TranslateStore) MaxID() (uint64, error) {
 	return s.MaxIDFunc()
 }
 
+func (s *TranslateStore) PartitionID() int {
+	return s.PartitionIDFunc()
+}
+
 func (s *TranslateStore) ReadOnly() bool {
 	return s.ReadOnlyFunc()
 }
@@ -75,6 +81,14 @@ func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa
 	return s.EntryReaderFunc(ctx, offset)
 }
 
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	return 0, nil
+}
+
+func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) {
+	return 0, nil
+}
+
 var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil)
 
 type TranslateEntryReader struct {

pilosa.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"encoding/json"
 	"regexp"
+	"time"
 
 	"github.com/pkg/errors"
 )
@@ -29,8 +30,11 @@ var (
 	ErrIndexExists   = errors.New("index already exists")
 	ErrIndexNotFound = errors.New("index not found")
 
+	ErrForeignIndexNotFound = errors.New("foreign index not found")
+
 	// ErrFieldRequired is returned when no field is specified.
 	ErrFieldRequired  = errors.New("field required")
+	ErrColumnRequired = errors.New("column required")
 	ErrFieldExists    = errors.New("field already exists")
 	ErrFieldNotFound  = errors.New("field not found")
 
@@ -44,11 +48,12 @@ var (
 	ErrBSIGroupValueTooHigh     = errors.New("bsigroup value too high")
 	ErrInvalidRangeOperation    = errors.New("invalid range operation")
 	ErrInvalidBetweenValue      = errors.New("invalid value for between operation")
+	ErrDecimalOutOfRange        = errors.New("decimal value out of range")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")
 
-	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters")
+	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
 	ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
 
 	// ErrFragmentNotFound is returned when a fragment does not exist.
@@ -62,13 +67,20 @@ var (
 	// we won't need this error at all by 2.0 though.
 	ErrClusterDoesNotOwnShard = errors.New("node does not own shard")
 
+	// ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match
+	ErrPreconditionFailed = errors.New("precondition failed")
+
 	ErrNodeIDNotExists    = errors.New("node with provided ID does not exist")
 	ErrNodeNotCoordinator = errors.New("node is not the coordinator")
 	ErrResizeNotRunning   = errors.New("no resize job currently running")
+	ErrResizeNoReplicas   = errors.New("not enough data to perform resize (replica factor may need to be increased)")
 
 	ErrNotImplemented            = errors.New("not implemented")
 	ErrFieldsArgumentRequired    = errors.New("fields argument required")
 	ErrExpectedFieldListArgument = errors.New("expected field list argument")
+
+	ErrIntFieldWithKeys     = errors.New("int field cannot be created with 'keys=true' option")
+	ErrDecimalFieldWithKeys = errors.New("decimal field cannot be created with 'keys=true' option")
 )
 
 // apiMethodNotAllowedError wraps an error value indicating that a particular
@@ -117,8 +129,17 @@ func newNotFoundError(err error) NotFoundError {
 	return NotFoundError{err}
 }
 
+type PreconditionFailedError struct {
+	error
+}
+
+// newPreconditionFailedError returns err wrapped in a PreconditionFailedError.
+func newPreconditionFailedError(err error) PreconditionFailedError {
+	return PreconditionFailedError{err}
+}
+
 // Regular expression to validate index and field names.
-var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
+var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
 
 // ColumnAttrSet represents a set of attributes for a vertical column in an index.
 // Can have a set of attributes attached to it.
@@ -184,6 +205,10 @@ func stringSlicesAreEqual(a, b []string) bool {
 	return true
 }
 
+func timestamp() int64 {
+	return time.Now().UnixNano()
+}
+
 // AddressWithDefaults converts addr into a valid address,
 // using defaults when necessary.
 func AddressWithDefaults(addr string) (*URI, error) {

pilosa_internal_test.go

@@ -21,7 +21,7 @@ import (
 func TestValidateName(t *testing.T) {
 	names := []string{
 		"a", "ab", "ab1", "b-c", "d_e", "exists",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"longbutnottoolongaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa12345689012345689012345678901234567890",
 	}
 	for _, name := range names {
 		if validateName(name) != nil {
@@ -33,7 +33,7 @@ func TestValidateName(t *testing.T) {
 func TestValidateNameInvalid(t *testing.T) {
 	names := []string{
 		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
+		"long123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
 	}
 	for _, name := range names {
 		if validateName(name) == nil {

pql/ast.go

@@ -17,10 +17,13 @@ package pql
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/molecula/ext"
 )
 
 // Query represents a PQL query.
@@ -59,7 +62,7 @@ func (q *Query) lastCallStackElem() *callStackElem {
 
 func (q *Query) addPosNum(key, value string) {
 	q.addField(key)
-	q.addNumVal(value)
+	q.addNumVal(value, false)
 }
 
 func (q *Query) addPosStr(key, value string) {
@@ -84,19 +87,26 @@ func (q *Query) endConditional() {
 	if len(q.conditional) != 5 {
 		panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional))
 	}
-	low, _ := strconv.ParseInt(q.conditional[0], 10, 64)
+	low := parseNum(q.conditional[0], false)
 	field := q.conditional[2]
-	high, _ := strconv.ParseInt(q.conditional[4], 10, 64)
+	high := parseNum(q.conditional[4], false)
 
-	if q.conditional[1] == "<" {
+	var op Token
-		low++
+	switch q.conditional[1] + q.conditional[3] {
-	}
+	case "<<":
-	if q.conditional[3] == "<" {
+		op = BTWN_LT_LT
-		high--
+	case "<=<":
+		op = BTWN_LTE_LT
+	case "<<=":
+		op = BTWN_LT_LTE
+	case "<=<=":
+		op = BETWEEN
+	default:
+		panic(fmt.Sprintf("impossible conditional ops: '%s' and '%s'", q.conditional[1], q.conditional[3]))
 	}
 
 	elem := q.lastCallStackElem()
-	elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
+	elem.call.Args[field] = &Condition{Op: op, Value: []interface{}{low, high}}
 
 	q.conditional = nil
 }
@@ -124,6 +134,13 @@ func (q *Query) validateArgField(elem *callStackElem) {
 }
 
 func (q *Query) addVal(val interface{}) {
+	if vs, ok := val.(string); ok {
+		vsu, err := Unquote(vs)
+		if err != nil {
+			panic(err)
+		}
+		val = vsu
+	}
 	elem := q.lastCallStackElem()
 	if elem == nil || elem.lastField == "" {
 		panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))
@@ -147,21 +164,12 @@ func (q *Query) addVal(val interface{}) {
 	elem.lastCond = ILLEGAL
 }
 
-func (q *Query) addNumVal(val string) {
+func (q *Query) addNumVal(val string, asFloat bool) {
 	elem := q.lastCallStackElem()
 	if elem == nil || elem.lastField == "" {
 		panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
 	}
-	var ival interface{}
+	ival := parseNum(val, asFloat)
-	var err error
-	if strings.Contains(val, ".") {
-		ival, err = strconv.ParseFloat(val, 64)
-	} else {
-		ival, err = strconv.ParseInt(val, 10, 64)
-	}
-	if err != nil {
-		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
-	}
 	if elem.inList {
 		if elem.lastCond != ILLEGAL {
 			list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{})
@@ -259,11 +267,268 @@ type callStackElem struct {
 	inList    bool
 }
 
-// Call represents a function call in the AST.
+// Some call types may require special handling, which needs to occur
+// before distributing processing to individual shards.
+type CallType byte
+
+const (
+	// Normal calls can be executed per shard.
+	PrecallNone = CallType(iota)
+	// PreCallGlobal indicates a call which must be run globally *before*
+	// distributing the call to other shards. Example: A Distinct query,
+	// where every shard could potentially produce results for any shard,
+	// so you have to produce the results up front.
+	PrecallGlobal
+	// PreCallPerNode indicates a call which needs to be run per-shard
+	// in a way that lets it be done on each shard, but where it should
+	// be done prior to spawning per-shard goroutines. Example:
+	// A cross-index query, where each local shard may or may not need
+	// to get data from a remote node, but batches of shards can
+	// probably be gotten from the same remote node.
+	PrecallPerNode
+)
+
+// Call represents a function call in the AST. The Precomputed field
+// is used by the executor to handle non-standard call types; it does
+// these by actually executing them separately, then replacing them
+// in the call tree with a new call using the special precomputed
+// type, with the Precomputed field set to a map from shards to results.
 type Call struct {
 	Name        string
 	Args        map[string]interface{}
 	Children    []*Call
+	Type        CallType
+	Precomputed map[uint64]interface{}
+}
+
+// callInfo defines the arguments allowed for a particular PQL call, and
+// possibly things about its semantics. If allowUnknown is true, unfamiliar
+// non-reserved names are allowed on the assumption that they're field names.
+// Otherwise, only those names explicitly listed are allowed. Reserved args
+// (those with a leading underscore) are never allowed unless explicitly
+// present.
+//
+// The prototypes map maps from argument names to a value. If the value is
+// non-nil, the argument will be checked for type-matching. So, for instance,
+// `x: 10` would indicate that x must be an int.
+type callInfo struct {
+	allowUnknown bool
+	prototypes   map[string]interface{}
+	callType     CallType
+}
+
+// We want to be able to accept either a string or int64 for
+// field names. Special-case type:
+type stringOrInt64Type struct{}
+
+var stringOrInt64 stringOrInt64Type
+
+var allowUnderField = callInfo{
+	allowUnknown: true,
+	prototypes: map[string]interface{}{
+		"_field": "",
+	},
+}
+
+var allowField = callInfo{
+	allowUnknown: false,
+	prototypes: map[string]interface{}{
+		"field": "",
+	},
+}
+
+var callInfoByFunc = map[string]callInfo{
+	// the easy cases: things that take arbitrary inputs, because they're
+	// taking field=value cases
+	"Bitmap": {allowUnknown: true},
+	"Count":  {allowUnknown: true},
+	"Row":    {allowUnknown: true},
+	"Range":  {allowUnknown: true},
+
+	"Distinct": {allowUnknown: true},
+
+	// allow only "field=X" cases with string field names
+	"Max": allowField,
+	"Min": allowField,
+	"Sum": allowField,
+
+	// only take other calls, should never have "args"
+	"Difference": {allowUnknown: false},
+	"Intersect":  {allowUnknown: false},
+	"Not":        {allowUnknown: false},
+	"FieldValue": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"field":  "",
+			"column": stringOrInt64,
+		},
+	},
+	"All": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"limit":  int64(0),
+			"offset": int64(0),
+		},
+	},
+	"ClearRow": {allowUnknown: true},
+	"Store":    {allowUnknown: true},
+	"MinRow":   allowField,
+	"MaxRow":   allowField,
+	"Rows": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"_field":   "",
+			"field":    "",
+			"limit":    int64(0),
+			"column":   nil,
+			"previous": nil,
+			"from":     nil,
+			"to":       nil,
+		},
+	},
+	"Shift": {allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"n": int64(0),
+		},
+	},
+	"Union": {allowUnknown: false},
+	"Xor":   {allowUnknown: false},
+
+	// things that take _field
+	"TopN": allowUnderField,
+	// special cases:
+	"Clear": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"GroupBy": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"filter":    nil,
+			"limit":     int64(0),
+			"previous":  nil,
+			"aggregate": nil,
+			"having":    nil,
+		},
+	},
+	"Options": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"excludeRowAttrs": true,
+			"excludeColumns":  true,
+			"columnAttrs":     true,
+			"shards":          nil,
+		},
+	},
+	"Set": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col":       stringOrInt64,
+			"_timestamp": "",
+		},
+	},
+	"Precomputed": {
+		allowUnknown: true,
+	},
+	"SetBit": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"SetRowAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_row":   stringOrInt64,
+		},
+	},
+	"SetColumnAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_col":   stringOrInt64,
+		},
+	},
+	"IncludesColumn": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"column": stringOrInt64,
+		},
+	},
+}
+
+// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
+// arg validation.
+func RegisterPluginFuncs(ops []ext.BitmapOp) {
+	for _, op := range ops {
+		// ignore overlap for now. This should change.
+		if _, ok := callInfoByFunc[op.Name]; ok {
+			continue
+		}
+		ci := callInfo{allowUnknown: true}
+		if len(op.Reserved) > 0 {
+			// mark these as valid/known reserved words
+			ci.prototypes = make(map[string]interface{})
+			for _, res := range op.Reserved {
+				ci.prototypes[res] = nil
+			}
+		}
+		t := op.Func.BitmapOpType()
+		if t.Precall == ext.OpPrecallGlobal {
+			ci.callType = PrecallGlobal
+		}
+		callInfoByFunc[op.Name] = ci
+	}
+}
+
+// CheckCallInfo tries to validate that arguments are correct and valid for the
+// given call. It does not guarantee checking all possible errors; for instance,
+// if an argument is a field name, CheckCallInfo can't validate that the field
+// exists. It also updates with information like whether the call is expected
+// to require precalling.
+func (c *Call) CheckCallInfo() error {
+	valid, ok := callInfoByFunc[c.Name]
+	if !ok {
+		return fmt.Errorf("no arg validation for '%s'", c.Name)
+	}
+	c.Type = valid.callType
+	for k, v := range c.Args {
+		acceptable, ok := valid.prototypes[k]
+		if !ok && !valid.allowUnknown {
+			return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
+		}
+		if !ok && strings.HasPrefix(k, "_") {
+			return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
+		}
+		if acceptable == nil {
+			continue
+		}
+		// if the types are identical, that's fine
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
+			continue
+		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
+			switch v.(type) {
+			case string, int64:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
+					c.String(), k, v)
+			}
+		}
+		return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
+			c.String(), k, v, acceptable)
+	}
+	// call-specific checking
+	for _, child := range c.Children {
+		if err := child.CheckCallInfo(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // FieldArg determines which key-value pair contains the field and rowID,
@@ -283,13 +548,36 @@ func IsReservedArg(name string) bool {
 		return true
 	}
 	switch name {
-	case "from", "to":
+	case "from", "to", "index":
 		return true
 	default:
 		return false
 	}
 }
 
+// CallIndex handles guessing whether we've been asked to apply this to a
+// different index. An empty string means "no".
+func (c *Call) CallIndex() string {
+	if index, ok := c.Args["_index"]; ok {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	if index, ok := c.Args["index"]; ok && index != "" {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	return ""
+}
+
+// Arg is for reading the value at key from call.Args.
+// If the key is not in Call.Args, the value of the returned bool will be false.
+func (c *Call) Arg(key string) (interface{}, bool) {
+	v, ok := c.Args[key]
+	return v, ok
+}
+
 // BoolArg is for reading the value at key from call.Args as a bool. If the
 // key is not in Call.Args, the value of the returned bool will be false, and
 // the error will be nil. The value is assumed to be a bool. An error is
@@ -416,6 +704,11 @@ func (c *Call) Clone() *Call {
 			other.Children[i] = c.Children[i].Clone()
 		}
 	}
+	// @seebs "...it should be safe,
+	// because nothing should be writing to Precomputed
+	// once it's gotten created in the first place."
+	other.Precomputed = c.Precomputed
+
 	return other
 }
 
@@ -455,7 +748,7 @@ func (c *Call) String() string {
 		// the equal sign in the string representation.
 		switch v := c.Args[key].(type) {
 		case *Condition:
-			fmt.Fprintf(&buf, "%v %s", key, v.String())
+			fmt.Fprintf(&buf, "%s", v.StringWithSubj(key))
 		default:
 			fmt.Fprintf(&buf, "%v=%s", key, formatValue(v))
 		}
@@ -477,6 +770,36 @@ func (c *Call) HasConditionArg() bool {
 	return false
 }
 
+// TranslateInfo returns the relevant translation fields.
+func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) {
+	switch c.Name {
+	case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow":
+		// Positional args in new PQL syntax require special handling here.
+		fieldName, _ = c.FieldArg()
+		return "_" + columnLabel, fieldName, fieldName
+	case "SetRowAttrs":
+		// Positional args in new PQL syntax require special handling here.
+		return "", "_" + rowLabel, c.ArgString("_field")
+	case "Rows":
+		return "column", "previous", c.ArgString("_field")
+	case "IncludesColumn":
+		return "column", "", ""
+	case "GroupBy":
+		return "", "", ""
+	default:
+		return "col", "row", c.ArgString("_field")
+	}
+}
+
+func (c *Call) ArgString(key string) string {
+	value, ok := c.Args[key]
+	if !ok {
+		return ""
+	}
+	s, _ := value.(string)
+	return s
+}
+
 // Condition represents an operation & value.
 // When used in an argument map it represents a binary expression.
 type Condition struct {
@@ -489,10 +812,82 @@ func (cond *Condition) String() string {
 	return fmt.Sprintf("%s%s", cond.Op.String(), formatValue(cond.Value))
 }
 
-// IntSliceValue reads cond.Value as a slice of uint64.
+// StringWithSubj returns the string representation of the condition
-// If the value is a slice of uint64 it will convert
+// including the provided subject.
-// it to []int64. Otherwise, if it is not a []int64 it will return an error.
+func (cond *Condition) StringWithSubj(subj string) string {
-func (cond *Condition) IntSliceValue() ([]int64, error) {
+	switch cond.Op {
+	case EQ, NEQ, LT, LTE, GT, GTE:
+		return fmt.Sprintf("%s%s", subj, cond.String())
+	case BETWEEN, BTWN_LT_LTE, BTWN_LTE_LT, BTWN_LT_LT:
+		val, ok := cond.StringSliceValue()
+		if !ok || len(val) < 2 {
+			return ""
+		}
+		if cond.Op == BETWEEN {
+			return fmt.Sprintf("%s<=%s<=%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LTE {
+			return fmt.Sprintf("%s<%s<=%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LTE_LT {
+			return fmt.Sprintf("%s<=%s<%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LT {
+			return fmt.Sprintf("%s<%s<%s", val[0], subj, val[1])
+		}
+	}
+	return ""
+}
+
+func (cond *Condition) Uint64Value() (uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		if tval >= 0 {
+			return uint64(tval), true
+		}
+	case uint64:
+		return tval, true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Uint64SliceValue() ([]uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]uint64, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = uint64(tv)
+			case uint64:
+				ret[i] = tv
+			default:
+				return nil, false
+			}
+		}
+		return ret, true
+	}
+
+	return nil, false
+}
+
+func (cond *Condition) Int64Value() (int64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		return tval, true
+	case uint64:
+		// TODO: consider overflow?
+		return int64(tval), true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Int64SliceValue() ([]int64, bool) {
 	val := cond.Value
 
 	switch tval := val.(type) {
@@ -505,17 +900,48 @@ func (cond *Condition) IntSliceValue() ([]int64, error) {
 			case uint64:
 				ret[i] = int64(tv)
 			default:
-				return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv)
+				return nil, false
 			}
 		}
-		return ret, nil
+		return ret, true
+	}
+
+	return nil, false
+}
+
+// StringSliceValue returns the value(s) of the conditional
+// as a slice of strings. For example, if cond.Value is
+// []int64{-10,20}, this will return []string{"-10","20"}.
+// It also returns a bool indicating that the conversion
+// succeeded.
+func (cond *Condition) StringSliceValue() ([]string, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]string, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = strconv.FormatInt(tv, 10)
+			case uint64:
+				ret[i] = strconv.FormatUint(tv, 10)
+			case Decimal:
+				ret[i] = tv.String()
 			default:
-		return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval)
+				return nil, false
 			}
 		}
+		return ret, true
+	}
+
+	return nil, false
+}
 
 func formatValue(v interface{}) string {
 	switch v := v.(type) {
+	case nil:
+		return "null"
 	case string:
 		return fmt.Sprintf("%q", v)
 	case []interface{}:
@@ -560,3 +986,21 @@ func joinUint64Slice(a []uint64) string {
 	}
 	return "[" + strings.Join(other, ",") + "]"
 }
+
+func parseNum(val string, asFloat bool) interface{} {
+	var ival interface{}
+	var err error
+	if strings.Contains(val, ".") {
+		if asFloat {
+			ival, err = strconv.ParseFloat(val, 64)
+		} else {
+			ival, err = ParseDecimal(val)
+		}
+	} else {
+		ival, err = strconv.ParseInt(val, 10, 64)
+	}
+	if err != nil {
+		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
+	}
+	return ival
+}

pql/ast_test.go

@@ -15,7 +15,6 @@
 package pql_test
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
@@ -43,27 +42,26 @@ func TestCall_String(t *testing.T) {
 	})
 }
 
-// Ensure condition can handle values for BETWEEN operator.
+// Ensure condition string with subject is correct.
-func TestCondition_Value(t *testing.T) {
+func TestCondition_StringWithSubj(t *testing.T) {
-	t.Run("Between Values", func(t *testing.T) {
+	subj := "subj"
 	for _, tt := range []struct {
-			val []interface{}
+		op  pql.Token
-			exp []int64
+		val interface{}
+		exp string
 	}{
-			{[]interface{}{int64(4), int64(8)}, []int64{4, 8}},
+		{pql.BETWEEN, []interface{}{int64(4), int64(8)}, "4<=subj<=8"},
-			{[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}},
+		{pql.BETWEEN, []interface{}{uint64(5), uint64(9)}, "5<=subj<=9"},
-			{[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}},
+		{pql.BETWEEN, []interface{}{pql.Decimal{Value: -401, Scale: 2}, pql.Decimal{Value: 802, Scale: 1}}, "-4.01<=subj<=80.2"},
+		{pql.EQ, nil, "subj==null"},
+		{pql.NEQ, nil, "subj!=null"},
 	} {
 		c := &pql.Condition{
-				Op:    pql.BETWEEN,
+			Op:    tt.op,
 			Value: tt.val,
 		}
-			v, err := c.IntSliceValue()
+		if sws := c.StringWithSubj(subj); sws != tt.exp {
-			if err != nil {
+			t.Fatalf("invalid between string. expected: %s, got %s", tt.exp, sws)
-				t.Fatal(err)
-			} else if !reflect.DeepEqual(v, tt.exp) {
-				t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v)
 		}
 	}
-	})
 }

pql/decimal.go

@@ -0,0 +1,547 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pql
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// pow10 is a map used to avoid the float64 required by math.Pow10()
+var pow10 = map[int64]int64{
+	0:  1,
+	1:  10,
+	2:  100,
+	3:  1000,
+	4:  10000,
+	5:  100000,
+	6:  1000000,
+	7:  10000000,
+	8:  100000000,
+	9:  1000000000,
+	10: 10000000000,
+	11: 100000000000,
+	12: 1000000000000,
+	13: 10000000000000,
+	14: 100000000000000,
+	15: 1000000000000000,
+	16: 10000000000000000,
+	17: 100000000000000000,
+	18: 1000000000000000000,
+	//19: 10000000000000000000,
+}
+
+// Pow10 is a function which can be used in place of math.Pow10()
+// to avoid the float64 logic. Note that only powers 0-18 are
+// currently supported; anything else will return 0, which is
+// probably going to result in incorrect values.
+func Pow10(p int64) int64 {
+	return pow10[p]
+}
+
+// Decimal represents a decimal value; the intention
+// is to avoid relying on float64, and the primary
+// purpose is to have a predictable way to encode such
+// values used in query strings.
+// Scale is the number of digits to the right of the
+// decimal point.
+// Precision is currently not considered; precision, for
+// our purposes is implied to be the complete, known value.
+type Decimal struct {
+	Value int64
+	Scale int64
+}
+
+// NewDecimal returns a Decimal based on the provided arguments.
+func NewDecimal(value, scale int64) Decimal {
+	return Decimal{
+		Value: value,
+		Scale: scale,
+	}
+}
+
+// MinMax returns the minimum and maximum values
+// supported by the provided scale.
+func MinMax(scale int64) (Decimal, Decimal) {
+	min := NewDecimal(math.MinInt64, scale)
+	max := NewDecimal(math.MaxInt64, scale)
+	return min, max
+}
+
+// LessThan returns true if d < d2.
+func (d Decimal) LessThan(d2 Decimal) bool {
+	return d.lessThan(d2, false)
+}
+
+// LessThanOrEqualTo returns true if d <= d2.
+func (d Decimal) LessThanOrEqualTo(d2 Decimal) bool {
+	return d.lessThan(d2, true)
+}
+
+// GreaterThan returns true if d > d2.
+func (d Decimal) GreaterThan(d2 Decimal) bool {
+	return d.greaterThan(d2, false)
+}
+
+// GreaterThanOrEqualTo returns true if d >= d2.
+func (d Decimal) GreaterThanOrEqualTo(d2 Decimal) bool {
+	return d.greaterThan(d2, true)
+}
+
+// EqualTo returns true if d == d2.
+func (d Decimal) EqualTo(d2 Decimal) bool {
+	if d.Scale == d2.Scale {
+		return d.Value == d2.Value
+	}
+
+	quotientD := quotient(d)
+	quotientD2 := quotient(d2)
+	if quotientD != quotientD2 {
+		return false
+	}
+	remainderD, remainderD2 := remainder(d), remainder(d2)
+	if d.Scale < d2.Scale {
+		scaleDiff := d2.Scale - d.Scale
+		return (remainderD * pow10[scaleDiff]) == remainderD2
+	}
+	scaleDiff := d.Scale - d2.Scale
+	return remainderD == (remainderD2 * pow10[scaleDiff])
+}
+
+func (d Decimal) lessThan(d2 Decimal, eq bool) bool {
+	if d.Scale == d2.Scale {
+		if eq {
+			return d.Value <= d2.Value
+		}
+		return d.Value < d2.Value
+	}
+
+	quotientD, quotientD2 := quotient(d), quotient(d2)
+	if quotientD < quotientD2 {
+		return true
+	} else if quotientD == quotientD2 {
+		remainderD, remainderD2 := remainder(d), remainder(d2)
+		if d.Scale < d2.Scale {
+			scaleDiff := d2.Scale - d.Scale
+			if eq {
+				return (remainderD * pow10[scaleDiff]) <= remainderD2
+			}
+			return (remainderD * pow10[scaleDiff]) < remainderD2
+		}
+		scaleDiff := d.Scale - d2.Scale
+		if eq {
+			return remainderD <= (remainderD2 * pow10[scaleDiff])
+		}
+		return remainderD < (remainderD2 * pow10[scaleDiff])
+	}
+
+	return false
+}
+
+func (d Decimal) greaterThan(d2 Decimal, eq bool) bool {
+	if d.Scale == d2.Scale {
+		if eq {
+			return d.Value >= d2.Value
+		}
+		return d.Value > d2.Value
+	}
+
+	quotientD, quotientD2 := quotient(d), quotient(d2)
+	if quotientD > quotientD2 {
+		return true
+	} else if quotientD == quotientD2 {
+		remainderD, remainderD2 := remainder(d), remainder(d2)
+		if d.Scale < d2.Scale {
+			scaleDiff := d2.Scale - d.Scale
+			if eq {
+				return (remainderD * pow10[scaleDiff]) >= remainderD2
+			}
+			return (remainderD * pow10[scaleDiff]) > remainderD2
+		}
+		scaleDiff := d.Scale - d2.Scale
+		if eq {
+			return remainderD >= (remainderD2 * pow10[scaleDiff])
+		}
+		return remainderD > (remainderD2 * pow10[scaleDiff])
+	}
+
+	return false
+}
+
+// SupportedByScale returns true if d can be represented
+// by a decimal based on scale.
+// For example:
+// scale = 2:
+// min: -92233720368547758.08
+// max:  92233720368547758.07
+// would not support: NewDecimal(9223372036854775807, 0)
+func (d Decimal) SupportedByScale(scale int64) bool {
+	min, max := MinMax(scale)
+
+	if d.GreaterThanOrEqualTo(min) && d.LessThanOrEqualTo(max) {
+		return true
+	}
+	return false
+}
+
+// IsValid returns true if the decimal does not break
+// any assumption or resrictions on input.
+func (d Decimal) IsValid() bool {
+	if d.Scale < -18 || d.Scale > 19 {
+		return false
+	}
+	return true
+}
+
+// ToInt64 returns d as an int64 adjusted to the
+// provided scale.
+func (d Decimal) ToInt64(scale int64) int64 {
+	var ret int64
+	scaleDiff := scale - d.Scale
+	if scaleDiff == 0 {
+		ret = d.Value
+	} else if scaleDiff < 0 {
+		ret = d.Value / Pow10(-1*scaleDiff)
+	} else {
+		ret = d.Value * Pow10(scaleDiff)
+	}
+	return ret
+}
+
+// Float64 returns d as a float64.
+// TODO: this could potentially lose precision; we should audit
+// its use and protect against unexpected results.
+func (d Decimal) Float64() float64 {
+	var ret float64
+	if d.Scale == 0 {
+		ret = float64(d.Value)
+	} else {
+		ret = float64(d.Value) / math.Pow10(int(d.Scale))
+	}
+	return ret
+}
+
+// String returns the string representation of the decimal.
+func (d Decimal) String() string {
+	var s string
+
+	var neg bool
+	sval := fmt.Sprintf("%d", d.Value)
+
+	// Strip the negative sign off for now, and
+	// re-apply it at the end.
+	if sval[0] == '-' {
+		neg = true
+		sval = sval[1:]
+	}
+
+	if d.Scale == 0 {
+		s = sval
+	} else if d.Scale < 0 {
+		s = sval + strings.Repeat("0", int(-1*d.Scale))
+	} else {
+		var bufLen int
+		if int(d.Scale) < len(sval) {
+			bufLen = len(sval) + 1
+		} else {
+			bufLen = int(d.Scale) + 2
+		}
+
+		buf := make([]byte, bufLen)
+		j := 0
+		for i := range buf {
+			z := len(buf) - 1 - i // index into buf from the end
+			if i == int(d.Scale) {
+				buf[z] = '.'
+				continue
+			}
+			if len(sval) > j {
+				buf[z] = sval[len(sval)-1-j]
+				j++
+			} else {
+				buf[z] = '0'
+			}
+		}
+		s = string(buf)
+	}
+
+	if neg {
+		return "-" + s
+	}
+	return s
+}
+
+const (
+	stateSign         = "sign"
+	stateLeadingZeros = "zeros"
+	stateMantissa     = "mantissa"
+)
+
+// ParseDecimal parses a string into a Decimal.
+func ParseDecimal(s string) (Decimal, error) {
+	var sign bool
+	var value int64
+	var scale int64
+	var err error
+
+	// General steps:
+	// - Trim leading whitespace/zeros
+	// - Get the sign value
+	// - Trim leading zeros
+	// - Push characters into a buffer
+	// - Track position of decimal point
+	// - Trim trailing zeros of buffer
+	// - value = buffer -> int
+	// - scale = len(buffer) - tracked position
+
+	var decimalPos int = -1
+	var pos int
+	mantissa := make([]byte, len(s))
+
+	state := stateSign
+	var foundLeadingZero bool
+	for i := 0; i < len(s); i++ {
+		switch state {
+		case stateSign:
+			switch s[i] {
+			case ' ':
+				continue
+			case '-':
+				sign = true
+				fallthrough
+			case '+':
+				state = stateLeadingZeros
+			default:
+				state = stateLeadingZeros
+				i--
+			}
+		case stateLeadingZeros:
+			switch s[i] {
+			case '0':
+				foundLeadingZero = true
+				continue
+			default:
+				state = stateMantissa
+				i--
+			}
+		case stateMantissa:
+			switch s[i] {
+			case '.':
+				if decimalPos == -1 {
+					decimalPos = pos
+				} else {
+					return Decimal{}, errors.Errorf("invalid decimal string: %s", s)
+				}
+				continue
+			default:
+				mantissa[pos] = s[i]
+				pos++
+			}
+		}
+	}
+
+	// If we've gotten here and state is still in stateSign or
+	// it's in stateLeadingZeros without finding any zeros,
+	// it means no value was provided.
+	if state == stateSign || (state == stateLeadingZeros && !foundLeadingZero) {
+		return Decimal{}, errors.New("decimal string is empty")
+	}
+
+	// Trim trailing zeros/spaces of mantissa
+	// for any portion that would have been to the
+	// right of the decimal.
+	trimZeroCnt := 0
+	trimSpaceCnt := 0
+	for i := len(mantissa) - 1; i >= 0; i-- {
+		switch mantissa[i] {
+		case uint8(0), uint8(32): // nil/space
+			trimSpaceCnt++
+			continue
+		case uint8(48): // zero
+			trimZeroCnt++
+			continue
+		}
+		break
+	}
+	mantissa = mantissa[:len(mantissa)-trimSpaceCnt-trimZeroCnt]
+
+	// Based on where (or if) the decimal was found,
+	// calculate scale.
+	if decimalPos == -1 {
+		scale = -1 * int64(trimZeroCnt)
+	} else {
+		scale = int64(len(mantissa) - decimalPos)
+	}
+
+	// If mantissa is empty, treat it as "0".
+	if len(mantissa) == 0 {
+		mantissa = []byte{'0'}
+		sign = false
+		scale = 0
+	}
+
+	// If the mantissa can't be represented by an int64, but it contains
+	// enough decimal places such that we can sacrifice precision, then
+	// we do that. This is an attempt to be compatible with the way
+	// `strconv.ParseFloat` works.
+	if m, s, ok := reducePrecision(sign, mantissa, scale); ok {
+		mantissa = m
+		scale = s
+	} else {
+		return Decimal{}, errors.Errorf("value out of range: %s", mantissa)
+	}
+
+	// We have to use ParseUint here (as opposed to ParseInt) because
+	// math.MinInt64 is a valid value, but its absolute value is not.
+	// So this allows us to handle that one value without overflow, and
+	// then we check for the uint bounds in the next step.
+	uvalue, err := strconv.ParseUint(string(mantissa), 10, 64)
+	if err != nil {
+		return Decimal{}, errors.Wrap(err, "converting mantissa string to uint64")
+	}
+
+	if (sign && uvalue > -1*math.MinInt64) || (!sign && uvalue > math.MaxInt64) {
+		return Decimal{}, errors.New("value out of range")
+	}
+	value = int64(uvalue)
+
+	if sign {
+		value *= -1
+	}
+
+	return Decimal{
+		Value: value,
+		Scale: scale,
+	}, nil
+}
+
+// reducePrecision takes a []byte mantissa and scale, and if possible
+// will adjust the mantissa (by reducing precision) until it can be
+// represented by an int64. The returned bool indicates whether the
+// reduction was successful.
+func reducePrecision(sign bool, mantissa []byte, scale int64) ([]byte, int64, bool) {
+	// Trim leading zeros before considering length.
+	var zeroIdx int
+	for i := range mantissa {
+		if mantissa[i] == '0' {
+			zeroIdx++
+		} else {
+			break
+		}
+	}
+	mantissa = mantissa[zeroIdx:]
+
+	// If we zero out the mantissa to an empty
+	// string, that means it's value should be 0.
+	if len(mantissa) == 0 {
+		mantissa = []byte{'0'}
+		return mantissa, scale, true
+	}
+
+	lenMantissa := len(mantissa)
+	maxStr := "9223372036854775807"
+	if sign {
+		maxStr = "9223372036854775808"
+	}
+
+	if lenMantissa <= 18 || (lenMantissa == 19 && string(mantissa) <= maxStr) {
+		return mantissa, scale, true
+	}
+
+	// If we don't have any decimal places to sacrifice,
+	// we can't change anything.
+	if scale <= 0 {
+		return mantissa, scale, false
+	}
+
+	return reducePrecision(sign, mantissa[:len(mantissa)-1], scale-1)
+}
+
+func quotient(d Decimal) int64 {
+	if d.Scale == 0 {
+		return d.Value
+	} else if d.Scale > 0 && d.Scale < 19 {
+		return d.Value / pow10[d.Scale]
+	} else if d.Scale < 0 && d.Scale > -19 {
+		return d.Value * pow10[-1*d.Scale]
+	}
+	return 0
+}
+
+func remainder(d Decimal) int64 {
+	if d.Scale >= 0 && d.Scale < 19 {
+		return d.Value % pow10[d.Scale]
+	}
+	return 0
+}
+
+// UnmarshalJSON is a custom unmarshaller for the Decimal
+// type. The intention is to avoid the use of float64
+// anywhere, so this unmarhaller parses the decimal out
+// of the byte string.
+func (d *Decimal) UnmarshalJSON(data []byte) error {
+	o, err := ParseDecimal(string(data))
+	if err != nil {
+		return errors.Wrapf(err, "parsing decimal: %s", string(data))
+	}
+	d.Value = o.Value
+	d.Scale = o.Scale
+
+	return nil
+}
+
+// MarshalJSON is a custom marshaller for the Decimal type.
+func (d Decimal) MarshalJSON() ([]byte, error) {
+	return []byte(d.String()), nil
+}
+
+// UnmarshalYAML is a custom unmarshaller for the Decimal
+// type.
+func (d *Decimal) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var data string
+	if err := unmarshal(&data); err != nil {
+		return err
+	}
+
+	o, err := ParseDecimal(data)
+	if err != nil {
+		return errors.Wrapf(err, "parsing decimal: %s", data)
+	}
+	d.Value = o.Value
+	d.Scale = o.Scale
+
+	return nil
+}
+
+// MarshalYAML is a custom marshaller for the Decimal type.
+func (d Decimal) MarshalYAML() (interface{}, error) {
+	// TODO: I don't love that this results in a quoted string
+	// in the yaml document:
+	//
+	// min: "-100.05"
+	//
+	// It would be nice if we could get that to result in:
+	//
+	// min: -100.05
+	//
+	// Note that we _can_ do that by casting the output as
+	// float64 (for certain cases), but the whole point of
+	// Decimal is to avoid using float64.
+	return d.String(), nil
+}

pql/decimal_test.go

@@ -0,0 +1,262 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pql_test
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/pql"
+)
+
+// Ensure call can be converted into a string.
+func TestDecimal(t *testing.T) {
+	t.Run("Parse", func(t *testing.T) {
+		tests := []struct {
+			s      string
+			exp    pql.Decimal
+			expErr string
+		}{
+			{"0", pql.Decimal{0, 0}, ""},
+			{"-0", pql.Decimal{0, 0}, ""},
+			{"0.0", pql.Decimal{0, 0}, ""},
+			{"-0.00", pql.Decimal{0, 0}, ""},
+			{"123.4567", pql.Decimal{1234567, 4}, ""},
+			{"  123.4567", pql.Decimal{1234567, 4}, ""},
+			{"  123.4567  ", pql.Decimal{1234567, 4}, ""},
+			{"123.456700", pql.Decimal{1234567, 4}, ""},
+			{"00123.4567", pql.Decimal{1234567, 4}, ""},
+			{"+123.4567", pql.Decimal{1234567, 4}, ""},
+			{"-123.4567", pql.Decimal{-1234567, 4}, ""},
+			{"-00123.4567", pql.Decimal{-1234567, 4}, ""},
+			{"-12.25", pql.Decimal{-1225, 2}, ""},
+
+			{"123", pql.Decimal{123, 0}, ""},
+			{"-12300", pql.Decimal{-123, -2}, ""},
+			{"+012300", pql.Decimal{123, -2}, ""},
+			{"12300", pql.Decimal{123, -2}, ""},
+			{"12300.", pql.Decimal{123, -2}, ""},
+			{"12300.0", pql.Decimal{123, -2}, ""},
+			{"123.0", pql.Decimal{123, 0}, ""},
+
+			{".123", pql.Decimal{123, 3}, ""},
+			{"0.123", pql.Decimal{123, 3}, ""},
+			{"0.001230", pql.Decimal{123, 5}, ""},
+			{" 0.001230 ", pql.Decimal{123, 5}, ""},
+			{"-0.001230 ", pql.Decimal{-123, 5}, ""},
+
+			// int64 edges.
+			{".000009223372036854775807", pql.Decimal{9223372036854775807, 24}, ""},
+			{"-.000009223372036854775808", pql.Decimal{-9223372036854775808, 24}, ""},
+			{"92233720368547.75807", pql.Decimal{9223372036854775807, 5}, ""},
+			{"-92233720368547.75807", pql.Decimal{-9223372036854775807, 5}, ""},
+			{"9223372036854775807000", pql.Decimal{9223372036854775807, -3}, ""},
+			{"-9223372036854775807000", pql.Decimal{-9223372036854775807, -3}, ""},
+
+			// precision adjustment
+			{"2.666666666666666667", pql.Decimal{2666666666666666667, 18}, ""},
+			{"2.6666666666666666667", pql.Decimal{2666666666666666666, 18}, ""},
+			{"2.6666666666666666666667", pql.Decimal{2666666666666666666, 18}, ""},
+			{"-9.223372036854775808", pql.Decimal{-9223372036854775808, 18}, ""},
+			{"-9.223372036854775809", pql.Decimal{-922337203685477580, 17}, ""},
+			{"9.223372036854775807", pql.Decimal{9223372036854775807, 18}, ""},
+			{"9.223372036854775808", pql.Decimal{922337203685477580, 17}, ""},
+
+			// Error cases.
+			{"", pql.Decimal{}, "decimal string is empty"},
+			{"-", pql.Decimal{}, "decimal string is empty"},
+			{"*0.123", pql.Decimal{}, "invalid syntax"},
+			{"abc", pql.Decimal{}, "invalid syntax"},
+			{"0.12.3", pql.Decimal{}, "invalid decimal string"},
+			{"--12300", pql.Decimal{}, "invalid syntax"},
+
+			// These are no longer error cases since we introduced precision adjustment.
+			//{"922337203685477580.9", pql.Decimal{}, "value out of range"},
+			//{"-922337203685477580.9", pql.Decimal{}, "value out of range"},
+			{"9223372036854775808000", pql.Decimal{}, "value out of range"},
+			{"-9223372036854775809000", pql.Decimal{}, "value out of range"},
+		}
+		for i, test := range tests {
+			dec, err := pql.ParseDecimal(test.s)
+			if test.expErr != "" {
+				if err == nil || !strings.Contains(err.Error(), test.expErr) {
+					t.Fatalf("test %d expected error to contain: %s, but got: %v", i, test.expErr, err)
+				}
+			} else if err != nil {
+				t.Fatalf("test %d parsing string `%s`: %s", i, test.s, err)
+			} else if dec != test.exp {
+				t.Fatalf("test %d expected: %v, but got: %v", i, test.exp, dec)
+			}
+		}
+	})
+
+	t.Run("ToInt64", func(t *testing.T) {
+		tests := []struct {
+			dec   pql.Decimal
+			scale int64
+			exp   int64
+		}{
+			{pql.Decimal{0, 0}, 0, 0},  // 0 : 0
+			{pql.Decimal{0, 0}, 1, 0},  // 0 : 0.0
+			{pql.Decimal{0, 0}, -1, 0}, // 0 : 0
+
+			{pql.Decimal{1234567, 4}, 5, 12345670}, // 123.4567 : 123.45670
+			{pql.Decimal{1234567, 4}, 4, 1234567},  // 123.4567 : 123.4567
+			{pql.Decimal{1234567, 4}, 3, 123456},   // 123.4567 : 123.456
+
+			{pql.Decimal{-1234567, 4}, 5, -12345670}, // -123.4567 : -123.45670
+			{pql.Decimal{-1234567, 4}, 4, -1234567},  // -123.4567 : -123.4567
+			{pql.Decimal{-1234567, 4}, 3, -123456},   // -123.4567 : -123.456
+
+			{pql.Decimal{123, -2}, 5, 1230000000}, // 12300 : 12300.00000
+			{pql.Decimal{123, -2}, -1, 1230},      // 12300 : 1230
+			{pql.Decimal{123, 1}, -1, 1},          // 12.3 : 1
+			{pql.Decimal{123, 1}, -2, 0},          // 12.3 : 0
+		}
+		for i, test := range tests {
+			v := test.dec.ToInt64(test.scale)
+			if v != test.exp {
+				t.Fatalf("test %d expected: %d, but got: %d", i, test.exp, v)
+			}
+		}
+	})
+
+	t.Run("String", func(t *testing.T) {
+		tests := []struct {
+			s   string
+			exp string
+		}{
+			{"123.4567", "123.4567"},
+			{"  123.4567", "123.4567"},
+			{"  123.4567  ", "123.4567"},
+			{"123.456700", "123.4567"},
+			{"00123.4567", "123.4567"},
+			{"+123.4567", "123.4567"},
+			{"-123.4567", "-123.4567"},
+			{"-00123.4567", "-123.4567"},
+			{"-12.25", "-12.25"},
+
+			{"123", "123"},
+			{"-12300", "-12300"},
+			{"+012300", "12300"},
+			{"12300", "12300"},
+
+			{"12300.", "12300"},
+			{"12300.0", "12300"},
+			{"123.0", "123"},
+
+			{"0.123", "0.123"},
+			{"0.001230", "0.00123"},
+			{" 0.001230 ", "0.00123"},
+			{"-0.001230 ", "-0.00123"},
+		}
+		for i, test := range tests {
+			dec, err := pql.ParseDecimal(test.s)
+			if err != nil {
+				t.Fatalf("parsing string `%s`: %s", test.s, err)
+			}
+			if str := dec.String(); str != test.exp {
+				t.Fatalf("test %d expected: %s, but got: %s", i, test.exp, str)
+			}
+		}
+	})
+
+	t.Run("Comparisons", func(t *testing.T) {
+		tests := []struct {
+			d1     pql.Decimal
+			d2     pql.Decimal
+			expLT  bool
+			expLTE bool
+			expGT  bool
+			expGTE bool
+			expEQ  bool
+		}{
+			{pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), false, true, false, true, true},
+			{pql.NewDecimal(0, 0), pql.NewDecimal(10, 0), true, true, false, false, false},
+			{pql.NewDecimal(10, 0), pql.NewDecimal(0, 0), false, false, true, true, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(123456, 3), false, true, false, true, true},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(123456, 4), false, false, true, true, false},
+			{pql.NewDecimal(123456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(1233456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+
+			{pql.NewDecimal(0, 0), pql.NewDecimal(-10, 0), false, false, true, true, false},
+			{pql.NewDecimal(-10, 0), pql.NewDecimal(0, 0), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(-123456, 3), false, true, false, true, true},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(-123456, 4), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-1233456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+
+			{pql.NewDecimal(10, 0), pql.NewDecimal(-10, 0), false, false, true, true, false},
+			{pql.NewDecimal(-10, 0), pql.NewDecimal(10, 0), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(123456, 4), true, true, false, false, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(-123456, 4), false, false, true, true, false},
+			{pql.NewDecimal(-123456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(123456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-1233456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(1233456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+
+			{pql.NewDecimal(9223372036854775807, 0), pql.NewDecimal(9223372036854775807, 0), false, true, false, true, true},
+			{pql.NewDecimal(9223372036854775807, 2), pql.NewDecimal(9223372036854775807, 0), true, true, false, false, false},
+			{pql.NewDecimal(9223372036854775807, 19), pql.NewDecimal(9223372036854775807, 0), true, true, false, false, false},
+
+			{pql.NewDecimal(-9223372036854775808, 0), pql.NewDecimal(-9223372036854775808, 0), false, true, false, true, true},
+			{pql.NewDecimal(-9223372036854775808, 0), pql.NewDecimal(-9223372036854775807, 0), true, true, false, false, false},
+			{pql.NewDecimal(-9223372036854775808, 2), pql.NewDecimal(-9223372036854775808, 0), false, false, true, true, false},
+			{pql.NewDecimal(-9223372036854775808, 19), pql.NewDecimal(-9223372036854775807, 0), false, false, true, true, false},
+		}
+		for i, test := range tests {
+			if got := test.d1.LessThan(test.d2); got != test.expLT {
+				t.Fatalf("test LT %d expected %s < %s to be %v, but got: %v", i, test.d1, test.d2, test.expLT, got)
+			}
+			if got := test.d1.LessThanOrEqualTo(test.d2); got != test.expLTE {
+				t.Fatalf("test LTE %d expected %s <= %s to be %v, but got: %v", i, test.d1, test.d2, test.expLTE, got)
+			}
+			if got := test.d1.GreaterThan(test.d2); got != test.expGT {
+				t.Fatalf("test GT %d expected %s > %s to be %v, but got: %v", i, test.d1, test.d2, test.expGT, got)
+			}
+			if got := test.d1.GreaterThanOrEqualTo(test.d2); got != test.expGTE {
+				t.Fatalf("test GTE %d expected %s >= %s to be %v, but got: %v", i, test.d1, test.d2, test.expGTE, got)
+			}
+			if got := test.d1.EqualTo(test.d2); got != test.expEQ {
+				t.Fatalf("test EQ %d expected %s == %s to be %v, but got: %v", i, test.d1, test.d2, test.expEQ, got)
+			}
+		}
+	})
+
+	t.Run("JSON", func(t *testing.T) {
+		t.Run("Unmarshal", func(t *testing.T) {
+			tests := []struct {
+				json string
+				exp  pql.Decimal
+			}{
+				{"1234.56", pql.NewDecimal(123456, 2)},
+			}
+			for i, test := range tests {
+				b := []byte(test.json)
+				dec := &pql.Decimal{}
+				if err := json.Unmarshal(b, &dec); err != nil {
+					panic(err)
+				}
+				if !reflect.DeepEqual(*dec, test.exp) {
+					t.Fatalf("test %d expected: %T, but got: %T", i, test.exp, dec)
+				}
+			}
+		})
+	})
+}

pql/parser.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/pkg/errors"
 )
@@ -59,7 +61,9 @@ func (p *parser) Parse() (*Query, error) {
 	p.PQL = PQL{
 		Buffer: string(buf),
 	}
-	p.Init()
+	if err := p.Init(); err != nil {
+		return nil, errors.Wrap(err, "initializing")
+	}
 	err = p.PQL.Parse()
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing")
@@ -82,6 +86,90 @@ func (p *parser) Parse() (*Query, error) {
 			panic(v)
 		}
 	}
+	for _, call := range p.Query.Calls {
+		if call == nil {
+			return nil, fmt.Errorf("unexpected nil Call in query's call list")
+		}
+		if err := call.CheckCallInfo(); err != nil {
+			return nil, err
+		}
+	}
 
 	return &p.Query, nil
 }
+
+// Unquote interprets s as a single-quoted, double-quoted, or
+// backquoted Go string literal, returning the string value that s
+// quotes. It is a copy of stdlib's strconv.Unquote, but modified so
+// that if s is single-quoted, it can still be a string rather than
+// only character literal. This version of Unquote also accepts
+// unquoted strings and passes them back unchanged.
+func Unquote(s string) (string, error) {
+	n := len(s)
+	if n < 2 {
+		return s, nil
+	}
+	quote := s[0]
+	if quote != '"' && quote != '\'' && quote != '`' {
+		return s, nil
+	}
+	if quote != s[n-1] {
+		return "", strconv.ErrSyntax
+	}
+	s = s[1 : n-1]
+
+	if quote == '`' {
+		if contains(s, '`') {
+			return "", strconv.ErrSyntax
+		}
+		if contains(s, '\r') {
+			// -1 because we know there is at least one \r to remove.
+			buf := make([]byte, 0, len(s)-1)
+			for i := 0; i < len(s); i++ {
+				if s[i] != '\r' {
+					buf = append(buf, s[i])
+				}
+			}
+			return string(buf), nil
+		}
+		return s, nil
+	}
+	if quote != '"' && quote != '\'' {
+		return "", strconv.ErrSyntax
+	}
+	if contains(s, '\n') {
+		return "", strconv.ErrSyntax
+	}
+
+	// Is it trivial? Avoid allocation.
+	if !contains(s, '\\') && !contains(s, quote) {
+		switch quote {
+		case '"', '\'':
+			if utf8.ValidString(s) {
+				return s, nil
+			}
+		}
+	}
+
+	var runeTmp [utf8.UTFMax]byte
+	buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
+	for len(s) > 0 {
+		c, multibyte, ss, err := strconv.UnquoteChar(s, quote)
+		if err != nil {
+			return "", err
+		}
+		s = ss
+		if c < utf8.RuneSelf || !multibyte {
+			buf = append(buf, byte(c))
+		} else {
+			n := utf8.EncodeRune(runeTmp[:], c)
+			buf = append(buf, runeTmp[:n]...)
+		}
+	}
+	return string(buf), nil
+}
+
+// contains reports whether the string contains the byte c.
+func contains(s string, c byte) bool {
+	return strings.ContainsRune(s, rune(c))
+}

pql/parser_test.go

@@ -16,6 +16,7 @@ package pql_test
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
@@ -75,12 +76,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with only arguments.
 	t.Run("ArgumentsOnly", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
+		q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key":    "value",
 					"foo":    "bar",
@@ -96,19 +97,19 @@ func TestParser_Parse(t *testing.T) {
 		}
 	})
 
-	// Parse with float arguments.
+	// Parse with decimal arguments.
-	t.Run("WithFloatArgs", func(t *testing.T) {
+	t.Run("WithDecimalArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
+		q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
-					"key": 12.25,
+					"key": pql.Decimal{1225, 2},
-					"foo": 13.167,
+					"foo": pql.Decimal{13167, 3},
-					"bar": 2.,
+					"bar": pql.Decimal{2, 0},
-					"baz": 0.9,
+					"baz": pql.Decimal{9, 1},
 				},
 			},
 		) {
@@ -118,14 +119,14 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithNegativeArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
+		q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
-					"key": -12.25,
+					"key": pql.Decimal{-1225, 2},
 					"foo": int64(-13),
 				},
 			},
@@ -173,18 +174,19 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with condition arguments.
 	t.Run("WithCondition", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
+		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null, n == null)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": "foo",
-					"x":   &pql.Condition{Op: pql.EQ, Value: 12.25},
+					"x":   &pql.Condition{Op: pql.EQ, Value: pql.Decimal{1225, 2}},
 					"y":   &pql.Condition{Op: pql.GTE, Value: int64(100)},
 					"z":   &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
 					"m":   &pql.Condition{Op: pql.NEQ, Value: nil},
+					"n":   &pql.Condition{Op: pql.EQ, Value: nil},
 				},
 			},
 		) {
@@ -193,3 +195,68 @@ func TestParser_Parse(t *testing.T) {
 	})
 
 }
+
+func TestUnquote(t *testing.T) {
+	tests := []struct {
+		name   string
+		value  string
+		exp    string
+		expErr string
+	}{
+		{
+			name:  "simple double",
+			value: `"hello"`,
+			exp:   "hello",
+		},
+		{
+			name:  "simple single",
+			value: `'hello'`,
+			exp:   "hello",
+		},
+		{
+			name:  "double with esc",
+			value: `"he\"llo"`,
+			exp:   "he\"llo",
+		},
+		{
+			name:  "single with esc",
+			value: `'he\'llo'`,
+			exp:   "he'llo",
+		},
+		{
+			name:  "single with backslash and esc",
+			value: `'he\\\'llo'`,
+			exp:   `he\'llo`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := pql.Unquote(test.value)
+			if testErr(t, test.expErr, err) {
+				return
+			}
+			if got != test.exp {
+				t.Errorf("exp: '%s'\ngot: '%s'", test.exp, got)
+			}
+		})
+	}
+
+}
+
+func testErr(t *testing.T, exp string, actual error) (done bool) {
+	t.Helper()
+	if exp == "" && actual == nil {
+		return false
+	}
+	if exp == "" && actual != nil {
+		t.Fatalf("unexpected error: %v", actual)
+	}
+	if exp != "" && actual == nil {
+		t.Fatalf("expected error like '%s'", exp)
+	}
+	if !strings.Contains(actual.Error(), exp) {
+		t.Fatalf("unmatched errs exp/got\n%s\n%v", exp, actual)
+	}
+	return true
+}

pql/pql.peg

@@ -6,20 +6,25 @@ type PQL Peg {
 
 
 Calls <- sp (Call sp)* !.
-Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
+Call <-  'Set' {p.startCall("Set")} open col comma dargs (comma timestamp)? close {p.endCall()}
-       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
+       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma fargs close {p.endCall()}
-       / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
+       / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma fargs close {p.endCall()}
-       / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
+       / 'Clear' {p.startCall("Clear")} open col comma dargs close {p.endCall()}
-       / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
+       / 'ClearRow' {p.startCall("ClearRow")} open darg close {p.endCall()}
-       / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
+       / 'Store' {p.startCall("Store")} open Call comma darg close {p.endCall()}
        / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
        / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
-       / 'Range' {p.startCall("Range")} open field sp '=' sp value comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
+       / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
        / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
-allargs <- Call (comma Call)* (comma args)? / args / sp
+allargs <- Call (comma Call)* (comma dargs)? / dargs / sp
-args <- arg (comma args)? sp
+fargs <- farg (comma fargs)? sp
-arg <- (   field sp '=' sp value
+farg <- ( field sp '=' sp fvalue
-         / field sp COND sp value
+        / field sp COND sp fvalue
+        / conditional
+        )
+dargs <- darg (comma dargs)? sp
+darg <- ( field sp '=' sp dvalue
+        / field sp COND sp dvalue
         / conditional
         )
 COND <- ( '><' { p.addBTWN() }
@@ -32,41 +37,58 @@ COND <- ( '><' { p.addBTWN() }
         )
 
 conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
-condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
+condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])}
 condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
 condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
 
-value <- ( item
+dvalue <- ( ditem
-         / lbrack { p.startList() } list rbrack { p.endList() }
+         / lbrack { p.startList() } dlist rbrack { p.endList() }
          )
-list <- item (comma list)?
+fvalue <- ( fitem
-item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
+          / lbrack { p.startList() } flist rbrack { p.endList() }
+          )
+dlist <- ditem (comma dlist)?
+flist <- fitem (comma flist)?
+ditem <- ( itema
+         / decimal
+         / itemb
+         )
+fitem <- ( itema
+         / float
+         / itemb
+         )
+itema <- ( 'null' &(comma / sp close) { p.addVal(nil) }
          / 'true' &(comma / sp close) { p.addVal(true) }
          / 'false' &(comma / sp close) { p.addVal(false) }
          / timestampfmt { p.addVal(buffer[begin:end]) }
-         / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
+         )
-         / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
+itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
-         / < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
          / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
-         / < '"'  doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }
+         / < '"'  doublequotedstring '"' > { p.addVal(buffer[begin:end]) }
-         / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
+         / < '\''  singlequotedstring  '\'' > { p.addVal(buffer[begin:end]) }
+         )
+float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) }
+         / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) }
+         )
+decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false) }
+           / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) }
            )
 
-doublequotedstring <- ( '\\"' / '\\\\' / [^"] )*
+doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )*
-singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )*
+singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' /  [^'\\] )*
 
-fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' / '-' )*
+fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
 field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
 reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
 posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }
 uint <- [1-9] [0-9]* / '0'
 col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
-        / '\'' <singlequotedstring> '\'' {p.addPosStr("_col", buffer[begin:end])}
+        / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])}
-        / '"' <doublequotedstring> '"' {p.addPosStr("_col", buffer[begin:end])}
+        / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])}
         )
 row <- ( <uint> {p.addPosNum("_row", buffer[begin:end])}
-        / '\'' <singlequotedstring> '\'' {p.addPosStr("_row", buffer[begin:end])}
+        / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])}
-        / '"' <doublequotedstring> '"' {p.addPosStr("_row", buffer[begin:end])}
+        / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])}
         )
 
 open <- '(' sp

pql/pqlpeg_test.go

@@ -24,7 +24,9 @@ import (
 func TestPEG(t *testing.T) {
 	p := PQL{Buffer: `
 SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
-	p.Init()
+	if err := p.Init(); err != nil {
+		t.Fatalf("initialization error: %v", err)
+	}
 	err := p.Parse()
 	if err != nil {
 		t.Fatalf("parse error: %v", err)
@@ -32,7 +34,9 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 	p.Execute()
 
 	p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
-	p.Init()
+	if err := p.Init(); err != nil {
+		t.Fatalf("initialization error: %v", err)
+	}
 	err = p.Parse()
 	if err == nil {
 		t.Fatalf("should have been an error because of the interior unescaped double quote")
@@ -46,7 +50,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 		t.Fatalf("Failed, got: %s", q)
 	}
 
-	_, err = ParseString("C(a=falsen0)")
+	_, err = ParseString("Row(a=falsen0)")
 	if err != nil {
 		t.Fatalf("falsen0 should have been parsed as a string")
 	}
@@ -55,7 +59,6 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 	if err != nil {
 		t.Fatalf("should have parsed: %v", err)
 	}
-
 	if q.String() != `Bitmap(did=="other", row=4)` {
 		t.Fatalf("got %s", q)
 	}
@@ -109,15 +112,15 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 2},
 		{
 			name:   "SetWithArbCall",
-			input:  "Set(1, a=4)Blerg(z=ha)",
+			input:  "Set(1, a=4)Row(z=ha)",
 			ncalls: 2},
 		{
 			name:   "SetArbSet",
-			input:  "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
+			input:  "Set(1, a=4)Row(z=ha)Set(2, z=99)",
 			ncalls: 3},
 		{
 			name:   "ArbSetArb",
-			input:  "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
+			input:  "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
 			ncalls: 3},
 		{
 			name:   "SetStringArg",
@@ -161,11 +164,11 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 1},
 		{
 			name:   "double quoted args",
-			input:  `B(a="zm''e")`,
+			input:  `Row(a="zm''e")`,
 			ncalls: 1},
 		{
 			name:   "single quoted args",
-			input:  `B(a='zm""e')`,
+			input:  `Row(a='zm""e')`,
 			ncalls: 1},
 		{
 			name:   "SetRowAttrs",
@@ -235,8 +238,16 @@ func TestPEGWorking(t *testing.T) {
 			name:   "RangeEQ",
 			input:  "Row(a == 4)",
 			ncalls: 1},
+		{
+			name:   "RangeEQNULL",
+			input:  "Row(a == null)",
+			ncalls: 1},
 		{
 			name:   "RangeNEQ",
+			input:  "Row(a != 4)",
+			ncalls: 1},
+		{
+			name:   "RangeNEQNull",
 			input:  "Row(a != null)",
 			ncalls: 1},
 		{
@@ -320,7 +331,7 @@ func TestPEGErrors(t *testing.T) {
 			input: "Set(, 1, a=4)"},
 		{
 			name:  "StartinCommaArb",
-			input: "Zeeb(, a=4)"},
+			input: "Row(, a=4)"},
 		{
 			name:  "SetRowAttrs0args",
 			input: "SetRowAttrs(blah, 9)"},
@@ -533,8 +544,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
+						Op:    BTWN_LTE_LT,
-						Value: []interface{}{int64(4), int64(8)},
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -545,8 +556,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
+						Op:    BTWN_LT_LT,
-						Value: []interface{}{int64(5), int64(8)},
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -569,8 +580,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
+						Op:    BTWN_LT_LTE,
-						Value: []interface{}{int64(5), int64(9)},
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -585,11 +596,11 @@ func TestPQLDeepEquality(t *testing.T) {
 			}},
 		{
 			name: "Weird dash",
-			call: "Sum(field-=f)",
+			call: "Count(dashy-=f)",
 			exp: &Call{
-				Name: "Sum",
+				Name: "Count",
 				Args: map[string]interface{}{
-					"field-": "f",
+					"dashy-": "f",
 				},
 			}},
 		{
@@ -672,8 +683,8 @@ func TestPQLDeepEquality(t *testing.T) {
 						Name: "Row",
 						Args: map[string]interface{}{
 							"a": &Condition{
-								Op:    BETWEEN,
+								Op:    BTWN_LT_LT,
-								Value: []interface{}{int64(5), int64(8)},
+								Value: []interface{}{int64(4), int64(9)},
 							},
 						},
 					},

pql/token.go

@@ -28,7 +28,17 @@ const (
 	LTE    // <=
 	GT     // >
 	GTE    // >=
-	BETWEEN // ><
+
+	BETWEEN // ><  (this is like a <= x <= b)
+
+	// not used in lexing/parsing, but so that the parser can signal
+	// to the executor how to treat the arguments. We used to just add
+	// 1 to the arguments if they were LT so the executor could assume
+	// it was always <=, <=, but then we needed to support
+	// floats/decimals and couldn't do that any more.
+	BTWN_LT_LTE // a < x <= b
+	BTWN_LTE_LT // a <= x < b
+	BTWN_LT_LT  // a < x < b
 )
 
 var tokens = [...]string{

prometheus/prometheus.go

@@ -27,7 +27,7 @@ import (
 
 const (
 	// namespace is prepended to each metric event name with "_"
-	namespace = "pilosa"
+	defaultNamespace = "pilosa"
 )
 
 // Ensure client implements interface.
@@ -46,11 +46,22 @@ type prometheusClient struct {
 	gaugeVecs   map[string]*prometheus.GaugeVec
 	observers   map[string]prometheus.Observer
 	summaryVecs map[string]*prometheus.SummaryVec
+	namespace   string
+}
+
+// ClientOption is a functional option type for prometheusClient
+type ClientOption func(c *prometheusClient)
+
+// OptClientPrefix is a functional option on prometheusClient used to set the namespace
+func OptClientNamespace(namespace string) ClientOption {
+	return func(c *prometheusClient) {
+		c.namespace = namespace
+	}
 }
 
 // NewPrometheusClient returns a new instance of StatsClient.
-func NewPrometheusClient() (*prometheusClient, error) {
+func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) {
-	return &prometheusClient{
+	client := &prometheusClient{
 		logger:      logger.NopLogger,
 		counters:    make(map[string]prometheus.Counter),
 		counterVecs: make(map[string]*prometheus.CounterVec),
@@ -58,7 +69,14 @@ func NewPrometheusClient() (*prometheusClient, error) {
 		gaugeVecs:   make(map[string]*prometheus.GaugeVec),
 		observers:   make(map[string]prometheus.Observer),
 		summaryVecs: make(map[string]*prometheus.SummaryVec),
-	}, nil
+		namespace:   defaultNamespace,
+	}
+
+	for _, opt := range opts {
+		opt(client)
+	}
+
+	return client, nil
 }
 
 // Open no-op to satisfy interface
@@ -76,7 +94,7 @@ func (c *prometheusClient) Tags() []string {
 
 // labels returns an instance of prometheus.Labels with the value of the set tags.
 func (c *prometheusClient) labels() prometheus.Labels {
-	return tagsToLabels(c.tags)
+	return tagsToLabels(c.tags, c.logger)
 }
 
 // WithTags returns a new client with additional tags appended.
@@ -90,6 +108,7 @@ func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient {
 		gaugeVecs:   c.gaugeVecs,
 		observers:   c.observers,
 		summaryVecs: c.summaryVecs,
+		namespace:   c.namespace,
 	}
 }
 
@@ -103,7 +122,7 @@ func (c *prometheusClient) Count(name string, value int64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.CounterOpts{
-		Namespace: namespace,
+		Namespace: c.namespace,
 		Name:      name,
 	}
 	if len(labels) == 0 {
@@ -152,7 +171,7 @@ func (c *prometheusClient) Gauge(name string, value float64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.GaugeOpts{
-		Namespace: namespace,
+		Namespace: c.namespace,
 		Name:      name,
 	}
 	if len(labels) == 0 {
@@ -193,7 +212,7 @@ func (c *prometheusClient) Histogram(name string, value float64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.SummaryOpts{
-		Namespace:  namespace,
+		Namespace:  c.namespace,
 		Name:       name,
 		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 	}
@@ -233,8 +252,7 @@ func (c *prometheusClient) Set(name string, value string, rate float64) {
 
 // Timing tracks timing information for a metric.
 func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) {
-	durationMs := value / time.Second
+	c.Histogram(name, value.Seconds(), rate)
-	c.Histogram(name, float64(durationMs), rate)
 }
 
 // SetLogger sets the logger for client.
@@ -277,12 +295,13 @@ func unionStringSlice(a, b []string) []string {
 	return other
 }
 
-func tagsToLabels(tags []string) (labels prometheus.Labels) {
+func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) {
 	labels = make(prometheus.Labels)
 	for _, tag := range tags {
 		tagParts := strings.SplitAfterN(tag, ":", 2)
 		if len(tagParts) != 2 {
 			// only process tags in "key:value" form
+			logger.Printf("Error: invalid Prometheus label: %v\n", tag)
 			continue
 		}
 		labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1]

proto/interface.go

@@ -0,0 +1,314 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// StreamClient is an interface for a stream
+// which can return a RowResponse sent to a
+// stream via Send().
+type StreamClient interface {
+	Recv() (*RowResponse, error)
+}
+
+// StreamServer is an interface for a stream
+// which can accept a RowResponse to be later
+// returned by the stream via Recv().
+type StreamServer interface {
+	Send(*RowResponse) error
+}
+
+// ToTabler is an interface for any type that can
+// represent itself as a TableResponse.
+type ToTabler interface {
+	ToTable() (*TableResponse, error)
+}
+
+// ToRowser is an interface for any type that can
+// represent itself as one or more RowResponses.
+// ToRows takes a callback function which should be
+// called for each row in the response.
+type ToRowser interface {
+	ToRows(func(*RowResponse) error) error
+}
+
+// RowsToTable is a helper function which takes a ToRowser,
+// along with the number of rows, and returns a TableResponse.
+// Obviously passing the number of rows seems unnecessary,
+// and we could remove that requirement, but for now we
+// do it to allow for pre-allocation of the rows slice.
+func RowsToTable(tr ToRowser, n int) (*TableResponse, error) {
+	var headers []*ColumnInfo
+	rows := make([]*Row, n)
+
+	// This callback gets called for every "row" in r.
+	// Each row populates its position in the pre-allocated
+	// `rows`. The headers get set based on those received
+	// in the first row.
+	var idx int
+	cb := func(rr *RowResponse) error {
+		if idx == 0 {
+			headers = rr.GetHeaders()
+		}
+		rows[idx] = &Row{Columns: rr.GetColumns()}
+		idx++
+		return nil
+	}
+
+	if err := tr.ToRows(cb); err != nil {
+		return nil, errors.Wrap(err, "calling callback")
+	}
+
+	return &TableResponse{
+		Headers: headers,
+		Rows:    rows,
+	}, nil
+}
+
+// EOF acts as an io.EOF encoded into a RowResponse.
+var EOF *RowResponse = &RowResponse{
+	StatusError: &StatusError{
+		Code:    0,
+		Message: "EOF",
+	},
+}
+
+// Error is a helper function to create a RowResponse
+// based on an error message. If the error is a grpc
+// Status, then the status code is passed through.
+func Error(err error) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrap prepends a message to the existing status
+// error message.
+func ErrorWrap(err error, message string) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: message + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrapf prepends a message to the existing status
+// error message with the format specifier.
+func ErrorWrapf(err error, format string, args ...interface{}) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: fmt.Sprintf(format, args...) + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorCode is a helper function to create a RowResponse
+// based on a grpc status code and an error message.
+func ErrorCode(err error, c codes.Code) *RowResponse {
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(c),
+			Message: err.Error(),
+		},
+	}
+}
+
+// RowResponseSorter implements the sort interface for a
+// provided []RowResponse based on the column index, type,
+// and sort direction.
+type RowResponseSorter struct {
+	colIdx        []int
+	colDescending []bool
+	colType       []string
+
+	rrs []*RowResponse
+}
+
+// NewRowResponseSorter return a new RowResponseSorter. It
+// does input validation and returns an error if the inputs
+// aren't compatible.
+func NewRowResponseSorter(idxs []int, dirs []bool, typs []string, rrs []*RowResponse) (*RowResponseSorter, error) {
+	// Ensure the input slices are non-empty and equal size.
+	if len(idxs) == 0 {
+		return nil, errors.New("index list cannot be empty")
+	}
+	if len(dirs) != len(idxs) || len(typs) != len(idxs) {
+		return nil, errors.New("index, direction, and type lists must be the same size")
+	}
+
+	// Ensure the provided data types are supported by the sorter.
+	for i := range typs {
+		switch typs[i] {
+		case "[]uint64", "[]string", "bool", "float64", "int64", "string", "uint64":
+			// pass
+		default:
+			return nil, fmt.Errorf("unsupported data type: %s", typs[i])
+		}
+	}
+
+	// Ensure max(colIdx) is within size of rr.Columns.
+	if len(rrs) > 0 {
+		var maxColIdx int
+		for i := range idxs {
+			if idxs[i] > maxColIdx {
+				maxColIdx = idxs[i]
+			}
+		}
+		if maxColIdx >= len(rrs[0].Columns) {
+			return nil, fmt.Errorf("column index is out of range: %d", maxColIdx)
+		}
+	}
+
+	return &RowResponseSorter{
+		colIdx:        idxs,
+		colDescending: dirs,
+		colType:       typs,
+		rrs:           rrs,
+	}, nil
+
+}
+
+func (r RowResponseSorter) Len() int      { return len(r.rrs) }
+func (r RowResponseSorter) Swap(i, j int) { r.rrs[i], r.rrs[j] = r.rrs[j], r.rrs[i] }
+func (r RowResponseSorter) Less(i, j int) bool {
+	ri := r.rrs[i]
+	rj := r.rrs[j]
+
+	for i, idx := range r.colIdx {
+		coli := ri.Columns[idx]
+		colj := rj.Columns[idx]
+		var comp int
+		switch r.colType[i] {
+		case "[]uint64":
+			ai := coli.GetUint64ArrayVal().Vals
+			aj := colj.GetUint64ArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					piv := ai[ii]
+					pjv := aj[ii]
+					if piv == pjv {
+						continue
+					} else if piv < pjv {
+						return -1
+					} else {
+						return 1
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "[]string":
+			ai := coli.GetStringArrayVal().Vals
+			aj := colj.GetStringArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					sComp := strings.Compare(ai[ii], aj[ii])
+					if sComp == 0 {
+						continue
+					} else {
+						return sComp
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "bool":
+			bi := coli.GetBoolVal()
+			bj := colj.GetBoolVal()
+			if bi == bj {
+				comp = 0
+			} else if !bi && bj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "float64":
+			fi := coli.GetFloat64Val()
+			fj := colj.GetFloat64Val()
+			if fi == fj {
+				comp = 0
+			} else if fi < fj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "int64":
+			ni := coli.GetInt64Val()
+			nj := colj.GetInt64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "string":
+			comp = strings.Compare(coli.GetStringVal(), colj.GetStringVal())
+		case "uint64":
+			ni := coli.GetUint64Val()
+			nj := colj.GetUint64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		}
+
+		isDescending := r.colDescending[i]
+
+		switch comp {
+		case 0:
+			continue
+		case -1:
+			if isDescending {
+				return false
+			}
+			return true
+		case 1:
+			if isDescending {
+				return true
+			}
+			return false
+		}
+	}
+	return false
+}

proto/pilosa.proto

@@ -0,0 +1,81 @@
+syntax = "proto3";
+package pilosa;
+
+message QueryPQLRequest {
+  string index = 1;
+  string pql = 2;
+}
+
+message StatusError{
+  uint32 Code = 1;
+  string Message = 2;
+}
+
+message RowResponse{
+  repeated ColumnInfo headers = 1;
+  repeated ColumnResponse columns = 2;
+  StatusError StatusError = 3;
+}
+
+message Row {
+  repeated ColumnResponse columns = 1;
+}
+
+message TableResponse{
+  repeated ColumnInfo headers = 1;
+  repeated Row rows = 2;
+  StatusError StatusError = 3;
+}
+
+message ColumnInfo {
+  string name = 1;
+  string datatype = 2;
+}
+
+message ColumnResponse{
+  oneof columnVal {
+    string stringVal = 1;
+    uint64 uint64Val = 2;
+    int64 int64Val = 3;
+    bool boolVal = 4;
+    bytes blobVal = 5;
+    Uint64Array uint64ArrayVal = 6;
+    StringArray stringArrayVal = 7;
+    double float64Val = 8;
+    Decimal decimalVal = 9;
+  }
+}
+
+message Decimal {
+	int64 value = 1;
+	int64 scale = 2;
+}
+
+message InspectRequest {
+  string index = 1;
+  IdsOrKeys columns = 2;
+  repeated string filterFields = 3;
+  uint64 limit = 4;
+  uint64 offset = 5;
+}
+
+message Uint64Array {
+  repeated uint64 vals = 1;
+}
+
+message StringArray {
+  repeated string vals = 1;
+}
+
+message IdsOrKeys {
+  oneof type {
+    Uint64Array ids = 1;
+    StringArray keys = 2;
+  }
+}
+
+service Pilosa {
+  rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};
+  rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {};
+  rpc Inspect(InspectRequest) returns (stream RowResponse) {};
+}