Merge pull request #329 from travisturner/cluster-startup

avoid deadlock on translationSync.Reset during startup

.circleci/config.yml

@@ -1,17 +1,28 @@
-version: 2
-defaults: &defaults
-  working_directory: /go/src/github.com/pilosa/pilosa
-  docker:
-    - image: circleci/golang:1.13
-  environment:
-    GO111MODULE: "on"
-fast-checkout: &fast-checkout
-  attach_workspace:
-    at: .
+version: 2.1
+
+executors:
+  golang:
+    parameters:
+      version:
+        type: string
+        default: "1.14"
+    docker:
+      - image: circleci/golang:<< parameters.version >>
+    working_directory: /go/src/github.com/pilosa/pilosa
+    environment:
+      GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12.
+
+commands:
+  add-github-auth:
+    steps:
+      - run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com"
+
 jobs:
   setup:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
+      - add-github-auth
       - checkout
       - restore_cache:
           keys:
@@ -25,82 +36,76 @@ jobs:
           root: .
           paths: "*"
   check-license-headers:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
       - run: make check-license-headers
   linter:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
+      - attach_workspace:
+          at: .
+      - add-github-auth
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
       - run: make golangci-lint
   test-build-arm:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
-    <<: *defaults
+  test:
+    parameters:
+      golang_version:
+        type: string
+        default: "1.14"
+      shard_width:
+        type: string
+        default: "20"
+      test_make_target:
+        type: string
+        default: "test"
+      test_flags:
+        type: string
+        default: ""
+      goarch:
+        type: string
+        default: amd64
+    executor:
+      name: golang
+      version: << parameters.golang_version >>
     steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
       - run: sudo apt-get install lsof
       - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
+          command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
           no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
   cluster-tests:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
       - setup_remote_docker
       - run: make clustertests-build
   prerelease:
-    <<: *base-test
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
       - run: make prerelease
       - store_artifacts:
           path: build
@@ -108,9 +113,11 @@ jobs:
           root: .
           paths: build
   release:
-    <<: *defaults
+    executor:
+      name: golang
     steps:
-      - *fast-checkout
+      - attach_workspace:
+          at: .
       - run: make release
       - store_artifacts:
           path: build
@@ -121,23 +128,48 @@ jobs:
     docker:
       - image: circleci/python:2.7-jessie
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
+      - attach_workspace:
+          at: .
       - run: sudo pip install awscli
       - run: make prerelease-upload
   dockerhub-upload:
-    <<: *defaults
+    parameters:
+      tag_branch:
+        type: boolean
+        default: true
+      tag_tag:
+        type: boolean
+        default: false
+      tag_latest:
+        type: boolean
+        default: false
+      target_name:
+        type: string
+        default: moleculacorp/pilosa
+    executor:
+      name: golang
     steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
+      - attach_workspace:
+          at: .
+      - add-github-auth
       - setup_remote_docker
       - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
+      - when:
+          condition: << parameters.tag_branch >>
+          steps:
+            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:<< pipeline.git.branch >>
+      - when:
+          condition: << parameters.tag_tag >>
+          steps:
+            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:$(git describe --tags)
+      - when:
+          condition: << parameters.tag_latest >>
+          steps:
+            - run: make docker-tag-push DOCKER_TARGET=<< parameters.target_name >>:latest
+        
 workflows:
-  version: 2
-  test:
+  build:
     jobs:
       - setup
       - linter:
@@ -149,22 +181,26 @@ workflows:
       - test-build-arm:
           requires:
             - setup
-      - test-golang-1.13-enterprise:
+      - test:
+          name: test-golang-<< matrix.golang_version >>
+          matrix:
+            parameters:
+              golang_version: ["1.14", "1.13", "1.12", "1.11"]
           requires:
             - setup
-      - test-golang-1.13-race:
+      - test:
+          name: test-race
+          test_make_target: test-race
           requires:
             - setup
-      - test-golang-1.13-386:
+      - test:
+          name: test-386
+          goarch: "386"
           requires:
             - setup
-      - test-golang-1.13:
-          requires:
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
+      - test:
+          name: test-shardwidth-22
+          shard_width: "22"
           requires:
             - setup
       - cluster-tests:
@@ -174,22 +210,30 @@ workflows:
           requires:
             - linter
             - check-license-headers
-            - test-golang-1.13
-      - release:
+            - test-golang-1.14
+      - dockerhub-upload:
+          name: dockerhub-upload-unstable
+          tag_branch: true
+          tag_tag: false
+          tag_latest: false
           requires:
             - linter
             - check-license-headers
-            - test-golang-1.13
+            - test-golang-1.14
+          filters:
+            branches:
+              only: enterprise
+      - dockerhub-upload:
+          name: dockerhub-upload-stable
+          tag_branch: true
+          tag_tag: true
+          tag_latest: true
+          requires:
+            - linter
+            - check-license-headers
+            - test-golang-1.14
           filters:
             tags:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,8 +1,11 @@
 FROM golang:1.13.0 as builder
 
+ARG BUILD_FLAGS
+ARG MAKE_FLAGS
+
 COPY . pilosa
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
 
 FROM alpine:3.9.4
 
@@ -18,5 +21,9 @@ COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
 
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
 ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
+    && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -13,11 +13,20 @@ ENTERPRISE ?= 0
 ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
 RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
+NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
 BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+
+PLUGINS=distinct
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export PLUGINS
 
 # Run tests and compile Pilosa
 default: test build
@@ -32,7 +41,11 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)
+
+# Run test suite with race flag
+test-race:
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 30m -v
 
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@@ -81,14 +94,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
+clustertests-build: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
@@ -115,14 +128,29 @@ generate-stringer:
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+# dunno if protoc-gen-gofast is actually needed here
+generate-proto-grpc: require-protoc require-protoc-gen-gofast
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+
 # `go generate` all needed packages
 generate: generate-protoc generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
-docker:
-	docker build -t "pilosa:$(VERSION)" .
+docker: vendor
+	docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
 	@echo Created docker image: pilosa:$(VERSION)
 
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# Create Docker image from Dockerfile (enterprise)
+docker-enterprise: vendor
+	docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" .
+	@echo Created docker image: pilosa-enterprise:$(VERSION)
+
 # Compile Pilosa inside Docker container
 docker-build:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
@@ -133,7 +161,10 @@ docker-test:
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +192,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
 
 ######################
 # Build dependencies #

NOTICE

@@ -115,3 +115,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

api.go

@@ -23,7 +23,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -146,9 +148,11 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 	}
 	execOpts := &execOptions{
 		Remote:          req.Remote,
+		Profile:         req.Profile,
 		ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
 		ExcludeColumns:  req.ExcludeColumns,  // NOTE: Kept for Pilosa 1.x compat.
 		ColumnAttrs:     req.ColumnAttrs,     // NOTE: Kept for Pilosa 1.x compat.
+		EmbeddedData:    req.EmbeddedData,    // precomputed values that needed to be passed with the request
 	}
 	resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
 	if err != nil {
@@ -181,7 +185,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 	if err != nil {
 		return nil, errors.Wrap(err, "sending CreateIndex message")
 	}
-	api.holder.Stats.Count("createIndex", 1, 1.0)
+	api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
 	return index, nil
 }
 
@@ -225,7 +229,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 		api.server.logger.Printf("problem sending DeleteIndex message: %s", err)
 		return errors.Wrap(err, "sending DeleteIndex message")
 	}
-	api.holder.Stats.Count("deleteIndex", 1, 1.0)
+	api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
 	return nil
 }
 
@@ -272,7 +276,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Printf("problem sending CreateField message: %s", err)
 		return nil, errors.Wrap(err, "sending CreateField message")
 	}
-	api.holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return field, nil
 }
 
@@ -315,26 +319,56 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() error {
 			for viewName, viewData := range j.req.Views {
+				// The logic here corresponds to the logic in fragment.cleanViewName().
+				// Unfortunately, the logic in that method is not completely exclusive
+				// (i.e. an "other" view named with format YYYYMMDD would be handled
+				// incorrectly). One way to address this would be to change the logic
+				// overall so there weren't conflicts. For now, we just
+				// rely on the field type to inform the intended view name.
 				if viewName == "" {
 					viewName = viewStandard
-				} else {
+				} else if j.field.Type() == FieldTypeTime {
 					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
 				}
-				fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
-				if fileMagic == roaring.MagicNumber { // if pilosa roaring format
-					if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, j.req.Clear); err != nil {
-						return errors.Wrap(err, "importing pilosa roaring")
+
+				// TODO: deprecate ImportRoaringRequest.Clear, but
+				// until we do, we need to check its value to provide
+				// backward compatibility.
+				doAction := j.req.Action
+				if doAction == "" {
+					if j.req.Clear {
+						doAction = RequestActionClear
+					} else {
+						doAction = RequestActionSet
 					}
-				} else {
-					// must make a copy of data to operate on locally on standard roaring format.
-					// field.importRoaring changes the standard roaring run format to pilosa roaring
-					data := make([]byte, len(viewData))
-					copy(data, viewData)
-					if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, j.req.Clear); err != nil {
-						return errors.Wrap(err, "importing standard roaring")
+				}
+
+				var doClear bool
+				switch doAction {
+				case RequestActionOverwrite:
+					if err := j.field.importRoaringOverwrite(j.ctx, viewData, j.shard, viewName, j.req.Block); err != nil {
+						return errors.Wrap(err, "importing roaring as overwrite")
+					}
+				case RequestActionClear:
+					doClear = true
+					fallthrough
+				case RequestActionSet:
+					fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
+					if fileMagic == roaring.MagicNumber { // if pilosa roaring format
+						if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, doClear); err != nil {
+							return errors.Wrap(err, "importing pilosa roaring")
+						}
+					} else {
+						// must make a copy of data to operate on locally on standard roaring format.
+						// field.importRoaring changes the standard roaring run format to pilosa roaring
+						data := make([]byte, len(viewData))
+						copy(data, viewData)
+						if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, doClear); err != nil {
+							return errors.Wrap(err, "importing standard roaring")
+						}
 					}
 				}
 			}
@@ -380,12 +414,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 	if field == nil {
 		return newNotFoundError(ErrFieldNotFound)
 	}
-
-	// only set and time fields are supported
-	if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime {
-		return NewBadRequestError(errors.New("roaring import is only supported for set and time fields"))
-	}
-
 	errCh := make(chan error, len(nodes))
 
 	for _, node := range nodes {
@@ -459,7 +487,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Printf("problem sending DeleteField message: %s", err)
 		return errors.Wrap(err, "sending DeleteField message")
 	}
-	api.holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -491,7 +519,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
 		api.server.logger.Printf("problem sending DeleteAvailableShard message: %s", err)
 		return errors.Wrap(err, "sending DeleteAvailableShard message")
 	}
-	api.holder.Stats.CountWithCustomTags("deleteAvailableShard", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -540,8 +568,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		var colStr string
 		var err error
 
-		if field.keys() {
-			if rowStr, err = field.translateStore.TranslateID(rowID); err != nil {
+		if field.Keys() {
+			// TODO: handle case: field.ForeignIndex
+			if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil {
 				return errors.Wrap(err, "translating row")
 			}
 		} else {
@@ -549,7 +578,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		}
 
 		if index.Keys() {
-			if colStr, err = index.translateStore.TranslateID(columnID); err != nil {
+			if store := index.TranslateStore(api.cluster.idPartition(indexName, columnID)); store == nil {
+				return errors.Wrap(err, "partition does not exist")
+			} else if colStr, err = store.TranslateID(columnID); err != nil {
 				return errors.Wrap(err, "translating column")
 			}
 		} else {
@@ -660,6 +691,30 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName
 	return f, nil
 }
 
+// TranslateData returns all translation data in the specified partition.
+func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
+	defer span.Finish()
+
+	if err := api.validate(apiTranslateData); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+
+	// Retrieve index from holder.
+	idx := api.holder.Index(indexName)
+	if idx == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Retrieve translatestore from holder.
+	store := idx.TranslateStore(partition)
+	if store == nil {
+		return nil, ErrTranslateStoreNotFound
+	}
+
+	return store, nil
+}
+
 // Hosts returns a list of the hosts in the cluster including their ID,
 // URL, and which is the coordinator.
 func (api *API) Hosts(ctx context.Context) []*Node {
@@ -714,7 +769,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
 		return errors.Wrap(err, "deserializing cluster message")
 	}
 
-	// Forward the error message.
+	// Forward the message.
 	if err := api.server.receiveMessage(msg); err != nil {
 		return errors.Wrap(err, "receiving message")
 	}
@@ -889,10 +944,17 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
 	return attrs, nil
 }
 
-// ImportOptions holds the options for the API.Import method.
+// ImportOptions holds the options for the API.Import
+// method.
+//
+// TODO(2.0) we have entirely missed the point of functional options
+// by exporting this structure. If it needs to be exported for some
+// reason, we should consider not using functional options here which
+// just adds complexity.
 type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
+	Presorted      bool
 }
 
 // ImportOption is a functional option type for API.Import.
@@ -916,6 +978,13 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption {
 	}
 }
 
+func OptImportOptionsPresorted(b bool) ImportOption {
+	return func(o *ImportOptions) error {
+		o.Presorted = b
+		return nil
+	}
+}
+
 // Import bulk imports data into a particular index,field,shard.
 func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
@@ -941,12 +1010,12 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
 	// check to see if keys need translation.
 	if !options.IgnoreKeyCheck {
 		// Translate row keys.
-		if field.keys() {
+		if field.Keys() {
 			if len(req.RowIDs) != 0 {
 				return errors.New("row ids cannot be used because field uses string keys")
 			}
-			if req.RowIDs, err = field.translateStore.TranslateKeys(req.RowKeys); err != nil {
-				return errors.Wrap(err, "translating rows")
+			if req.RowIDs, err = api.cluster.translateFieldKeys(ctx, field, req.RowKeys...); err != nil {
+				return errors.Wrapf(err, "translating field keys")
 			}
 		}
 
@@ -955,14 +1024,14 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
 		}
 
 		// For translated data, map the columnIDs to shards. If
 		// this node does not own the shard, forward to the node that does.
-		if index.Keys() || field.keys() {
+		if index.Keys() || field.Keys() {
 			m := make(map[uint64][]Bit)
 
 			for i, colID := range req.ColumnIDs {
@@ -1036,6 +1105,10 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 		return errors.Wrap(err, "validating api method")
 	}
 
+	if err := req.Validate(); err != nil {
+		return errors.Wrap(err, "validating import value request")
+	}
+
 	// Set up import options.
 	options, err := setUpImportOptions(opts...)
 	if err != nil {
@@ -1056,60 +1129,138 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
+			req.Shard = math.MaxUint64
+		}
 
-			// For translated data, map the columnIDs to shards. If
-			// this node does not own the shard, forward to the node that does.
-			m := make(map[uint64][]FieldValue)
-
-			for i, colID := range req.ColumnIDs {
-				shard := colID / ShardWidth
-				if _, ok := m[shard]; !ok {
-					m[shard] = make([]FieldValue, 0)
-				}
-				m[shard] = append(m[shard], FieldValue{
-					Value:    req.Values[i],
-					ColumnID: colID,
-				})
+		// Translate values when the field uses keys (for example, when
+		// the field has a ForeignIndex with keys).
+		if field.Keys() {
+			// Perform translation.
+			uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues)
+			if err != nil {
+				return err
 			}
 
-			// Signal to the receiving nodes to ignore checking for key translation.
-			opts = append(opts, OptImportOptionsIgnoreKeyCheck(true))
-
-			var eg errgroup.Group
-			for shard, vals := range m {
-				// TODO: if local node owns this shard we don't need to go through the client
-				shard := shard
-				vals := vals
-				eg.Go(func() error {
-					return api.server.defaultClient.ImportValue(ctx, req.Index, req.Field, shard, vals, opts...)
-				})
+			// Because the BSI field supports negative values, we have to
+			// convert the uint64 keys to a slice of int64.
+			ints := make([]int64, len(uints))
+			for i := range uints {
+				ints[i] = int64(uints[i])
 			}
-			return eg.Wait()
+			req.Values = ints
 		}
 	}
 
-	// Validate shard ownership.
-	if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
+	if !options.Presorted {
+		sort.Sort(req)
+	}
+
+	// if we're importing into a specific shard
+	if req.Shard != math.MaxUint64 {
+		// Check that column IDs match the stated shard.
+		if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard {
+			return errors.Errorf("shard %d specified, but import spans shards %d to %d", req.Shard, s1, s2)
+		}
+		// Validate shard ownership. TODO - we should forward to the
+		// correct node rather than barfing here.
+		if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
+			return errors.Wrap(err, "validating shard ownership")
+		}
+		// Import columnIDs into existence field.
+		if !options.Clear {
+			if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
+				api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+				return errors.Wrap(err, "importing existence columns")
+			}
+		}
+
+		// Import into fragment.
+		if len(req.Values) > 0 {
+			err = field.importValue(req.ColumnIDs, req.Values, options)
+			if err != nil {
+				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+			}
+		} else if len(req.FloatValues) > 0 {
+			err = field.importFloatValue(req.ColumnIDs, req.FloatValues, options)
+			if err != nil {
+				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+			}
+		}
+
+		return errors.Wrap(err, "importing value")
+	}
+
+	options.IgnoreKeyCheck = true
+	start := 0
+	shard := req.ColumnIDs[0] / ShardWidth
+	var eg errgroup.Group // TODO make this a pooled errgroup
+	for i, colID := range req.ColumnIDs {
+		if colID/ShardWidth != shard {
+			subreq := &ImportValueRequest{
+				Index:     req.Index,
+				Field:     req.Field,
+				Shard:     shard,
+				ColumnIDs: req.ColumnIDs[start:i],
+			}
+			if req.Values != nil {
+				subreq.Values = req.Values[start:i]
+			} else if req.FloatValues != nil {
+				subreq.FloatValues = req.FloatValues[start:i]
+			}
+
+			eg.Go(func() error {
+				return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+			})
+			start = i
+			shard = colID / ShardWidth
+		}
+	}
+	subreq := &ImportValueRequest{
+		Index:     req.Index,
+		Field:     req.Field,
+		Shard:     shard,
+		ColumnIDs: req.ColumnIDs[start:],
+	}
+	if req.Values != nil {
+		subreq.Values = req.Values[start:]
+	} else if req.FloatValues != nil {
+		subreq.FloatValues = req.FloatValues[start:]
+	}
+	eg.Go(func() error {
+		// TODO we should elevate the logic for figuring out which
+		// node(s) to send to into API instead of having those details
+		// in the client implementation.
+		return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+	})
+	return eg.Wait()
+
+}
+
+func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.ImportColumnAttrs")
+	defer span.Finish()
+
+	index, err := api.Index(ctx, req.Index)
+	if err != nil {
+		return errors.Wrap(err, "getting index")
+	}
+
+	if err := api.validateShardOwnership(req.Index, uint64(req.Shard)); err != nil {
 		return errors.Wrap(err, "validating shard ownership")
 	}
 
-	// Import columnIDs into existence field.
-	if !options.Clear {
-		if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
-			api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
-			return errors.Wrap(err, "importing existence columns")
-		}
+	bulkAttrs := make(map[uint64]map[string]interface{})
+	for n := 0; n < len(req.ColumnIDs); n++ {
+		bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}
 	}
-
-	// Import into fragment.
-	err = field.importValue(req.ColumnIDs, req.Values, options)
-	if err != nil {
-		api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+	if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil {
+		api.server.logger.Printf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing column attrs")
 	}
-	return errors.Wrap(err, "importing")
+	return nil
 }
 
 func importExistenceColumns(index *Index, columnIDs []uint64) error {
@@ -1286,14 +1437,75 @@ func (api *API) Info() serverInfo {
 }
 
 // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
-func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) {
+func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")
 	defer span.Finish()
-	return api.holder.TranslateEntryReader(ctx, offsets)
+
+	// Ensure all readers are cleaned up if any error.
+	var a []TranslateEntryReader
+	defer func() {
+		if err != nil {
+			for i := range a {
+				a[i].Close()
+			}
+		}
+	}()
+
+	// Fetch all index partition readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for partitionID, offset := range indexMap.Partitions {
+			store := index.TranslateStore(partitionID)
+			if store == nil {
+				return nil, ErrTranslateStoreNotFound
+			}
+
+			r, err := store.EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "index partition translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	// Fetch all field readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for fieldName, offset := range indexMap.Fields {
+			field := index.Field(fieldName)
+			if field == nil {
+				return nil, ErrFieldNotFound
+			}
+
+			r, err := field.TranslateStore().EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "field translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	return NewMultiTranslateEntryReader(ctx, a), nil
+}
+
+func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	return api.cluster.translateIndexKey(ctx, indexName, key)
+}
+
+func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	return api.cluster.translateIndexIDs(ctx, indexName, ids)
 }
 
 // TranslateKeys handles a TranslateKeyRequest.
-func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
+func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) {
 	var req TranslateKeysRequest
 	if buf, err := ioutil.ReadAll(r); err != nil {
 		return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error"))
@@ -1302,13 +1514,22 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	}
 
 	// Lookup store for either index or field and translate keys.
-	store, err := api.holder.TranslateStore(req.Index, req.Field)
-	if err != nil {
-		return nil, err
-	}
-	ids, err := store.TranslateKeys(req.Keys)
-	if err != nil {
-		return nil, err
+	var ids []uint64
+	if req.Field == "" {
+		if ids, err = api.cluster.translateIndexKeys(ctx, req.Index, req.Keys); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			ids, err = api.cluster.translateIndexKeys(ctx, fi, req.Keys)
+			if err != nil {
+				return nil, err
+			}
+		} else if ids, err = api.cluster.translateFieldKeys(ctx, field, req.Keys...); err != nil {
+			return nil, errors.Wrapf(err, "translating field keys")
+		}
 	}
 
 	// Encode response.
@@ -1319,6 +1540,42 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	return buf, nil
 }
 
+// TranslateIDs handles a TranslateIDRequest.
+func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) {
+	var req TranslateIDsRequest
+	if buf, err := ioutil.ReadAll(r); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error"))
+	} else if err := api.Serializer.Unmarshal(buf, &req); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error"))
+	}
+
+	// Lookup store for either index or field and translate ids.
+	var keys []string
+	if req.Field == "" {
+		if keys, err = api.cluster.translateIndexIDs(ctx, req.Index, req.IDs); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			keys, err = api.cluster.translateIndexIDs(ctx, fi, req.IDs)
+			if err != nil {
+				return nil, err
+			}
+		} else if keys, err = field.TranslateStore().TranslateIDs(req.IDs); err != nil {
+			return nil, err
+		}
+	}
+
+	// Encode response.
+	buf, err := api.Serializer.Marshal(&TranslateIDsResponse{Keys: keys})
+	if err != nil {
+		return nil, errors.Wrap(err, "translate ids response encoding error")
+	}
+	return buf, nil
+}
+
 // PrimaryReplicaNodeURL returns the URL of the cluster's primary replica.
 func (api *API) PrimaryReplicaNodeURL() url.URL {
 	node := api.cluster.PrimaryReplicaNode()
@@ -1328,6 +1585,34 @@ func (api *API) PrimaryReplicaNodeURL() url.URL {
 	return node.URI.URL()
 }
 
+func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, remote bool) (*Transaction, error) {
+	if err := api.validate(apiStartTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
+}
+
+func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) {
+	if err := api.validate(apiFinishTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.server.FinishTransaction(ctx, id, remote)
+}
+
+func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	if err := api.validate(apiTransactions); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.server.Transactions(ctx)
+}
+
+func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) {
+	if err := api.validate(apiGetTransaction); err != nil {
+		return nil, errors.Wrap(err, "validating api method")
+	}
+	return api.server.GetTransaction(ctx, id, remote)
+}
+
 type serverInfo struct {
 	ShardWidth       uint64 `json:"shardWidth"`
 	Memory           uint64 `json:"memory"`
@@ -1352,6 +1637,7 @@ const (
 	apiFragmentBlockData
 	apiFragmentBlocks
 	apiFragmentData
+	apiTranslateData
 	apiField
 	apiFieldAttrDiff
 	//apiHosts // not implemented
@@ -1374,6 +1660,10 @@ const (
 	//apiVersion // not implemented
 	apiViews
 	apiApplySchema
+	apiStartTransaction
+	apiFinishTransaction
+	apiTransactions
+	apiGetTransaction
 )
 
 var methodsCommon = map[apiMethod]struct{}{
@@ -1382,8 +1672,9 @@ var methodsCommon = map[apiMethod]struct{}{
 }
 
 var methodsResizing = map[apiMethod]struct{}{
-	apiFragmentData: {},
-	apiResizeAbort:  {},
+	apiFragmentData:  {},
+	apiTranslateData: {},
+	apiResizeAbort:   {},
 }
 
 var methodsNormal = map[apiMethod]struct{}{
@@ -1408,4 +1699,8 @@ var methodsNormal = map[apiMethod]struct{}{
 	apiShardNodes:           {},
 	apiViews:                {},
 	apiApplySchema:          {},
+	apiStartTransaction:     {},
+	apiFinishTransaction:    {},
+	apiTransactions:         {},
+	apiGetTransaction:       {},
 }

api/client/grpc.go

@@ -0,0 +1,213 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +31,137 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c[0]
+	m1 := c[1]
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		index := "i"
+		field := "f"
+		attrKey := "k"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:   attrKey,
+			ColumnIDs: columnIDs0,
+			AttrVals:  attrVals0,
+			Shard:     0,
+			Index:     index,
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:   attrKey,
+			ColumnIDs: columnIDs1,
+			AttrVals:  attrVals1,
+			Shard:     1,
+			Index:     index,
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -71,14 +203,15 @@ func TestAPI_Import(t *testing.T) {
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
 		for i := 1; i <= 10; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
@@ -129,58 +262,6 @@ func TestAPI_Import(t *testing.T) {
 		}
 
 	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
@@ -219,12 +300,13 @@ func TestAPI_ImportValue(t *testing.T) {
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
@@ -258,6 +340,172 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		fld, err := m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+		if err := m1.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: %+v", ids)
+		}
+
+		sum, count, err := fld.FloatSum(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatsum: %v", err)
+		} else if sum != 0.1+1.1+2.1+3.1+4.1+5.1+6.1+7.1+8.1+9.1 {
+			t.Fatalf("unexpected sum: %f", sum)
+		} else if count != 10 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		min, count, err := fld.FloatMin(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatmin: %v", err)
+		} else if min != 0.1 {
+			t.Fatalf("unexpected min: %f", min)
+		} else if count != 1 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		max, count, err := fld.FloatMax(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatmax: %v", err)
+		} else if max != 9.1 {
+			t.Fatalf("unexpected max: %f", max)
+		} else if count != 1 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		val, exists, err := fld.FloatValue(1)
+		if err != nil {
+			t.Fatalf("unepxected err getting floatvalue")
+		} else if !exists {
+			t.Fatalf("column 1 should exist")
+		} else if val != 1.1 {
+			t.Fatalf("unexpected floatvalue %f", val)
+		}
+
+		changed, err := fld.SetFloatValue(11, 11.1)
+		if err != nil {
+			t.Fatalf("setting float value: %v", err)
+		} else if !changed {
+			t.Fatalf("expected change")
+		}
+
+		val, exists, err = fld.FloatValue(11)
+		if err != nil {
+			t.Fatalf("getting float val: %v", err)
+		} else if !exists {
+			t.Fatalf("should exist")
+		} else if val != 11.1 {
+			t.Fatalf("unexpected val: %f", 11.1)
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err == nil {
+			t.Fatal("expected error creating field")
+		}
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		if err := m0.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: %+v", ids)
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.

apimethod_string.go

@@ -19,25 +19,26 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
-	_ = x[apiFieldAttrDiff-12]
-	_ = x[apiImport-13]
-	_ = x[apiImportValue-14]
-	_ = x[apiIndex-15]
-	_ = x[apiIndexAttrDiff-16]
-	_ = x[apiQuery-17]
-	_ = x[apiRecalculateCaches-18]
-	_ = x[apiRemoveNode-19]
-	_ = x[apiResizeAbort-20]
-	_ = x[apiSetCoordinator-21]
-	_ = x[apiShardNodes-22]
-	_ = x[apiViews-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiTranslateData-11]
+	_ = x[apiField-12]
+	_ = x[apiFieldAttrDiff-13]
+	_ = x[apiImport-14]
+	_ = x[apiImportValue-15]
+	_ = x[apiIndex-16]
+	_ = x[apiIndexAttrDiff-17]
+	_ = x[apiQuery-18]
+	_ = x[apiRecalculateCaches-19]
+	_ = x[apiRemoveNode-20]
+	_ = x[apiResizeAbort-21]
+	_ = x[apiSetCoordinator-22]
+	_ = x[apiShardNodes-23]
+	_ = x[apiViews-24]
+	_ = x[apiApplySchema-25]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 332, 345, 353, 367}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

boltdb/translate.go

@@ -11,10 +11,13 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
@@ -31,9 +34,14 @@ var (
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
 )
 
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+)
+
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +53,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
-	index string
-	field string
+	index       string
+	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +81,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -108,6 +128,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,12 +160,12 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKey converts a string key to an integer ID.
 // If key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	// Find id by key under read lock.
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
+		id, _ = findIDByKey(tx.Bucket([]byte("keys")), key)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -156,13 +181,16 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
 		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
+
+		var boltKey []byte
+		if id, boltKey = findIDByKey(bkt, key); id != 0 {
 			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
+		}
+
+		id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+		if err := bkt.Put(boltKey, u64tob(id)); err != nil {
 			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil {
 			return err
 		}
 		written = true
@@ -178,8 +206,8 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	return id, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
+// If a key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if len(keys) == 0 {
 		return nil, nil
@@ -193,7 +221,7 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket([]byte("keys"))
 		for i, key := range keys {
-			if id := findIDByKey(bkt, key); id != 0 {
+			if id, _ := findIDByKey(bkt, key); id != 0 {
 				ids[i] = id
 				found++
 			}
@@ -218,13 +246,15 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 				continue
 			}
 
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
+			var boltKey []byte
+			if ids[i], boltKey = findIDByKey(bkt, key); ids[i] != 0 {
 				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
+			}
+
+			ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+			if err := bkt.Put(boltKey, u64tob(ids[i])); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
+			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), boltKey); err != nil {
 				return err
 			}
 			written = true
@@ -286,7 +316,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +341,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
-			max = btou64(key)
-		}
+		max = maxID(tx)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +349,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -387,13 +470,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
-	if value := bkt.Get([]byte(key)); value != nil {
-		return btou64(value)
+// emptyKey is a sentinel byte slice which stands for "" as a key.
+var emptyKey = []byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }

boltdb/translate_test.go

@@ -14,9 +14,12 @@
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"testing"
 	"time"
 
@@ -28,24 +31,23 @@ func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	// Ensure initial key translates to first ID for shard
+	id1, err := s.TranslateKey("foo")
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
 	if id, err := s.TranslateKey("bar"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, id1+1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
 	if id, err := s.TranslateKey("foo"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, id1; got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
@@ -55,31 +57,30 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Ensure initial keys translate to incrementing IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids1, err := s.TranslateKeys([]string{"foo", "bar"})
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids1[1], ids1[0]+1; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
 	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
 	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], ids1[0]; got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], ids1[0]+2; got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], ids1[1]; got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
@@ -89,24 +90,38 @@ func TestTranslateStore_TranslateID(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo")
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("")
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
@@ -114,12 +129,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"})
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -136,7 +152,8 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.TranslateKeys([]string{"foo", "bar"})
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +168,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1[0]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +177,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1[1]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		id2, err := s.TranslateKey("baz")
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, id2; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -198,20 +216,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			id, err := s.TranslateKey("foo")
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- id
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -302,11 +325,76 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore()
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.TranslateKeys(batch1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
+			t.Fatal(err)
+		} else if ids[0] != expIDs[0] {
+			t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
+		} else if ids[1] == expIDs[1] {
+			t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
 func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 	s := MustNewTranslateStore()

broadcast.go

@@ -69,6 +69,7 @@ const (
 	messageTypeRecalculateCaches
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -116,6 +117,8 @@ func getMessage(typ byte) Message {
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -155,6 +158,8 @@ func getMessageType(m Message) byte {
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -23,7 +24,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/lru"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -172,6 +175,7 @@ func (c *rankCache) Add(id uint64, n uint64) {
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -185,6 +189,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -228,7 +233,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -238,7 +243,7 @@ func (c *rankCache) invalidate() {
 	if time.Since(c.updateTime).Seconds() < 10 {
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -256,7 +261,7 @@ func (c *rankCache) recalculate() {
 	// Store the count of the item at the threshold index.
 	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,7 +277,7 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@@ -316,10 +321,54 @@ func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +444,73 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 

cache_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
 }

client.go

@@ -17,6 +17,7 @@ package pilosa
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // Bit represents the intersection of a row and a column. It can be specified by
@@ -44,6 +45,8 @@ type FieldValue struct {
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
@@ -51,7 +54,6 @@ type InternalClient interface {
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
 	Nodes(ctx context.Context) ([]*Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,6 +61,7 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@@ -68,7 +71,14 @@ type InternalClient interface {
 	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
 	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
+	RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error)
 	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
+
+	StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
+	FinishTransaction(ctx context.Context, id string) (*Transaction, error)
+	Transactions(ctx context.Context) (map[string]*Transaction, error)
+	GetTransaction(ctx context.Context, id string) (*Transaction, error)
 }
 
 //===============
@@ -76,6 +86,8 @@ type InternalClient interface {
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
 	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
 }
 
 type nopInternalQueryClient struct{}
@@ -84,6 +96,14 @@ func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index
 	return nil, nil
 }
 
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
 func newNopInternalQueryClient() *nopInternalQueryClient {
 	return &nopInternalQueryClient{}
 }
@@ -123,15 +143,30 @@ func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest
 func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
+func (n nopInternalClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+func (n nopInternalClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
+	return nil
+}
+
 func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }
@@ -172,3 +207,19 @@ func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte
 func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
 	return nil, nil
 }
+func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return nil, nil
+}
+func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return nil, nil
+}

cluster.go

@@ -40,8 +40,8 @@ import (
 )
 
 const (
-	// defaultPartitionN is the default number of partitions in a cluster.
-	defaultPartitionN = 256
+	// DefaultPartitionN is the default number of partitions in a cluster.
+	DefaultPartitionN = 256
 
 	// ClusterState represents the state returned in the /status endpoint.
 	ClusterStateStarting = "STARTING"
@@ -71,6 +71,7 @@ const (
 type Node struct {
 	ID            string `json:"id"`
 	URI           URI    `json:"uri"`
+	GRPCURI       URI    `json:"grpc-uri"`
 	IsCoordinator bool   `json:"isCoordinator"`
 	State         string `json:"state"`
 }
@@ -223,6 +224,9 @@ type cluster struct { // nolint: maligned
 	joined  bool
 
 	abortAntiEntropyCh chan struct{}
+	muAntiEntropy      sync.Mutex
+
+	translationSyncer translationSyncer
 
 	mu         sync.RWMutex
 	jobs       map[int64]*resizeJob
@@ -241,7 +245,7 @@ type cluster struct { // nolint: maligned
 func newCluster() *cluster {
 	return &cluster{
 		Hasher:     &jmphasher{},
-		partitionN: defaultPartitionN,
+		partitionN: DefaultPartitionN,
 		ReplicaN:   1,
 
 		joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
@@ -249,6 +253,8 @@ func newCluster() *cluster {
 		closing:             make(chan struct{}),
 		joining:             make(chan struct{}),
 
+		translationSyncer: NopTranslationSyncer,
+
 		InternalClient: newNopInternalClient(),
 
 		logger: logger.NopLogger,
@@ -390,14 +396,6 @@ func (c *cluster) addNode(node *Node) error {
 		return nil
 	}
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// add to topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -417,14 +415,6 @@ func (c *cluster) removeNode(nodeID string) error {
 	// remove from cluster
 	c.removeNodeBasicSorted(nodeID)
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// remove from topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -477,7 +467,7 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	switch state {
 	case ClusterStateNormal, ClusterStateDegraded:
-		// If state is RESIZING -> NORMAL then run cleanup.
+		// If state is RESIZING -> [NORMAL, DEGRADED] then run cleanup.
 		if c.state == ClusterStateResizing {
 			doCleanup = true
 		}
@@ -485,8 +475,28 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	c.state = state
 
-	if state == ClusterStateResizing {
-		c.abortAntiEntropy()
+	switch state {
+	case ClusterStateNormal:
+		// Because the cluster state is changing to NORMAL,
+		// we [potentially] need to reset the translation sync.
+		// If, for example, the cluster has changed size and is
+		// now settling to NORMAL, the partition ownership may
+		// have changed, and this will force that to be recalculated.
+		//
+		// We can't call Reset() if Server.Open() hasn't run yet,
+		// because that's where we start monitorResetTranslationSync()
+		// which reads the reset channel. If we get here before
+		// Server.Open(), this will deadlock on that channel read.
+		// In order to address this, we call Reset() in a goroutine
+		// so even if it blocks waiting for monitorResetTranslationSync()
+		// to start, it doesn't cause a deadlock, and once Server.Open()
+		// is called, then the sync reset (or in the STARTING case, the
+		// initial sync start) will happen.
+		go func() {
+			if err := c.translationSyncer.Reset(); err != nil {
+				c.logger.Printf("error resetting translation syncer: %s", err)
+			}
+		}()
 	}
 
 	// TODO: consider NOT running cleanup on an active node that has
@@ -575,6 +585,19 @@ func (c *cluster) determineClusterState() (clusterState string) {
 	if c.haveTopologyAgreement() && c.allNodesReady() {
 		return ClusterStateNormal
 	}
+	// TODO:
+	// If the cluster is still STARTING, there's no need to put it into
+	// state DEGRADED. It's possible to force a starting cluster to go
+	// into state DEGRADED by, for example, restarting a 2-node cluster
+	// with replica=3. In that case, the coordinator would come up and
+	// it would immediately trigger this condition. Checking for
+	// state != STARTING here would prevent that. Unfortunately, based
+	// on test TestClusteringNodesReplica2, we expect a DEGRADED cluster
+	// to go back into state STARTING if it loses more replicas than
+	// can support queries. In that case, we might actually want it to
+	// go from STARTING back to DEGRADED. Leaving it as is for now, but
+	// noting that it's a little confusing that a cluster starting up
+	// could possibly go into state DEGRADED.
 	if len(c.Topology.nodeIDs)-len(c.nodeIDs()) < c.ReplicaN && c.allNodesReady() {
 		return ClusterStateDegraded
 	}
@@ -867,8 +890,72 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour
 	return m, nil
 }
 
-// partition returns the partition that a shard belongs to.
-func (c *cluster) partition(index string, shard uint64) int {
+// translationNodes returns a list of translationResizeNodes - for each node
+// in the `to` cluster - required to move from cluster `c` to cluster `to`. unprotected.
+// Because the parition scheme for every index is the same, this is used as a template
+// to create index-specific `TranslationResizeSource`s.
+func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResizeNode, error) {
+	m := make(map[string][]*translationResizeNode)
+
+	// Determine if a node is being added or removed.
+	action, diffNodeID, err := c.diff(to)
+	if err != nil {
+		return nil, errors.Wrap(err, "diffing")
+	}
+
+	// Initialize the map with all the nodes in `to`.
+	for _, n := range to.nodes {
+		m[n.ID] = nil
+	}
+
+	for pid := 0; pid < c.partitionN; pid++ {
+		fNodes := c.partitionNodes(pid)
+		tNodes := to.partitionNodes(pid)
+
+		// For `to` cluster, we include all nodes containing a
+		// replica for the partition. The source for each replica
+		// will be the primary in the `from` cluster. For the `from`
+		// cluster, we only need the first node, unless that node is
+		// being removed, then we use the second node. If no second
+		// node exists in that case, then we have to raise an error
+		// indicating that not enough replicas exist to support
+		// the resize.
+		if len(tNodes) > 0 {
+			var foundPrimary bool
+			for i := range fNodes {
+				if action == resizeJobActionRemove && fNodes[i].ID == diffNodeID {
+					continue
+				}
+				// We only need to add the source if the nodes differ;
+				// in other words if the primary partition is on the
+				// same node, it doesn't need to retrieve it.
+				for n := range tNodes {
+					if tNodes[n].ID != fNodes[i].ID {
+						m[tNodes[n].ID] = append(m[tNodes[n].ID],
+							&translationResizeNode{
+								node:        fNodes[i],
+								partitionID: pid,
+							})
+					}
+				}
+				foundPrimary = true
+				break
+			}
+			if !foundPrimary {
+				return nil, ErrResizeNoReplicas
+			}
+		}
+	}
+
+	return m, nil
+}
+
+// shardPartition returns the partition that a shard belongs to.
+func (c *cluster) shardPartition(index string, shard uint64) int {
+	return shardPartition(index, shard, c.partitionN)
+}
+
+func shardPartition(index string, shard uint64, partitionN int) int {
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], shard)
 
@@ -876,7 +963,25 @@ func (c *cluster) partition(index string, shard uint64) int {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(index))
 	_, _ = h.Write(buf[:])
-	return int(h.Sum64() % uint64(c.partitionN))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// keyPartition returns the partition that a key belongs to.
+func (c *cluster) keyPartition(index, key string) int {
+	return keyPartition(index, key, c.partitionN)
+}
+
+func keyPartition(index, key string, partitionN int) int {
+	// Hash the bytes and mod by partition count.
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(index))
+	_, _ = h.Write([]byte(key))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// idPartition returns the partition that an id belongs to.
+func (c *cluster) idPartition(index string, id uint64) int {
+	return shardPartition(index, id/ShardWidth, c.partitionN)
 }
 
 // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use.
@@ -888,7 +993,19 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
 
 // shardNodes returns a list of nodes that own a fragment. unprotected
 func (c *cluster) shardNodes(index string, shard uint64) []*Node {
-	return c.partitionNodes(c.partition(index, shard))
+	return c.partitionNodes(c.shardPartition(index, shard))
+}
+
+// KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use.
+func (c *cluster) KeyNodes(index, key string) []*Node {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.keyNodes(index, key)
+}
+
+// keyNodes returns a list of nodes that own a key. unprotected
+func (c *cluster) keyNodes(index, key string) []*Node {
+	return c.partitionNodes(c.keyPartition(index, key))
 }
 
 // ownsShard returns true if a host owns a fragment.
@@ -900,7 +1017,6 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
 
 // partitionNodes returns a list of nodes that own a partition. unprotected.
 func (c *cluster) partitionNodes(partitionID int) []*Node {
-
 	// Default replica count to between one and the number of nodes.
 	// The replica count can be zero if there are no nodes.
 	replicaN := c.ReplicaN
@@ -922,11 +1038,18 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
 	return nodes
 }
 
+// ownsPartition returns true if a host owns a partition.
+func (c *cluster) ownsPartition(nodeID string, partition int) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return Nodes(c.partitionNodes(partition)).ContainsID(nodeID)
+}
+
 // containsShards is like OwnsShards, but it includes replicas.
 func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 {
 	var shards []uint64
 	availableShards.ForEach(func(i uint64) {
-		p := c.partition(index, i)
+		p := c.shardPartition(index, i)
 		// Determine the nodes for partition.
 		nodes := c.partitionNodes(p)
 		for _, n := range nodes {
@@ -998,11 +1121,11 @@ func (c *cluster) waitForStarted() error {
 	if !c.isCoordinator() {
 		// In the case where a node has been restarted and memberlist has
 		// not had enough time to determine the node went down/up, then
-		// the coorninator needs to be alerted that this node is back up
+		// the coordinator needs to be alerted that this node is back up
 		// (and now in a state of STARTING) so that it can be put to the correct
 		// cluster state.
 		// TODO: Because the normal code path already sends a NodeJoin event (via
-		// memberlist), this it a bit redundant in most cases. Perhaps determine
+		// memberlist), this is a bit redundant in most cases. Perhaps determine
 		// that the node has been restarted and don't do this step.
 		msg := &NodeEvent{
 			Event: NodeJoin,
@@ -1127,7 +1250,6 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
 	// Broadcast cluster status changes to the cluster.
 	status := c.unprotectedStatus()
 	return c.unprotectedSendSync(status) // TODO fix c.Status
-
 }
 
 func (c *cluster) sendTo(node *Node, m Message) error {
@@ -1234,38 +1356,82 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (*
 		toCluster.addNodeBasicSorted(nodeAction.node)
 	}
 
-	// multiIndex is a map of sources initialized with all the nodes in toCluster.
-	multiIndex := make(map[string][]*ResizeSource)
+	indexes := c.holder.Indexes()
 
+	// fragmentSourcesByNode is a map of Node.ID to sources of fragment data.
+	// It is initialized with all the nodes in toCluster.
+	fragmentSourcesByNode := make(map[string][]*ResizeSource)
 	for _, n := range toCluster.nodes {
-		multiIndex[n.ID] = nil
+		fragmentSourcesByNode[n.ID] = nil
 	}
 
-	// Add to multiIndex the instructions for each index.
-	for _, idx := range c.holder.Indexes() {
+	// Add to fragmentSourcesByNode the instructions for each index.
+	for _, idx := range indexes {
 		fragSources, err := c.fragSources(toCluster, idx)
 		if err != nil {
 			return nil, errors.Wrap(err, "getting sources")
 		}
 
-		for id, sources := range fragSources {
-			multiIndex[id] = append(multiIndex[id], sources...)
+		for nodeid, sources := range fragSources {
+			fragmentSourcesByNode[nodeid] = append(fragmentSourcesByNode[nodeid], sources...)
 		}
 	}
 
-	for id, sources := range multiIndex {
+	// translationSourcesByNode is a map of Node.ID to sources of partitioned
+	// key translation data for indexes.
+	// It is initialized with all the nodes in toCluster.
+	translationSourcesByNode := make(map[string][]*TranslationResizeSource)
+	for _, n := range toCluster.nodes {
+		translationSourcesByNode[n.ID] = nil
+	}
+
+	if len(indexes) > 0 {
+		// Add to translationSourcesByNode the instructions for the cluster.
+		translationNodes, err := c.translationNodes(toCluster)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting translation sources")
+		}
+
+		// Create a list of TranslationResizeSource for each index,
+		// using translationNodes as a template.
+		translationSources := make(map[string][]*TranslationResizeSource)
+		for _, idx := range indexes {
+			// Only include indexes with keys.
+			if !idx.Keys() {
+				continue
+			}
+			indexName := idx.Name()
+			for node, resizeNodes := range translationNodes {
+				for i := range resizeNodes {
+					translationSources[node] = append(translationSources[node],
+						&TranslationResizeSource{
+							Node:        resizeNodes[i].node,
+							Index:       indexName,
+							PartitionID: resizeNodes[i].partitionID,
+						})
+				}
+			}
+		}
+
+		for nodeid, sources := range translationSources {
+			translationSourcesByNode[nodeid] = sources
+		}
+	}
+
+	for _, node := range toCluster.nodes {
 		// If a host doesn't need to request data, mark it as complete.
-		if len(sources) == 0 {
-			j.IDs[id] = true
+		if len(fragmentSourcesByNode[node.ID]) == 0 && len(translationSourcesByNode[node.ID]) == 0 {
+			j.IDs[node.ID] = true
 			continue
 		}
 		instr := &ResizeInstruction{
-			JobID:         j.ID,
-			Node:          toCluster.unprotectedNodeByID(id),
-			Coordinator:   c.unprotectedCoordinatorNode(),
-			Sources:       sources,
-			NodeStatus:    c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node.
-			ClusterStatus: c.unprotectedStatus(),
+			JobID:              j.ID,
+			Node:               toCluster.unprotectedNodeByID(node.ID),
+			Coordinator:        c.unprotectedCoordinatorNode(),
+			Sources:            fragmentSourcesByNode[node.ID],
+			TranslationSources: translationSourcesByNode[node.ID],
+			NodeStatus:         c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node.
+			ClusterStatus:      c.unprotectedStatus(),
 		}
 		j.Instructions = append(j.Instructions, instr)
 	}
@@ -1302,7 +1468,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 		return errors.Wrap(err, "merging cluster status")
 	}
 
-	c.logger.Printf("done MergeClusterStatus, start goroutine")
+	c.logger.Printf("done MergeClusterStatus, start goroutine (%s)", c.Node.ID)
 
 	// The actual resizing runs in a goroutine because we don't want to block
 	// the distribution of other ResizeInstructions to the rest of the cluster.
@@ -1349,9 +1515,8 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 
 			// Request each source file in ResizeSources.
 			for _, src := range instr.Sources {
-				c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
-
 				srcURI := src.Node.URI
+				c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, srcURI)
 
 				// Retrieve field.
 				f := c.holder.Field(src.Index, src.Field)
@@ -1372,7 +1537,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 				}
 
 				// Stream shard from remote node.
-				c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
+				c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, srcURI)
 				rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.View, src.Shard, srcURI)
 				if err != nil {
 					// For now it is an acceptable error if the fragment is not found
@@ -1386,7 +1551,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 					}
 					return errors.Wrap(err, "retrieving shard")
 				} else if rd == nil {
-					return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI)
+					return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, srcURI)
 				}
 
 				// Write to local field and always close reader.
@@ -1398,6 +1563,37 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
 					return errors.Wrap(err, "copying remote shard")
 				}
 			}
+
+			// Request each translation source file in TranslationResizeSources.
+			for _, src := range instr.TranslationSources {
+				srcURI := src.Node.URI
+
+				idx := c.holder.Index(src.Index)
+				if idx == nil {
+					return ErrIndexNotFound
+				}
+
+				// Retrieve partition from remote node.
+				c.logger.Printf("retrieve translate partition %d for index %s from host %s", src.PartitionID, src.Index, srcURI)
+				rd, err := c.InternalClient.RetrieveTranslatePartitionFromURI(ctx, src.Index, src.PartitionID, srcURI)
+				if err != nil {
+					return errors.Wrap(err, "retrieving translate partition")
+				} else if rd == nil {
+					return fmt.Errorf("partition %d doesn't exist on host: %s", src.PartitionID, src.Node.URI)
+				}
+
+				// Write to local store and always close reader.
+				if err := func() error {
+					defer rd.Close()
+					// Get the translate store for this index/partition.
+					store := idx.TranslateStore(src.PartitionID)
+					_, err = store.ReadFrom(rd)
+					return errors.Wrap(err, "reading from reader")
+				}(); err != nil {
+					return errors.Wrap(err, "copying remote partition")
+				}
+			}
+
 			return nil
 		}(); err != nil {
 			complete.Error = err.Error()
@@ -1794,6 +1990,17 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 
 // nodeJoin should only be called by the coordinator.
 func (c *cluster) nodeJoin(node *Node) error {
+	c.abortAntiEntropy()
+	// Technically there is a race condition here which could
+	// allow the anti-entropy process to re-start (and acquire
+	// the lock) before this lock has time to succeed. In that
+	// case, the user would have to wait through an entire
+	// anti-entropy cycle. We decided it wasn't worth the
+	// complexity (of, for example, implementing this with
+	// channels) to avoid that rare case.
+	c.muAntiEntropy.Lock()
+	defer c.muAntiEntropy.Unlock()
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.logger.Printf("node join event on coordinator, node: %s, id: %s", node.URI, node.ID)
@@ -1864,6 +2071,17 @@ func (c *cluster) nodeJoin(node *Node) error {
 
 // nodeLeave initiates the removal of a node from the cluster.
 func (c *cluster) nodeLeave(nodeID string) error {
+	c.abortAntiEntropy()
+	// Technically there is a race condition here which could
+	// allow the anti-entropy process to re-start (and acquire
+	// the lock) before this lock has time to succeed. In that
+	// case, the user would have to wait through an entire
+	// anti-entropy cycle. We decided it wasn't worth the
+	// complexity (of, for example, implementing this with
+	// channels) to avoid that rare case.
+	c.muAntiEntropy.Lock()
+	defer c.muAntiEntropy.Unlock()
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	// Refuse the request if this is not the coordinator.
@@ -1873,8 +2091,8 @@ func (c *cluster) nodeLeave(nodeID string) error {
 	}
 
 	if c.state != ClusterStateNormal && c.state != ClusterStateDegraded {
-		return fmt.Errorf("cluster must be '%s' to remove a node but is '%s'",
-			ClusterStateNormal, c.state)
+		return fmt.Errorf("cluster must be '%s' or '%s' to remove a node but is '%s'",
+			ClusterStateNormal, ClusterStateDegraded, c.state)
 	}
 
 	// Ensure that node is in the cluster.
@@ -2047,6 +2265,180 @@ func (c *cluster) setStatic(hosts []string) error {
 	return nil
 }
 
+// translateFieldKey gets a single key from translateFieldKeys.
+func (c *cluster) translateFieldKey(ctx context.Context, field *Field, key string) (uint64, error) {
+	ids, err := c.translateFieldKeys(ctx, field, key)
+	if err != nil {
+		return 0, err
+	} else if len(ids) == 0 {
+		return 0, errors.New("translating key on coordinator returned empty set")
+	}
+	return ids[0], nil
+}
+
+// translateFieldKeys is basically a wrapper around
+// field.TranslateStore().TranslateKey(key), but in
+// the case where the local node's translate store
+// is read-only (i.e. it's not the primary translate
+// store), then this method will forward the translation
+// request to the coordinator.
+func (c *cluster) translateFieldKeys(ctx context.Context, field *Field, keys ...string) ([]uint64, error) {
+	ids, err := field.TranslateStore().TranslateKeys(keys)
+	// If we get a "read only" error, then forward the request
+	// to the coordinator.
+	if errors.Cause(err) == ErrTranslateStoreReadOnly {
+		coordinatorNode := c.coordinatorNode()
+		if ids, err := c.InternalClient.TranslateKeysNode(ctx, &coordinatorNode.URI, field.Index(), field.Name(), keys); err != nil {
+			return ids, errors.Wrap(err, "translating keys on coordinator")
+		} else {
+			return ids, nil
+		}
+	}
+	return ids, err
+}
+
+func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}})
+	if err != nil {
+		return 0, err
+	}
+	return keyMap[key], nil
+}
+
+func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys []string) ([]uint64, error) {
+	keySet := make(map[string]struct{})
+	for _, key := range keys {
+		keySet[key] = struct{}{}
+	}
+
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, keySet)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make([]uint64, len(keys))
+	for i := range keys {
+		ids[i] = keyMap[keys[i]]
+	}
+	return ids, nil
+}
+
+func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}) (map[string]uint64, error) {
+	keyMap := make(map[string]uint64)
+
+	idx := c.holder.Index(indexName)
+	if idx == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split keys by partition.
+	keysByPartition := make(map[int][]string, c.partitionN)
+	for key := range keySet {
+		partitionID := c.keyPartition(indexName, key)
+		keysByPartition[partitionID] = append(keysByPartition[partitionID], key)
+	}
+
+	// Translate keys by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range keysByPartition {
+		partitionID := partitionID
+		keys := keysByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var ids []uint64
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, indexName, "", keys); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range keys {
+				keyMap[keys[i]] = ids[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return keyMap, nil
+}
+
+func (c *cluster) translateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	idSet := make(map[uint64]struct{})
+	for _, id := range ids {
+		idSet[id] = struct{}{}
+	}
+
+	idMap, err := c.translateIndexIDSet(ctx, indexName, idSet)
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, len(ids))
+	for i := range ids {
+		keys[i] = idMap[ids[i]]
+	}
+	return keys, nil
+}
+
+func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idSet map[uint64]struct{}) (map[uint64]string, error) {
+	idMap := make(map[uint64]string)
+
+	index := c.holder.Index(indexName)
+	if index == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split ids by partition.
+	idsByPartition := make(map[int][]uint64, c.partitionN)
+	for id := range idSet {
+		partitionID := c.idPartition(indexName, id)
+		idsByPartition[partitionID] = append(idsByPartition[partitionID], id)
+	}
+
+	// Translate ids by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range idsByPartition {
+		partitionID := partitionID
+		ids := idsByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var keys []string
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if keys, err = c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, indexName, "", ids); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range ids {
+				idMap[ids[i]] = keys[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return idMap, nil
+}
+
 // ClusterStatus describes the status of the cluster including its
 // state and node topology.
 type ClusterStatus struct {
@@ -2058,12 +2450,13 @@ type ClusterStatus struct {
 // ResizeInstruction contains the instruction provided to a node
 // during a cluster resize operation.
 type ResizeInstruction struct {
-	JobID         int64
-	Node          *Node
-	Coordinator   *Node
-	Sources       []*ResizeSource
-	NodeStatus    *NodeStatus
-	ClusterStatus *ClusterStatus
+	JobID              int64
+	Node               *Node
+	Coordinator        *Node
+	Sources            []*ResizeSource
+	TranslationSources []*TranslationResizeSource
+	NodeStatus         *NodeStatus
+	ClusterStatus      *ClusterStatus
 }
 
 // ResizeSource is the source of data for a node acting on a
@@ -2076,6 +2469,21 @@ type ResizeSource struct {
 	Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
 }
 
+// TranslationResizeSource is the source of translation data for
+// a node acting on a ResizeInstruction.
+type TranslationResizeSource struct {
+	Node        *Node
+	Index       string
+	PartitionID int
+}
+
+// translateResizeNode holds the node/partition pairs used
+// to create a TranslationResizeSource for each index.
+type translationResizeNode struct {
+	node        *Node
+	partitionID int
+}
+
 // Schema contains information about indexes and their configuration.
 type Schema struct {
 	Indexes []*IndexInfo
@@ -2205,3 +2613,15 @@ type FieldStatus struct {
 // RecalculateCaches is an internal message for recalculating all caches
 // within a holder.
 type RecalculateCaches struct{}
+
+// Transaction Actions
+const (
+	TRANSACTION_START    = "start"
+	TRANSACTION_FINISH   = "finish"
+	TRANSACTION_VALIDATE = "validate"
+)
+
+type TransactionMessage struct {
+	Transaction *Transaction
+	Action      string
+}

cluster_internal_test.go

@@ -96,7 +96,7 @@ func newIndexWithTempPath(name string) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	index, err := NewIndex(path, name, DefaultPartitionN)
 	if err != nil {
 		panic(err)
 	}
@@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
+	defer idx.Close()
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
@@ -351,7 +352,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := c.shardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -948,6 +949,9 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
 
 }
 func TestCluster_confirmNodeDownTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	r := mux.NewRouter()
 	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
@@ -973,10 +977,12 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
 	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
 		t.Errorf("expected node to be down")
 	}
-
 }
 
 func TestCluster_confirmNodeDownDown(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	uri := URI{}
 	uri.Scheme = "http"
 	uri.Host = "DoesntMatter"
@@ -985,5 +991,4 @@ func TestCluster_confirmNodeDownDown(t *testing.T) {
 	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
 		t.Errorf("expected node to be down")
 	}
-
 }

cmd/import.go

@@ -53,9 +53,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")

cmd/import_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 
 	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 func TestImportHelp(t *testing.T) {
@@ -58,8 +59,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
-					Min:       -10,
+					Max:       pql.NewDecimal(100, 0),
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/root_test.go

@@ -55,6 +55,9 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 	if err := w.Close(); err != nil {
 		return output, fmt.Errorf("closing cmd's stdout: %v", err)
 	}
+
+	// NOTE: if cmd.Execute doesn't return, then this select (and
+	// therefore the one-second timeout, won't be reached)
 	select {
 	case <-done:
 	case <-time.After(time.Second * 1):
@@ -64,7 +67,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
 }
 
 // ExecNewRootCommand executes the pilosa root command with the given arguments
-// and returns it's output. It will fail if the command does not complete within
+// and returns its output. It will fail if the command does not complete within
 // 1 second.
 func ExecNewRootCommand(t *testing.T, args ...string) (string, error) {
 	out, w := io.Pipe()

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -53,6 +53,7 @@ func TestServerConfig(t *testing.T) {
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	max-writes-per-request = 3000
 
 	[cluster]
@@ -96,6 +97,7 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		disabled = true
@@ -122,6 +124,7 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		hosts = [

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pkg/errors"
 )
@@ -102,11 +104,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
-			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
+			} else if cmd.FieldOptions.Min != pql.NewDecimal(0, 0) || cmd.FieldOptions.Max != pql.NewDecimal(0, 0) {
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +167,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -285,9 +289,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +315,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +334,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
-				return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-			}
+			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
 
-		// Parse FieldValue.
-		value, err := strconv.ParseInt(record[1], 10, 64)
-		if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
-		}
-		val.Value = value
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
 			}
-			a = a[:0]
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
+			value, err := strconv.ParseInt(record[1], 10, 64)
+			if err != nil {
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
+			}
+			req.Values = append(req.Values, value)
+		}
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
@@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
-		qry := "Count(Row(f=foo0))"
-		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
-		if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
-		}
+		if err := test.RetryUntil(2*time.Second, func() error {
+			qry := "Count(Row(f=foo0))"
+			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
+			if err != nil {
+				return fmt.Errorf("Querying data for validation: %s", err)
+			}
 
-		// Read body and unmarshal response.
-		exp := `{"results":[100]}` + "\n"
-		if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
-		} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+			// Read body and unmarshal response.
+			exp := `{"results":[100]}` + "\n"
+			if body, err := ioutil.ReadAll(resp.Body); err != nil {
+				return fmt.Errorf("reading: %s", err)
+			} else if !reflect.DeepEqual(body, []byte(exp)) {
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.

ctl/server.go

@@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
@@ -70,14 +71,14 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
 
 	// Metric
-	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), statsd or none.")
+	flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
 	flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
 	flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
 	flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
 
 	// Tracing
 	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
-	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type or 'off' to disable tracing completely.")
+	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
 	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
 
 	// Profiling

diagnostics.go

@@ -233,7 +233,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {

docs/configuration.md

@@ -403,7 +403,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
 
 #### Tracing Sampler Type
 
-* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely.
+* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely. Default is 'off'.
 * Flag: `tracing.sampler-type`
 * Env: `PILOSA_TRACING_SAMPLER_TYPE`
 * Config:

docs/data-model.md

@@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
 #### Set
 
 Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
+Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
+```go
+[]byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+```
+(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
+In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
 
 ``` request
 curl localhost:10101/index/repository/field/info \

docs/getting-started.md

@@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
 ``` response
 {"success":true}
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
@@ -325,7 +325,7 @@ Next, let's create the `repository` index:
 	repository := schema.Index("repository")
 ```
 
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -615,7 +615,7 @@ Next, let's create the `repository` index:
 ```
 	Index repository = schema.index("repository");
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -818,7 +818,7 @@ Next, let's create the `repository` index:
 ```
 repository = schema.index("repository")
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```

docs/query-language.md

@@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
 * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
 * `UINT` An unsigned integer (e.g. 42839).
 * `BOOL` A boolean value, `true` or `false`.
@@ -864,7 +864,7 @@ Rows(job)
 **Spec:**
 
 ```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
+GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>, aggregate=<CALL>)
 ```
 
 **Description:**
@@ -874,14 +874,18 @@ taking one row each from the specified `Rows` calls. It returns only those
 combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
- Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
- query.
+Intersect, etc.) which will be intersected with each result prior to returning
+the count. This is analagous to a WHERE clause applied to a relational GROUP BY
+query.
 
 The optional `limit` argument limits the number of results returned. The results
 are ordered, so as long as the data isn't changing, the same query will return
 the same result set.
 
+The optional `aggregate` argument takes a `Sum()` query which will be used to
+calculate the sum & count of each group. This is similar to using a `SUM()` in
+the SELECT clause of a relation GROUP BY query.
+
 Paging through results is supported by passing the `previous` argument to each
 of the `Rows` calls in the GroupBy. Take the last result from your previous
 `GroupBy` query, and pass each row ID in that result as the `previous` argument

encoding/proto/proto.go

@@ -17,10 +17,12 @@ package proto
 import (
 	"fmt"
 	"sort"
+	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/internal"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -225,6 +227,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeImportRoaringRequest(msg, mt)
 		return nil
+	case *pilosa.ImportColumnAttrsRequest:
+		msg := &internal.ImportColumnAttrsRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ImportColumnAttrsRequest")
+		}
+		decodeImportColumnAttrsRequest(msg, mt)
+		return nil
 	case *pilosa.ImportResponse:
 		msg := &internal.ImportResponse{}
 		err := proto.Unmarshal(buf, msg)
@@ -265,6 +275,30 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeTranslateKeysResponse(msg, mt)
 		return nil
+	case *pilosa.TranslateIDsRequest:
+		msg := &internal.TranslateIDsRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling TranslateIDsRequest")
+		}
+		decodeTranslateIDsRequest(msg, mt)
+		return nil
+	case *pilosa.TranslateIDsResponse:
+		msg := &internal.TranslateIDsResponse{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling TranslateIDsResponse")
+		}
+		decodeTranslateIDsResponse(msg, mt)
+		return nil
+	case *pilosa.TransactionMessage:
+		msg := &internal.TransactionMessage{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling TransactionMessage")
+		}
+		decodeTransactionMessage(msg, mt)
+		return nil
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -318,6 +352,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeImportValueRequest(mt)
 	case *pilosa.ImportRoaringRequest:
 		return encodeImportRoaringRequest(mt)
+	case *pilosa.ImportColumnAttrsRequest:
+		return encodeImportColumnAttrsRequest(mt)
 	case *pilosa.ImportResponse:
 		return encodeImportResponse(mt)
 	case *pilosa.BlockDataRequest:
@@ -328,6 +364,12 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeTranslateKeysRequest(mt)
 	case *pilosa.TranslateKeysResponse:
 		return encodeTranslateKeysResponse(mt)
+	case *pilosa.TranslateIDsRequest:
+		return encodeTranslateIDsRequest(mt)
+	case *pilosa.TranslateIDsResponse:
+		return encodeTranslateIDsResponse(mt)
+	case *pilosa.TransactionMessage:
+		return encodeTransactionMessage(mt)
 	}
 	return nil
 }
@@ -369,12 +411,14 @@ func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
 
 func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
 	return &internal.ImportValueRequest{
-		Index:      m.Index,
-		Field:      m.Field,
-		Shard:      m.Shard,
-		ColumnIDs:  m.ColumnIDs,
-		ColumnKeys: m.ColumnKeys,
-		Values:     m.Values,
+		Index:        m.Index,
+		Field:        m.Field,
+		Shard:        m.Shard,
+		ColumnIDs:    m.ColumnIDs,
+		ColumnKeys:   m.ColumnKeys,
+		Values:       m.Values,
+		FloatValues:  m.FloatValues,
+		StringValues: m.StringValues,
 	}
 }
 
@@ -386,23 +430,40 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import
 			Name: viewName,
 			Data: viewData,
 		}
-		i += 1
+		i++
 	}
 	return &internal.ImportRoaringRequest{
-		Clear: m.Clear,
-		Views: views,
+		Clear:  m.Clear,
+		Action: m.Action,
+		Block:  uint64(m.Block),
+		Views:  views,
+	}
+}
+
+func encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest {
+	return &internal.ImportColumnAttrsRequest{
+		Index:     m.Index,
+		Shard:     m.Shard,
+		AttrKey:   m.AttrKey,
+		AttrVals:  m.AttrVals,
+		ColumnIDs: m.ColumnIDs,
 	}
 }
 
 func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
-	return &internal.QueryRequest{
+	r := &internal.QueryRequest{
 		Query:           m.Query,
 		Shards:          m.Shards,
 		ColumnAttrs:     m.ColumnAttrs,
 		Remote:          m.Remote,
 		ExcludeRowAttrs: m.ExcludeRowAttrs,
 		ExcludeColumns:  m.ExcludeColumns,
+		EmbeddedData:    make([]*internal.Row, len(m.EmbeddedData)),
 	}
+	for i := range m.EmbeddedData {
+		r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i])
+	}
+	return r
 }
 
 func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
@@ -415,12 +476,18 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		pb.Results[i] = &internal.QueryResult{}
 
 		switch result := m.Results[i].(type) {
+		case pilosa.SignedRow:
+			pb.Results[i].Type = queryResultTypeSignedRow
+			pb.Results[i].SignedRow = encodeSignedRow(result)
 		case *pilosa.Row:
 			pb.Results[i].Type = queryResultTypeRow
 			pb.Results[i].Row = encodeRow(result)
 		case []pilosa.Pair:
 			pb.Results[i].Type = queryResultTypePairs
 			pb.Results[i].Pairs = encodePairs(result)
+		case *pilosa.PairsField:
+			pb.Results[i].Type = queryResultTypePairsField
+			pb.Results[i].PairsField = encodePairsField(result)
 		case pilosa.ValCount:
 			pb.Results[i].Type = queryResultTypeValCount
 			pb.Results[i].ValCount = encodeValCount(result)
@@ -442,10 +509,13 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		case pilosa.Pair:
 			pb.Results[i].Type = queryResultTypePair
 			pb.Results[i].Pairs = []*internal.Pair{encodePair(result)}
+		case pilosa.PairField:
+			pb.Results[i].Type = queryResultTypePairField
+			pb.Results[i].Pairs = []*internal.Pair{encodePairField(result)}
 		case nil:
 			pb.Results[i].Type = queryResultTypeNil
 		default:
-			panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type))
+			panic(fmt.Errorf("unknown type: %T", m.Results[i]))
 		}
 	}
 
@@ -458,12 +528,13 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 
 func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction {
 	return &internal.ResizeInstruction{
-		JobID:         m.JobID,
-		Node:          encodeNode(m.Node),
-		Coordinator:   encodeNode(m.Coordinator),
-		Sources:       encodeResizeSources(m.Sources),
-		NodeStatus:    encodeNodeStatus(m.NodeStatus),
-		ClusterStatus: encodeClusterStatus(m.ClusterStatus),
+		JobID:              m.JobID,
+		Node:               encodeNode(m.Node),
+		Coordinator:        encodeNode(m.Coordinator),
+		Sources:            encodeResizeSources(m.Sources),
+		TranslationSources: encodeTranslationResizeSources(m.TranslationSources),
+		NodeStatus:         encodeNodeStatus(m.NodeStatus),
+		ClusterStatus:      encodeClusterStatus(m.ClusterStatus),
 	}
 }
 
@@ -485,6 +556,22 @@ func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource {
 	}
 }
 
+func encodeTranslationResizeSources(srcs []*pilosa.TranslationResizeSource) []*internal.TranslationResizeSource {
+	new := make([]*internal.TranslationResizeSource, 0, len(srcs))
+	for _, src := range srcs {
+		new = append(new, encodeTranslationResizeSource(src))
+	}
+	return new
+}
+
+func encodeTranslationResizeSource(m *pilosa.TranslationResizeSource) *internal.TranslationResizeSource {
+	return &internal.TranslationResizeSource{
+		Node:        encodeNode(m.Node),
+		Index:       m.Index,
+		PartitionID: int32(m.PartitionID),
+	}
+}
+
 func encodeSchema(m *pilosa.Schema) *internal.Schema {
 	return &internal.Schema{
 		Indexes: encodeIndexInfos(m.Indexes),
@@ -501,8 +588,9 @@ func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index {
 
 func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index {
 	return &internal.Index{
-		Name:   idx.Name,
-		Fields: encodeFieldInfos(idx.Fields),
+		Name:    idx.Name,
+		Options: encodeIndexMeta(&idx.Options),
+		Fields:  encodeFieldInfos(idx.Fields),
 	}
 }
 
@@ -532,15 +620,17 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
 		return nil
 	}
 	return &internal.FieldOptions{
-		Type:        o.Type,
-		CacheType:   o.CacheType,
-		CacheSize:   o.CacheSize,
-		Min:         o.Min,
-		Max:         o.Max,
-		Base:        o.Base,
-		BitDepth:    uint64(o.BitDepth),
-		TimeQuantum: string(o.TimeQuantum),
-		Keys:        o.Keys,
+		Type:         o.Type,
+		CacheType:    o.CacheType,
+		CacheSize:    o.CacheSize,
+		Min:          &internal.Decimal{Value: o.Min.Value, Scale: o.Min.Scale},
+		Max:          &internal.Decimal{Value: o.Max.Value, Scale: o.Max.Scale},
+		Base:         o.Base,
+		Scale:        o.Scale,
+		BitDepth:     uint64(o.BitDepth),
+		TimeQuantum:  string(o.TimeQuantum),
+		Keys:         o.Keys,
+		ForeignIndex: o.ForeignIndex,
 	}
 }
 
@@ -722,12 +812,6 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac
 	return &internal.RecalculateCaches{}
 }
 
-func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse {
-	return &internal.TranslateKeysResponse{
-		IDs: response.IDs,
-	}
-}
-
 func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest {
 	return &internal.TranslateKeysRequest{
 		Index: request.Index,
@@ -736,6 +820,58 @@ func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.
 	}
 }
 
+func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse {
+	return &internal.TranslateKeysResponse{
+		IDs: response.IDs,
+	}
+}
+
+func encodeTranslateIDsRequest(request *pilosa.TranslateIDsRequest) *internal.TranslateIDsRequest {
+	return &internal.TranslateIDsRequest{
+		Index: request.Index,
+		Field: request.Field,
+		IDs:   request.IDs,
+	}
+}
+
+func encodeTranslateIDsResponse(response *pilosa.TranslateIDsResponse) *internal.TranslateIDsResponse {
+	return &internal.TranslateIDsResponse{
+		Keys: response.Keys,
+	}
+}
+
+func encodeTransactionMessage(msg *pilosa.TransactionMessage) *internal.TransactionMessage {
+	return &internal.TransactionMessage{
+		Action:      msg.Action,
+		Transaction: encodeTransaction(msg.Transaction),
+	}
+}
+
+func encodeTransaction(trns *pilosa.Transaction) *internal.Transaction {
+	if trns == nil {
+		return nil
+	}
+	return &internal.Transaction{
+		ID:        trns.ID,
+		Active:    trns.Active,
+		Exclusive: trns.Exclusive,
+		Timeout:   int64(trns.Timeout),
+		Deadline:  encodeTransactionDeadline(trns.Deadline),
+		Stats:     encodeTransactionStats(trns.Stats),
+	}
+}
+
+func encodeTransactionDeadline(deadline time.Time) int64 {
+	if deadline.Year() > 2262 || deadline.Year() < 1678 {
+		return 0
+	}
+	return deadline.UnixNano()
+}
+
+func encodeTransactionStats(stats pilosa.TransactionStats) *internal.TransactionStats {
+	return &internal.TransactionStats{}
+}
+
 func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) {
 	m.JobID = ri.JobID
 	m.Node = &pilosa.Node{}
@@ -744,6 +880,8 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeIns
 	decodeNode(ri.Coordinator, m.Coordinator)
 	m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources))
 	decodeResizeSources(ri.Sources, m.Sources)
+	m.TranslationSources = make([]*pilosa.TranslationResizeSource, len(ri.TranslationSources))
+	decodeTranslationResizeSources(ri.TranslationSources, m.TranslationSources)
 	m.NodeStatus = &pilosa.NodeStatus{}
 	decodeNodeStatus(ri.NodeStatus, m.NodeStatus)
 	m.ClusterStatus = &pilosa.ClusterStatus{}
@@ -766,6 +904,20 @@ func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) {
 	m.Shard = rs.Shard
 }
 
+func decodeTranslationResizeSources(srcs []*internal.TranslationResizeSource, m []*pilosa.TranslationResizeSource) {
+	for i := range srcs {
+		m[i] = &pilosa.TranslationResizeSource{}
+		decodeTranslationResizeSource(srcs[i], m[i])
+	}
+}
+
+func decodeTranslationResizeSource(rs *internal.TranslationResizeSource, m *pilosa.TranslationResizeSource) {
+	m.Node = &pilosa.Node{}
+	decodeNode(rs.Node, m.Node)
+	m.Index = rs.Index
+	m.PartitionID = int(rs.PartitionID)
+}
+
 func decodeSchema(s *internal.Schema, m *pilosa.Schema) {
 	m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes))
 	decodeIndexes(s.Indexes, m.Indexes)
@@ -780,6 +932,8 @@ func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) {
 
 func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) {
 	m.Name = idx.Name
+	m.Options = pilosa.IndexOptions{}
+	decodeIndexMeta(idx.Options, &m.Options)
 	m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields))
 	decodeFields(idx.Fields, m.Fields)
 }
@@ -805,12 +959,19 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions)
 	m.Type = options.Type
 	m.CacheType = options.CacheType
 	m.CacheSize = options.CacheSize
-	m.Min = options.Min
-	m.Max = options.Max
+	decodeDecimal(options.Min, &m.Min)
+	decodeDecimal(options.Max, &m.Max)
 	m.Base = options.Base
+	m.Scale = options.Scale
 	m.BitDepth = uint(options.BitDepth)
 	m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
 	m.Keys = options.Keys
+	m.ForeignIndex = options.ForeignIndex
+}
+
+func decodeDecimal(d *internal.Decimal, m *pql.Decimal) {
+	m.Value = d.Value
+	m.Scale = d.Scale
 }
 
 func decodeNodes(a []*internal.Node, m []*pilosa.Node) {
@@ -853,8 +1014,10 @@ func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateI
 }
 
 func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) {
-	m.Keys = pb.Keys
-	m.TrackExistence = pb.TrackExistence
+	if pb != nil {
+		m.Keys = pb.Keys
+		m.TrackExistence = pb.TrackExistence
+	}
 }
 
 func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) {
@@ -963,6 +1126,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
 	m.Remote = pb.Remote
 	m.ExcludeRowAttrs = pb.ExcludeRowAttrs
 	m.ExcludeColumns = pb.ExcludeColumns
+	m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
+	for i := range pb.EmbeddedData {
+		m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i])
+	}
 }
 
 func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
@@ -983,6 +1150,8 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV
 	m.ColumnIDs = pb.ColumnIDs
 	m.ColumnKeys = pb.ColumnKeys
 	m.Values = pb.Values
+	m.FloatValues = pb.FloatValues
+	m.StringValues = pb.StringValues
 }
 
 func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {
@@ -991,9 +1160,19 @@ func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.Imp
 		views[view.Name] = view.Data
 	}
 	m.Clear = pb.Clear
+	m.Action = pb.Action
+	m.Block = int(pb.Block)
 	m.Views = views
 }
 
+func decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) {
+	m.Index = pb.Index
+	m.Shard = pb.Shard
+	m.AttrKey = pb.AttrKey
+	m.AttrVals = pb.AttrVals
+	m.ColumnIDs = pb.ColumnIDs
+}
+
 func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) {
 	m.Err = pb.Err
 }
@@ -1052,11 +1231,43 @@ func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.T
 	m.IDs = pb.IDs
 }
 
+func decodeTranslateIDsRequest(pb *internal.TranslateIDsRequest, m *pilosa.TranslateIDsRequest) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.IDs = pb.IDs
+}
+
+func decodeTranslateIDsResponse(pb *internal.TranslateIDsResponse, m *pilosa.TranslateIDsResponse) {
+	m.Keys = pb.Keys
+}
+
+func decodeTransactionMessage(pb *internal.TransactionMessage, m *pilosa.TransactionMessage) {
+	m.Action = pb.Action
+	if pb.Transaction == nil {
+		m.Transaction = nil
+		return
+	} else if m.Transaction == nil {
+		m.Transaction = &pilosa.Transaction{}
+	}
+	decodeTransaction(pb.Transaction, m.Transaction)
+}
+
+func decodeTransaction(pb *internal.Transaction, trns *pilosa.Transaction) {
+
+	trns.ID = pb.ID
+	trns.Active = pb.Active
+	trns.Exclusive = pb.Exclusive
+	trns.Timeout = time.Duration(pb.Timeout)
+	trns.Deadline = time.Unix(0, pb.Deadline)
+	// TODO m.Stats... once it has anything
+}
+
 // QueryResult types.
 const (
 	queryResultTypeNil uint32 = iota
 	queryResultTypeRow
 	queryResultTypePairs
+	queryResultTypePairsField
 	queryResultTypeValCount
 	queryResultTypeUint64
 	queryResultTypeBool
@@ -1064,14 +1275,20 @@ const (
 	queryResultTypeGroupCounts
 	queryResultTypeRowIdentifiers
 	queryResultTypePair
+	queryResultTypePairField
+	queryResultTypeSignedRow
 )
 
 func decodeQueryResult(pb *internal.QueryResult) interface{} {
 	switch pb.Type {
+	case queryResultTypeSignedRow:
+		return decodeSignedRow(pb.SignedRow)
 	case queryResultTypeRow:
 		return decodeRow(pb.Row)
 	case queryResultTypePairs:
 		return decodePairs(pb.Pairs)
+	case queryResultTypePairsField:
+		return decodePairsField(pb.PairsField)
 	case queryResultTypeValCount:
 		return decodeValCount(pb.ValCount)
 	case queryResultTypeUint64:
@@ -1088,21 +1305,40 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
 		return decodeGroupCounts(pb.GroupCounts)
 	case queryResultTypePair:
 		return decodePair(pb.Pairs[0])
+	case queryResultTypePairField:
+		return decodePairField(pb.Pairs[0])
 	}
 	panic(fmt.Sprintf("unknown type: %d", pb.Type))
 }
 
-// DecodeRow converts r from its internal representation.
+// decodeRow converts r from its internal representation.
 func decodeRow(pr *internal.Row) *pilosa.Row {
 	if pr == nil {
-		return nil
+		return pilosa.NewRow()
 	}
 
-	r := pilosa.NewRow()
+	var r *pilosa.Row
+	if len(pr.Roaring) > 0 {
+		r = pilosa.NewRowFromRoaring(pr.Roaring)
+	} else {
+		r = pilosa.NewRow()
+		for _, v := range pr.Columns {
+			r.SetBit(v)
+		}
+	}
 	r.Attrs = decodeAttrs(pr.Attrs)
 	r.Keys = pr.Keys
-	for _, v := range pr.Columns {
-		r.SetBit(v)
+
+	return r
+}
+
+func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow {
+	if pr == nil {
+		return pilosa.SignedRow{}
+	}
+	r := pilosa.SignedRow{
+		Pos: decodeRow(pr.Pos),
+		Neg: decodeRow(pr.Neg),
 	}
 	return r
 }
@@ -1151,6 +1387,7 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount {
 		other[i] = pilosa.GroupCount{
 			Group: decodeFieldRows(a[i].Group),
 			Count: a[i].Count,
+			Sum:   a[i].Sum,
 		}
 	}
 	return other
@@ -1166,6 +1403,9 @@ func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow {
 		} else {
 			other[i].RowKey = fr.RowKey
 		}
+		if fr.Value != nil {
+			other[i].Value = &fr.Value.Value
+		}
 	}
 	return other
 }
@@ -1178,6 +1418,17 @@ func decodePairs(a []*internal.Pair) []pilosa.Pair {
 	return other
 }
 
+func decodePairsField(a *internal.PairsField) *pilosa.PairsField {
+	other := &pilosa.PairsField{
+		Pairs: make([]pilosa.Pair, len(a.Pairs)),
+	}
+	for i := range a.Pairs {
+		other.Pairs[i] = decodePair(a.Pairs[i])
+	}
+	other.Field = a.Field
+	return other
+}
+
 func decodePair(pb *internal.Pair) pilosa.Pair {
 	return pilosa.Pair{
 		ID:    pb.ID,
@@ -1186,10 +1437,33 @@ func decodePair(pb *internal.Pair) pilosa.Pair {
 	}
 }
 
+func decodePairField(pb *internal.Pair) pilosa.PairField {
+	return pilosa.PairField{
+		Pair: pilosa.Pair{
+			ID:    pb.ID,
+			Key:   pb.Key,
+			Count: pb.Count,
+		},
+		//Field: pb.Field, // TODO: in order to have this, we need PairField in QueryResponse.
+	}
+}
+
 func decodeValCount(pb *internal.ValCount) pilosa.ValCount {
 	return pilosa.ValCount{
-		Val:   pb.Val,
-		Count: pb.Count,
+		Val:        pb.Val,
+		FloatVal:   pb.FloatVal,
+		DecimalVal: decodeDecimalStruct(pb.DecimalVal),
+		Count:      pb.Count,
+	}
+}
+
+func decodeDecimalStruct(pb *internal.Decimal) *pql.Decimal {
+	if pb == nil {
+		return nil
+	}
+	return &pql.Decimal{
+		Value: pb.Value,
+		Scale: pb.Scale,
 	}
 }
 
@@ -1209,16 +1483,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
 	}
 }
 
+func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
+	ir := &internal.SignedRow{
+		Pos: encodeRow(r.Pos),
+		Neg: encodeRow(r.Neg),
+	}
+	return ir
+}
+
 func encodeRow(r *pilosa.Row) *internal.Row {
 	if r == nil {
 		return nil
 	}
 
-	return &internal.Row{
-		Columns: r.Columns(),
-		Keys:    r.Keys,
-		Attrs:   encodeAttrs(r.Attrs),
+	ir := &internal.Row{
+		Keys:  r.Keys,
+		Attrs: encodeAttrs(r.Attrs),
 	}
+	if true {
+		ir.Columns = r.Columns()
+	} else {
+		ir.Roaring = r.Roaring()
+	}
+	return ir
 }
 
 func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers {
@@ -1235,6 +1522,7 @@ func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount {
 		result[i] = &internal.GroupCount{
 			Group: encodeFieldRows(counts[i].Group),
 			Count: counts[i].Count,
+			Sum:   counts[i].Sum,
 		}
 	}
 	return result
@@ -1249,6 +1537,9 @@ func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow {
 				Field: fr.Field,
 				RowID: fr.RowID,
 			}
+			if fr.Value != nil {
+				other[i].Value = &internal.Int64{Value: *fr.Value}
+			}
 		} else {
 			other[i] = &internal.FieldRow{
 				Field:  fr.Field,
@@ -1267,6 +1558,17 @@ func encodePairs(a pilosa.Pairs) []*internal.Pair {
 	return other
 }
 
+func encodePairsField(a *pilosa.PairsField) *internal.PairsField {
+	other := &internal.PairsField{
+		Pairs: make([]*internal.Pair, len(a.Pairs)),
+	}
+	for i := range a.Pairs {
+		other.Pairs[i] = encodePair(a.Pairs[i])
+	}
+	other.Field = a.Field
+	return other
+}
+
 func encodePair(p pilosa.Pair) *internal.Pair {
 	return &internal.Pair{
 		ID:    p.ID,
@@ -1275,10 +1577,33 @@ func encodePair(p pilosa.Pair) *internal.Pair {
 	}
 }
 
+func encodePairField(p pilosa.PairField) *internal.Pair {
+	/*
+		// TODO: in order to have this, we need PairField in QueryResponse.
+		return &internal.Pair{
+			Pair:  encodePair(p.Pair),
+			Field: p.Field,
+		}
+	*/
+	return encodePair(p.Pair)
+}
+
 func encodeValCount(vc pilosa.ValCount) *internal.ValCount {
 	return &internal.ValCount{
-		Val:   vc.Val,
-		Count: vc.Count,
+		Val:        vc.Val,
+		FloatVal:   vc.FloatVal,
+		DecimalVal: encodeDecimal(vc.DecimalVal),
+		Count:      vc.Count,
+	}
+}
+
+func encodeDecimal(p *pql.Decimal) *internal.Decimal {
+	if p == nil {
+		return nil
+	}
+	return &internal.Decimal{
+		Value: p.Value,
+		Scale: p.Scale,
 	}
 }
 

executor_internal_test.go

@@ -15,9 +15,11 @@
 package pilosa
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -25,8 +27,13 @@ import (
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
+	holder := NewHolder(DefaultPartitionN)
+
+	cluster := NewTestCluster(1)
+
 	e := &executor{
-		Holder: NewHolder(),
+		Holder:  holder,
+		Cluster: cluster,
 	}
 	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
 	err := e.Holder.Open()
@@ -46,12 +53,12 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("creating fields %v, %v, %v", erra, errb, errc)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
+	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`)
 	if err != nil {
 		t.Fatalf("parsing query: %v", err)
 	}
 	c := query.Calls[0]
-	err = e.translateGroupByCall("i", idx, c)
+	err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 	if err != nil {
 		t.Fatalf("translating call: %v", err)
 	}
@@ -64,6 +71,19 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		}
 	}
 
+	if having, hok := c.Args["having"].(*pql.Call); !hok {
+		t.Fatal("expected having to be a call")
+	} else if cond, cok := having.Args["count"].(*pql.Condition); !cok {
+		t.Fatal("expected condition to be a count")
+	} else if cond.Op != pql.GT {
+		t.Fatal("expected condition op to be >")
+	} else {
+		val, ok := cond.Uint64Value()
+		if !ok || val != uint64(10) {
+			t.Fatal("expected condition val to be uint64(10)")
+		}
+	}
+
 	errTests := []struct {
 		pql string
 		err string
@@ -102,7 +122,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 				t.Fatalf("parsing query: %v", err)
 			}
 			c := query.Calls[0]
-			err = e.translateGroupByCall("i", idx, c)
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 			if err == nil {
 				t.Fatalf("expected error, but translated call is '%s", c)
 			}
@@ -222,3 +242,200 @@ func TestFieldRowMarshalJSON(t *testing.T) {
 		t.Fatalf("unexpected json: %s", b)
 	}
 }
+
+func TestExecutor_GroupCountCondition(t *testing.T) {
+	t.Run("satisfiesCondition", func(t *testing.T) {
+		type condCheck struct {
+			cond string
+			exp  bool
+		}
+		tests := []struct {
+			groupCount GroupCount
+			checks     []condCheck
+		}{
+			{
+				groupCount: GroupCount{Count: 100},
+				checks: []condCheck{
+					{cond: "count == 99", exp: false},
+					{cond: "count != 99", exp: true},
+					{cond: "count < 99", exp: false},
+					{cond: "count <= 99", exp: false},
+					{cond: "count > 99", exp: true},
+					{cond: "count >= 99", exp: true},
+
+					{cond: "count == 100", exp: true},
+					{cond: "count != 100", exp: false},
+					{cond: "count < 100", exp: false},
+					{cond: "count <= 100", exp: true},
+					{cond: "count > 100", exp: false},
+					{cond: "count >= 100", exp: true},
+
+					{cond: "count == 101", exp: false},
+					{cond: "count != 101", exp: true},
+					{cond: "count < 101", exp: true},
+					{cond: "count <= 101", exp: true},
+					{cond: "count > 101", exp: false},
+					{cond: "count >= 101", exp: false},
+
+					{cond: "98 < count < 100", exp: false},
+					{cond: "98 < count <= 100", exp: true},
+					{cond: "98 < count < 101", exp: true},
+					{cond: "100 <= count < 102", exp: true},
+					{cond: "100 < count < 102", exp: false},
+					{cond: "98 <= count <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: 100},
+				checks: []condCheck{
+					{cond: "sum == 99", exp: false},
+					{cond: "sum != 99", exp: true},
+					{cond: "sum < 99", exp: false},
+					{cond: "sum <= 99", exp: false},
+					{cond: "sum > 99", exp: true},
+					{cond: "sum >= 99", exp: true},
+
+					{cond: "sum == 100", exp: true},
+					{cond: "sum != 100", exp: false},
+					{cond: "sum < 100", exp: false},
+					{cond: "sum <= 100", exp: true},
+					{cond: "sum > 100", exp: false},
+					{cond: "sum >= 100", exp: true},
+
+					{cond: "sum == 101", exp: false},
+					{cond: "sum != 101", exp: true},
+					{cond: "sum < 101", exp: true},
+					{cond: "sum <= 101", exp: true},
+					{cond: "sum > 101", exp: false},
+					{cond: "sum >= 101", exp: false},
+
+					{cond: "98 < sum < 100", exp: false},
+					{cond: "98 < sum <= 100", exp: true},
+					{cond: "98 < sum < 101", exp: true},
+					{cond: "100 <= sum < 102", exp: true},
+					{cond: "100 < sum < 102", exp: false},
+					{cond: "98 <= sum <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: -100},
+				checks: []condCheck{
+					{cond: "sum == -99", exp: false},
+					{cond: "sum != -99", exp: true},
+					{cond: "sum < -99", exp: true},
+					{cond: "sum <= -99", exp: true},
+					{cond: "sum > -99", exp: false},
+					{cond: "sum >= -99", exp: false},
+
+					{cond: "sum == -100", exp: true},
+					{cond: "sum != -100", exp: false},
+					{cond: "sum < -100", exp: false},
+					{cond: "sum <= -100", exp: true},
+					{cond: "sum > -100", exp: false},
+					{cond: "sum >= -100", exp: true},
+
+					{cond: "sum == -101", exp: false},
+					{cond: "sum != -101", exp: true},
+					{cond: "sum < -101", exp: false},
+					{cond: "sum <= -101", exp: false},
+					{cond: "sum > -101", exp: true},
+					{cond: "sum >= -101", exp: true},
+
+					{cond: "-100 < sum < -98", exp: false},
+					{cond: "-100 <= sum < -98", exp: true},
+					{cond: "-101 < sum < -98", exp: true},
+					{cond: "-102 < sum <= -100", exp: true},
+					{cond: "-102 < sum < -100", exp: false},
+					{cond: "-102 <= sum <= -98", exp: true},
+				},
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
+				for j, check := range test.checks {
+					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
+
+						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
+						if err != nil {
+							t.Fatalf("parsing query: %v", err)
+						}
+						c := query.Calls[0]
+						having := c.Args["having"].(*pql.Call)
+
+						var got bool
+						for subj, cond := range having.Args {
+							switch subj {
+							case "count", "sum":
+								condition, ok := cond.(*pql.Condition)
+								if !ok {
+									t.Fatalf("not a valid condition")
+								}
+								got = test.groupCount.satisfiesCondition(subj, condition)
+							}
+						}
+
+						if got != check.exp {
+							t.Fatalf("expected: %v, but got: %v", check.exp, got)
+						}
+					})
+				}
+			})
+		}
+	})
+}
+
+func TestValCountComparisons(t *testing.T) {
+	tests := []struct {
+		name       string
+		vc         ValCount
+		other      ValCount
+		expLarger  ValCount
+		expSmaller ValCount
+	}{
+		{
+			name: "zero",
+		},
+		{
+			name:       "ints",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 3, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 1},
+			expSmaller: ValCount{Val: 3, Count: 2},
+		},
+		{
+			name:       "floats",
+			vc:         ValCount{FloatVal: 10.2, Count: 1},
+			other:      ValCount{FloatVal: 3.4, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
+			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
+		},
+		{
+			name:       "intsEquality",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 10, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 3},
+			expSmaller: ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:       "floatsEquality",
+			vc:         ValCount{FloatVal: 10.7, Count: 1},
+			other:      ValCount{FloatVal: 10.7, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
+			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			gotLarger := test.vc.larger(test.other)
+			if gotLarger != test.expLarger {
+				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
+			}
+
+			gotSmaller := test.vc.smaller(test.other)
+			if gotSmaller != test.expSmaller {
+				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
+			}
+		})
+	}
+}

extension.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+
+	"github.com/molecula/ext"
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
+func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
+	return wrappedBitmap{bm}
+}
+
+// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
+// an extension Bitmap.
+type wrappedBitmap struct{ *roaring.Bitmap }
+
+// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
+func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
+	if inner, ok := bm.(wrappedBitmap); ok {
+		if inner.Bitmap != nil {
+			return inner.Bitmap
+		}
+		return roaring.NewFileBitmap()
+	}
+	return roaring.NewFileBitmap()
+}
+
+func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
+	return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
+}
+
+func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
+	shifted, err := b.Bitmap.Shift(n)
+	return wrappedBitmap{shifted}, err
+}
+
+func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Flip(start, last)}
+}
+
+func (b wrappedBitmap) New() ext.Bitmap {
+	return WrapBitmap(roaring.NewFileBitmap())
+}
+
+// ContainerBits tries to get one container's worth of bits.
+func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
+	// it's an error to call this with a non-container-aligned offset
+	if offset&0xFFFF != 0 {
+		return nil
+	}
+	if b.Bitmap == nil {
+		fmt.Printf("ContainerBits on bitmap with no contents\n")
+		return nil
+	}
+	if b.Bitmap.Containers == nil {
+		fmt.Printf("ContainerBits on bitmap with nil Containers\n")
+		return nil
+	}
+	c := b.Bitmap.Containers.Get(offset >> 16)
+	if c == nil {
+		return nil
+	}
+	return c.AsBitmap(target)
+}

extensions/distinct.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build plugindistinct
+
+package extensions
+
+import (
+	_ "github.com/molecula/extensions/distinct"
+)

extensions/dummy.go

@@ -0,0 +1,18 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package contains only things which are conditional on build
+// tags.
+
+package extensions

field_internal_test.go

@@ -19,7 +19,10 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -89,6 +92,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, pql.GT, -8, -8, false},
 			{b1, pql.GT, 1005, 0, true},
 			{b1, pql.GT, 0, 0, false},
+			{b1, pql.GT, -300, -255, false},
 
 			{b2, pql.GT, 5, -95, false},
 			{b2, pql.GT, -8, -108, false},
@@ -141,6 +145,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 			{b1, 5, 20, 5, 20, false},
 			{b1, 20, 1005, 20, 255, false},
 			{b1, 1005, 2000, 0, 0, true},
+			{b1, 0, -1, 0, 0, true},
 
 			{b2, 5, 95, -95, -5, false},
 			{b2, 95, 120, -5, 20, false},
@@ -158,7 +163,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
 
 // Ensure field can open and retrieve a view.
 func TestField_DeleteView(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	viewName := viewStandard + "_v"
@@ -195,23 +200,23 @@ type TestField struct {
 }
 
 // NewTestField returns a new instance of TestField d/0.
-func NewTestField(opts FieldOption) *TestField {
+func NewTestField(t *testing.T, opts FieldOption) *TestField {
 	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
 	field, err := NewField(path, "i", "f", opts)
 	if err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
 	return &TestField{Field: field}
 }
 
-// MustOpenField returns a new, opened field at a temporary path. Panic on error.
-func MustOpenField(opts FieldOption) *TestField {
-	f := NewTestField(opts)
+// OpenField returns a new, opened field at a temporary path.
+func OpenField(t *testing.T, opts FieldOption) *TestField {
+	f := NewTestField(t, opts)
 	if err := f.Open(); err != nil {
-		panic(err)
+		t.Fatal(err)
 	}
 	return f
 }
@@ -258,7 +263,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
 
 // Ensure field can open and retrieve a view.
 func TestField_CreateViewIfNotExists(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	f := OpenField(t, OptFieldTypeDefault())
 	defer f.Close()
 
 	// Create view.
@@ -283,7 +288,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
 }
 
 func TestField_SetTimeQuantum(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	// Set & retrieve time quantum.
@@ -302,7 +307,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
 }
 
 func TestField_RowTime(t *testing.T) {
-	f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
+	f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
 	defer f.Close()
 
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
@@ -348,7 +353,8 @@ func TestField_RowTime(t *testing.T) {
 }
 
 func TestField_PersistAvailableShards(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -356,6 +362,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
@@ -366,10 +373,71 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
-	f := MustOpenField(OptFieldTypeDefault())
+	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
+	f := OpenField(t, OptFieldTypeDefault())
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap()
@@ -383,12 +451,14 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) {
 	if err := f.AddRemoteAvailableShards(bm); err != nil {
 		t.Fatal(err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Reload field and verify that shard data is persisted.
 	if err := f.Reopen(); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
-		t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+		t.Fatalf("unexpected available shards (reopen). expected: %v, \n but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+
 	}
 
 	bm1 := roaring.NewBitmap()
@@ -438,3 +508,314 @@ func TestBSIGroup_BaseDefaultValue(t *testing.T) {
 		}
 	}
 }
+
+func TestField_ApplyOptions(t *testing.T) {
+	for i, tt := range []struct {
+		opts    FieldOptions
+		expOpts FieldOptions
+	}{
+		{
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+		},
+	} {
+
+		fld := &Field{}
+		fld.options = *applyDefaultOptions(&FieldOptions{})
+
+		if err := fld.applyOptions(tt.opts); err != nil {
+			t.Fatal(err)
+		}
+
+		if fld.options.CacheType != tt.expOpts.CacheType {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheType value. expected: %s, but got: %s", i, tt.expOpts.CacheType, fld.options.CacheType)
+		} else if fld.options.CacheSize != tt.expOpts.CacheSize {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheSize value. expected: %d, but got: %d", i, tt.expOpts.CacheSize, fld.options.CacheSize)
+		}
+	}
+}
+
+// Ensure that importValue handles requiredDepth correctly.
+// This test sets the same column value to 1, then 8, then 1.
+// A previous bug was incorrectly determining bitDepth based
+// on the values in the import, and not taking existing values
+// into consideration. This would cause an import of 1/8/1
+// to result in a value of 9 instead of 1.
+func TestBSIGroup_importValue(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+
+		if row, err := f.Range(f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+	}
+}
+
+func TestIntField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []int64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []int64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []int64{10},
+			expMax:    ValCount{Val: 10, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []int64{10, 20},
+			expMax:    ValCount{Val: 20, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []int64{10, 20, 10, 10, 20},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11, 44, 1},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}
+
+// Ensure we get errors when they are expected.
+func TestDecimalField_MinMaxBoundaries(t *testing.T) {
+	for i, test := range []struct {
+		scale  int64
+		min    pql.Decimal
+		max    pql.Decimal
+		expErr bool
+	}{
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 0),
+			max:    pql.NewDecimal(math.MaxInt64, 0),
+			expErr: true,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(math.MinInt64, 3),
+			max:    pql.NewDecimal(math.MaxInt64, 3),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  3,
+			min:    pql.NewDecimal(-44, 0),
+			max:    pql.NewDecimal(88, 0),
+			expErr: false,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(1, 0),
+			max:    pql.NewDecimal(2, 0),
+			expErr: true,
+		},
+		{
+			scale:  19,
+			min:    pql.NewDecimal(math.MinInt64, 18),
+			max:    pql.NewDecimal(math.MaxInt64, 18),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, 20),
+			max:    pql.NewDecimal(2, 20),
+			expErr: true,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -1),
+			max:    pql.NewDecimal(2, -1),
+			expErr: false,
+		},
+		{
+			scale:  0,
+			min:    pql.NewDecimal(1, -19),
+			max:    pql.NewDecimal(2, -19),
+			expErr: true,
+		},
+	} {
+		t.Run("minmax"+strconv.Itoa(i), func(t *testing.T) {
+			_, err := NewField("no-path", "i", "f", OptFieldTypeDecimal(test.scale, test.min, test.max))
+			if err != nil && test.expErr {
+				if !strings.Contains(err.Error(), "is not supported") {
+					t.Fatal(err)
+				}
+			} else if err != nil && !test.expErr {
+				t.Fatalf("did not expect error, but got: %s", err)
+			} else if err == nil && test.expErr {
+				t.Fatal("expected error, but got none")
+			}
+		})
+	}
+}
+
+func TestDecimalField_MinMaxForShard(t *testing.T) {
+	f := OpenField(t, OptFieldTypeDecimal(3))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []float64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []float64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []float64{10.1},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []float64{10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+		{
+			name:      "another shard",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11, 44.39, 0.23},
+			expMax:    ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
+			expMin:    ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importFloatValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if !reflect.DeepEqual(maxvc, test.expMax) {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if !reflect.DeepEqual(minvc, test.expMin) {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}

field_test.go

@@ -158,6 +158,7 @@ func TestField_NameValidation(t *testing.T) {
 		"under_score",
 		"abc123",
 		"trailing_",
+		"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
 	}
 	invalidFieldNames := []string{
 		"",
@@ -168,7 +169,7 @@ func TestField_NameValidation(t *testing.T) {
 		"abc def",
 		"camelCase",
 		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
+		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
 	path, err := ioutil.TempDir("", "pilosa-field-")
@@ -227,3 +228,44 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(diff)
 	}
 }
+
+func TestField_ClearValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on field.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		if changed, err := f.ClearValue(100); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal(err)
+		}
+
+		// Read value.
+		if _, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if exists {
+			t.Fatal("expected value to not exist")
+		}
+	})
+}

fragment_internal_test.go

@@ -25,6 +25,8 @@ import (
 	"math/rand"
 	"os"
 	"reflect"
+	"runtime"
+	"runtime/debug"
 	"sort"
 	"sync/atomic"
 	"testing"
@@ -377,23 +379,29 @@ func TestFragment_Sum(t *testing.T) {
 	defer f.Clean(t)
 
 	// Set values.
-	if _, err := f.setValue(1000, bitDepth, 382); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.setValue(2000, bitDepth, 300); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.setValue(3000, bitDepth, 2818); err != nil {
-		t.Fatal(err)
-	} else if _, err := f.setValue(4000, bitDepth, 300); err != nil {
-		t.Fatal(err)
+	vals := []struct {
+		cid uint64
+		val int64
+	}{
+		{1000, 382},
+		{2000, 300},
+		{2500, -600},
+		{3000, 2818},
+		{4000, 300},
+	}
+	for _, v := range vals {
+		if _, err := f.setValue(v.cid, bitDepth, v.val); err != nil {
+			t.Fatal(err)
+		}
 	}
 
 	t.Run("NoFilter", func(t *testing.T) {
 		if sum, n, err := f.sum(nil, bitDepth); err != nil {
 			t.Fatal(err)
-		} else if n != 4 {
+		} else if n != 5 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != 3800 {
-			t.Fatalf("unexpected sum: %d", sum)
+		} else if got, exp := sum, int64(382+300-600+2818+300); got != exp {
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 
@@ -402,8 +410,8 @@ func TestFragment_Sum(t *testing.T) {
 			t.Fatal(err)
 		} else if n != 2 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != 600 {
-			t.Fatalf("unexpected sum: %d", sum)
+		} else if got, exp := sum, int64(300+300); got != exp {
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 
@@ -414,10 +422,10 @@ func TestFragment_Sum(t *testing.T) {
 	t.Run("ClearValue", func(t *testing.T) {
 		if sum, n, err := f.sum(nil, bitDepth); err != nil {
 			t.Fatal(err)
-		} else if n != 3 {
+		} else if n != 4 {
 			t.Fatalf("unexpected count: %d", n)
-		} else if sum != (3800 - 382) {
-			t.Fatalf("unexpected sum: got %d, expecting %d", sum, 3800-382)
+		} else if got, exp := sum, int64(3800-382-600); got != exp {
+			t.Fatalf("unexpected sum: got: %d, exp: %d", sum, exp)
 		}
 	})
 }
@@ -596,6 +604,21 @@ func TestFragment_Range(t *testing.T) {
 		}
 	})
 
+	t.Run("LTRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 1, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeOp(pql.LT, 1, 2); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{1}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
 	t.Run("GT", func(t *testing.T) {
 		f := mustOpenFragment("i", "f", viewStandard, 0, "")
 		defer f.Clean(t)
@@ -1382,6 +1405,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 
 	// Read into another fragment.
 	f1 := mustOpenFragment("i", "f", viewStandard, 0, "")
+	defer f1.Clean(t)
 	if rn, err := f1.ReadFrom(&buf); err != nil {
 		t.Fatal(err)
 	} else if wn != rn {
@@ -2054,7 +2078,11 @@ func BenchmarkImportRoaring(b *testing.B) {
 					b.StartTimer()
 					err := f.importRoaringT(data, false)
 					if err != nil {
-						f.awaitSnapshot()
+						// we don't actually particularly
+						// care whether this succeeds,
+						// but if it's happening we want
+						// it to be done.
+						_ = f.snapshotQueue.Await(f)
 						f.Clean(b)
 						b.Fatalf("import error: %v", err)
 					}
@@ -2093,7 +2121,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 							j := j
 							eg.Go(func() error {
 								err := frags[j].importRoaringT(data[j], false)
-								frags[j].awaitSnapshot()
+								// error unimportant if it happened, but we want
+								// any snapshots to have finished.
+								_ = frags[j].snapshotQueue.Await(frags[j])
 								return err
 							})
 						}
@@ -2131,11 +2161,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								// is excessive. force storage into snapshotted state, then use import
 								// to generate an op log and/or snapshot.
 								_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
-								frags[j].enqueueSnapshot()
-								frags[j].awaitSnapshot()
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
 								}
+								err = frags[j].snapshotQueue.Immediate(frags[j])
+								if err != nil {
+									b.Fatalf("snapshot after import: %v", err)
+								}
 							}
 							eg := errgroup.Group{}
 							b.StartTimer()
@@ -2143,7 +2175,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								j := j
 								eg.Go(func() error {
 									err := frags[j].importRoaringT(updata, false)
-									frags[j].awaitSnapshot()
+									err2 := frags[j].snapshotQueue.Await(frags[j])
+									if err == nil {
+										err = err2
+									}
 									return err
 								})
 							}
@@ -2205,20 +2240,38 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						// is excessive. force storage into snapshotted state, then use import
 						// to generate an op log and/or snapshot.
 						_, _, err := f.storage.ImportRoaringBits(data, false, false, 0)
-						f.enqueueSnapshot()
-						f.awaitSnapshot()
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Immediate(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StartTimer()
 						err = f.importRoaringT(updata, false)
-						f.awaitSnapshot()
 						if err != nil {
 							f.Clean(b)
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Await(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StopTimer()
-						stat, _ := f.file.Stat()
+						var stat os.FileInfo
+						var statTarget io.Writer
+						err = f.gen.Transaction(&statTarget, func() error {
+							targetFile, ok := statTarget.(*os.File)
+							if ok {
+								stat, _ = targetFile.Stat()
+							} else {
+								b.Errorf("couldn't stat file")
+							}
+							return nil
+						})
+						if err != nil {
+							b.Errorf("transaction error: %v", err)
+						}
 						fileSize[name] = stat.Size()
 						f.Clean(b)
 					}
@@ -2367,6 +2420,7 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
 			t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
 		}
 	}
+	f.Clean(t)
 }
 
 // getZipfRowsSliceRoaring generates a random fragment with the given number of
@@ -2512,16 +2566,28 @@ func (f *fragment) sanityCheck(t testing.TB) {
 }
 
 func (f *fragment) Clean(t testing.TB) {
-	f.awaitSnapshot()
+	f.mu.Lock()
+	err := f.snapshotQueue.Await(f)
+	f.mu.Unlock()
+	if err != nil {
+		t.Fatalf("snapshot failed before sanity check: %v", err)
+	}
 	f.sanityCheck(t)
+	if f.storage != nil && f.storage.Source != nil {
+		if f.storage.Source.Dead() {
+			t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+		}
+	}
 	errc := f.Close()
+	// prevent double-closes of generation during testing.
+	f.gen = nil
 	errf := os.Remove(f.path)
 	errp := os.Remove(f.cachePath())
 	if errc != nil || errf != nil {
 		t.Fatal("cleaning up fragment: ", errc, errf, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -2544,7 +2610,7 @@ func (f *fragment) CleanKeep(t testing.TB) {
 		t.Fatal("closing fragment: ", errc, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -2923,7 +2989,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(false)
 		for i := uint64(0); i < 4; i++ {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i {
 				t.Fatalf("expected row %d but got %d", i, id)
 			}
@@ -2934,7 +3000,7 @@ func TestFragmentRowIterator(t *testing.T) {
 				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
 			}
 		}
-		row, id, wrapped := iter.Next()
+		row, id, _, wrapped := iter.Next()
 		if row != nil {
 			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
 		}
@@ -2956,7 +3022,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(false)
 		for i := uint64(1); i < 8; i += 2 {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i {
 				t.Fatalf("expected row %d but got %d", i, id)
 			}
@@ -2967,7 +3033,7 @@ func TestFragmentRowIterator(t *testing.T) {
 				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
 			}
 		}
-		row, id, wrapped := iter.Next()
+		row, id, _, wrapped := iter.Next()
 		if row != nil {
 			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
 		}
@@ -2989,7 +3055,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(true)
 		for i := uint64(0); i < 5; i++ {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i%4 {
 				t.Fatalf("expected row %d but got %d", i%4, id)
 			}
@@ -3014,7 +3080,7 @@ func TestFragmentRowIterator(t *testing.T) {
 
 		iter := f.rowIterator(true)
 		for i := uint64(1); i < 10; i += 2 {
-			row, id, wrapped := iter.Next()
+			row, id, _, wrapped := iter.Next()
 			if id != i%8 {
 				t.Errorf("expected row %d but got %d", i%8, id)
 			}
@@ -3032,10 +3098,10 @@ func TestFragmentRowIterator(t *testing.T) {
 
 func TestUnionInPlaceMapped(t *testing.T) {
 	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	// note: clean has to be deferred first, because it has to run with
+	// the lock *not* held, because it is sometimes so it has to grab the
+	// lock...
 	defer f.Clean(t)
-	// I know this doesn't actually matter in our current context, but
-	// strictly speaking, we do say you have to hold the lock while calling
-	// unprotectedWriteToFragment...
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	r0 := rand.New(rand.NewSource(2))
@@ -3068,8 +3134,15 @@ func TestUnionInPlaceMapped(t *testing.T) {
 	f.storage.UnionInPlace(setBM1)
 	countUnion := f.storage.Count()
 	// UnionInPlace produces no ops log, we have to make it snapshot, to
-	// ensure that the on-disk representation is correct.
-	f.enqueueSnapshot()
+	// ensure that the on-disk representation is correct. Note, UIP is
+	// not used for things that are modifying real fragments, usually;
+	// it's used only in computation of things that usually don't go to
+	// disk, which is why we handle this specially in testing and not
+	// generically.
+	err = f.snapshotQueue.Immediate(f)
+	if err != nil {
+		t.Fatalf("snapshot after union-in-place: %v", err)
+	}
 
 	if count0 != countF {
 		t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF)
@@ -3178,6 +3251,25 @@ func TestFragmentPositionsForValue(t *testing.T) {
 	}
 }
 
+func TestIntLTRegression(t *testing.T) {
+	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	defer f.Clean(t)
+
+	_, err := f.setValue(1, 6, 33)
+	if err != nil {
+		t.Fatalf("setting value: %v", err)
+	}
+
+	row, err := f.rangeOp(pql.LT, 6, 33)
+	if err != nil {
+		t.Fatalf("doing range of: %v", err)
+	}
+
+	if !row.IsEmpty() {
+		t.Errorf("expected nothing, but got: %v", row.Columns())
+	}
+}
+
 func TestImportClearRestart(t *testing.T) {
 	tests := []struct {
 		rows []uint64
@@ -3257,7 +3349,7 @@ func TestImportClearRestart(t *testing.T) {
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
-				err = f.closeStorage(true)
+				err = f.closeStorage()
 				if err != nil {
 					t.Fatalf("closing storage: %v", err)
 				}
@@ -3291,7 +3383,7 @@ func TestImportClearRestart(t *testing.T) {
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
-				err = f2.closeStorage(true)
+				err = f2.closeStorage()
 				if err != nil {
 					t.Fatalf("f2 closing storage: %v", err)
 				}
@@ -3339,6 +3431,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) {
 
 func TestImportValueConcurrent(t *testing.T) {
 	f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+	defer f.Clean(t)
 	eg := &errgroup.Group{}
 	for i := 0; i < 4; i++ {
 		i := i
@@ -3363,7 +3456,7 @@ func TestImportMultipleValues(t *testing.T) {
 		cols      []uint64
 		vals      []int64
 		checkCols []uint64
-		checkVals []uint64
+		checkVals []int64
 		depth     uint
 	}{
 		{
@@ -3371,7 +3464,7 @@ func TestImportMultipleValues(t *testing.T) {
 			vals:      []int64{97, 100},
 			depth:     7,
 			checkCols: []uint64{0},
-			checkVals: []uint64{100},
+			checkVals: []int64{100},
 		},
 	}
 
@@ -3395,7 +3488,7 @@ func TestImportMultipleValues(t *testing.T) {
 					if !exists {
 						t.Errorf("column %d should exist", cc)
 					}
-					if n != 100 {
+					if n != cv {
 						t.Errorf("wrong value: %d is not %d", n, cv)
 					}
 				}
@@ -3405,6 +3498,66 @@ func TestImportMultipleValues(t *testing.T) {
 	}
 }
 
+func TestImportValueRowCache(t *testing.T) {
+	type testCase struct {
+		cols      []uint64
+		vals      []int64
+		checkCols []uint64
+		depth     uint
+	}
+	tests := []struct {
+		tc1 testCase
+		tc2 testCase
+	}{
+		{
+			tc1: testCase{
+				cols:      []uint64{2},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2},
+			},
+			tc2: testCase{
+				cols:      []uint64{1000},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2, 1000},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		for _, maxOpN := range []int{1, 10000} {
+			t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) {
+				f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+				f.MaxOpN = maxOpN
+				defer f.Clean(t)
+
+				// First import (tc1)
+				if err := f.importValue(test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc1.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns())
+				}
+
+				// Second import (tc2)
+				if err := f.importValue(test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc2.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns())
+				}
+			})
+		}
+	}
+}
+
 func TestFragmentConcurrentReadWrite(t *testing.T) {
 	f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
 	defer f.Clean(t)
@@ -3431,3 +3584,115 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 
 	t.Logf("%d", acc)
 }
+
+func TestRemapCache(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+	// request a panic that doesn't kill the program on fault
+	wouldFault := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(wouldFault)
+		if r := recover(); r != nil {
+			if err, ok := r.(error); ok {
+				// special case: if we caught a page fault, we diagnose that directly. sadly,
+				// we can't see the actual values that were used to generate this, probably.
+				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+					t.Fatalf("segfault trapped during remap test (expected failure mode)")
+				}
+			}
+			t.Fatalf("unexpected panic: %v", r)
+		}
+	}()
+	// create a container
+	_, err := f.storage.Add(65537)
+	if err != nil {
+		t.Fatalf("storage add: %v", err)
+	}
+	// cause the container to be mapped
+	err = f.Snapshot()
+	if err != nil {
+		t.Fatalf("storage snapshot: %v", err)
+	}
+	// freeze the row
+	_ = f.row(0)
+	// add a bit that isn't in that container, so that container doesn't
+	// change
+	_, err = f.storage.Add(2)
+	if err != nil {
+		t.Fatalf("storage add: %v", err)
+	}
+	// make the original container be the most recent, thus cached, container
+	_, err = f.bit(0, 65537)
+	if err != nil {
+		t.Fatalf("storage bit check: %v", err)
+	}
+	// force snapshot, remapping the containers
+	err = f.Snapshot()
+	if err != nil {
+		t.Fatalf("storage snapshot: %v", err)
+	}
+	// get rid of the old mapping
+	runtime.GC()
+	// try to read that container again
+	_, err = f.bit(0, 65537)
+	if err != nil {
+		t.Fatalf("storage bit check: %v", err)
+	}
+}
+
+func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+
+	// byShardWidth is a map of the same roaring (fragment) data generated
+	// with different shard widths.
+	// TODO: a better approach may be to generate this in the test based
+	// on shard width.
+	byShardWidth := make(map[uint64][]byte)
+	// row/col: 1/1
+	byShardWidth[1<<20] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
+	byShardWidth[1<<22] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
+
+	var b []byte
+	if data, ok := byShardWidth[ShardWidth]; ok {
+		b = data
+	}
+
+	defer f.Clean(t)
+	err := f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check the bit
+	res := f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("expecting 1 got: %v", res)
+	}
+	//clear the bit
+	changed, _ := f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+	// import again
+	err = f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check
+	res = f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("again expecting 1 got: %v", res)
+	}
+	changed, _ = f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("again expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+}

gendebug_test.go

@@ -0,0 +1,39 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func examineResults() {
+	results := reportGenerations()
+	if len(results) > 0 {
+		fmt.Printf("generations:\n")
+		for _, res := range results {
+			fmt.Printf("  %s\n", res)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	ret := m.Run()
+	examineResults()
+	os.Exit(ret)
+}

generation.go

@@ -0,0 +1,439 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pkg/errors"
+)
+
+// generation represents one "generation" of opening a data file.
+// This is what determines when it's safe to unmap a data file, if it
+// got mapped, and handles closing/reopening files if we need to
+// manage file handle availability. It's an interface because this
+// lets us write simpler code for specific cases, rather than handling
+// the whole matrix of mapped/unmapped, staying open/being reopened,
+// etcetera.
+//
+// You create a generation by calling newGeneration with a file
+// path. If it succeeds in opening that path, it calls a provided
+// setup function with the data from the generation, and a flag
+// indicating whether the data is mmapped. If the setup function
+// fails, newGeneration cleans things up and closes. Otherwise,
+// it returns a generation.
+//
+// The generation itself uses runtime.SetFinalizer to clean up when
+// the last reference to it goes away. You should store a pointer
+// to the generation in any object which is reliant on the generation.
+//
+// When you anticipate a generation should be done (for instance,
+// opening a new generation), the old one gets marked done, which
+// stashes a timestamp in it. Later operations can check whether
+// the timestamp is a while back, and if so, complain that something
+// might be wrong.
+//
+// In some cases, we don't have enough open file limit to keep every
+// file actually open. To address this, use the `Transaction` function,
+// which ensures that the file is open, stores a reference to it in
+// a provided `*io.Writer`, and then restores the previous value of
+// the io.Writer when it's done. For instance, for a bitmap, this might
+// be used with `&b.OpWriter`.
+//
+// newGeneration takes an optional previous generation; it calls
+// that generation's Done function after running the provided setup,
+// and bumps the generation count.
+type generation interface {
+	// Transaction runs the given transaction with the generation's
+	// file open. If the **os.File parameter is
+	// non-nil, the generation's file will be open, and stored
+	// into that pointer, during the execution of func, after
+	// which the previous contents are restored. Otherwise
+	// the file may or may not be open during the operation.
+	Transaction(*io.Writer, func() error) error
+	// Done() should be called exactly once, to indicate that a
+	// generation is expected not to be in use for long -- for instance,
+	// when a new generation replaces it.
+	Done()
+	// Generation count.
+	Generation() int64
+	// ID indicates the source -- path and generation number -- that
+	// this generation represents.
+	ID() string
+	// Dead indicates whether this generation is Done.
+	Dead() bool
+	// Bytes reports the storage associated with this generation, if any.
+	// DO NOT USE THIS. Except if you're debugging mmap segfaults.
+	Bytes() []byte
+}
+
+type mmapGeneration struct {
+	mu         sync.Mutex // mutex guards modifiers of generation, not of data
+	transMu    sync.Mutex // guards transactions, specifically
+	path       string
+	id         string
+	file       *os.File
+	data       []byte
+	generation int64     // generation counter
+	dead       bool      // we think this generation is dead
+	deadSince  time.Time // when this generation was marked dead
+	retries    int       // for cases where we're retrying
+	logger     logger.Logger
+}
+
+func (m *mmapGeneration) Dead() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.dead
+}
+
+func (m *mmapGeneration) ID() string {
+	return m.id
+}
+
+func (m *mmapGeneration) Generation() int64 {
+	return m.generation
+}
+
+// Transaction runs an exclusive call, ensuring that the file is open if
+// the *io.Writer parameter is present.
+func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
+	m.transMu.Lock()
+	defer m.transMu.Unlock()
+	// HEY LOOK CAREFULLY AT THIS BIT:
+	// We can't just defer this unlock. We specifically want to be
+	// sure to unlock the regular mutex *before* this function is over,
+	// and if we error out trying to open the file, we want to do it
+	// even sooner. If we deferred this, the transaction would block
+	// *everything*, including things like sanity checks against the
+	// generation being Dead(), but also including the deferred
+	// re-close-the-file.
+	m.mu.Lock()
+	// if we've been asked for a file pointer, we need to ensure that
+	// our file is open, and that the file pointer to it is stored in
+	// the requested location, then revert that when we're done.
+	// if we aren't asked for a file pointer, nothing needs the file
+	// open.
+	if m.dead {
+		elapsed := time.Since(m.deadSince)
+		m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
+	}
+	if fileP != nil {
+		if m.file == nil {
+			// we ignore the shouldClose response here; if this
+			// fragment was previously not being kept open, we're
+			// going to stick with that.
+			_, err := m.openFile()
+			if err != nil {
+				m.mu.Unlock()
+				return err
+			}
+			defer func() {
+				// report a close error if we have no other error to report
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				err := m.closeFile()
+				if transactionErr == nil {
+					transactionErr = err
+				}
+			}()
+		}
+		var fileStash io.Writer
+		fileStash, *fileP = *fileP, m.file
+		defer func() {
+			*fileP = fileStash
+		}()
+	}
+	// We are done locking the generation itself for now.
+	m.mu.Unlock()
+	wouldPanic := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(wouldPanic)
+		if r := recover(); r != nil {
+			if err, ok := r.(error); ok {
+				// special case: if we caught a page fault, we diagnose that directly. sadly,
+				// we can't see the actual values that were used to generate this, probably.
+				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+					if transactionErr == nil {
+						transactionErr = errors.New("invalid memory access during transaction")
+					} else {
+						transactionErr = fmt.Errorf("invalid memory access during transaction, previous error %v", transactionErr)
+					}
+					return
+				}
+			}
+			if transactionErr == nil {
+				transactionErr = fmt.Errorf("panic during transaction: %v", r)
+			} else {
+				transactionErr = fmt.Errorf("panic during erroring transaction: panic %v, previous error %v", r, transactionErr)
+			}
+		}
+	}()
+	return fn()
+}
+
+func (m *mmapGeneration) Bytes() []byte {
+	return m.data
+}
+
+// Done marks the generation done, and closes its file, but may not unmap it.
+// It's still conceptually possible to end up doing a Transaction against a
+// done generation, but it's a red flag.
+func (m *mmapGeneration) Done() {
+	if m == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.dead {
+		oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
+			m.id, time.Now(), m.deadSince)
+		panic(oops)
+	}
+	m.dead = true
+	m.deadSince = time.Now()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error closing generation %s: %v", m.id, err)
+	}
+	// If we're not debugging, the finalizer won't have been enabled
+	// previously. Finalizers have non-zero cost, so having them not be
+	// created until they're needed seems rewarding?
+	if !generationDebug {
+		runtime.SetFinalizer(m, generationFinalizer)
+	}
+	endGeneration(m.id)
+	// note, Done() doesn't close the file; only the finalizer actually
+	// does the shutdown.
+}
+
+// Try to close the file if it's currently open.
+func (m *mmapGeneration) closeFile() error {
+	var lastErr error
+	// report the most serious error encountered, but still close
+	// file even if something else failed.
+	if m.file != nil {
+		if err := m.file.Sync(); err != nil {
+			lastErr = fmt.Errorf("sync: %s", err)
+		}
+		if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
+			lastErr = fmt.Errorf("unlock: %s", err)
+		}
+		if err := syswrap.CloseFile(m.file); err != nil {
+			lastErr = fmt.Errorf("close file: %s", err)
+		}
+		m.file = nil
+	}
+	return lastErr
+}
+
+// openFile ensures the file is open and locked, or fails. If it does
+// open the file, it will also report the "you need to close this file
+// when you're done" flag from syswrap.
+func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
+	if m.file != nil {
+		return false, nil
+	}
+	m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return false, err
+	}
+	// do we actually want this in every openFile? I don't know.
+	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		m.file.Close()
+		m.file = nil
+		return false, fmt.Errorf("flock: %s", err)
+	}
+	return shouldClose, nil
+}
+
+func generationFinalizer(m *mmapGeneration) {
+	m.mu.Lock()
+	if !m.dead {
+		m.logger.Printf("finalizing generation %s which isn't dead yet\n",
+			m.id)
+	}
+	m.mu.Unlock()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("finalizing generation, closing file: %v\n", err)
+	}
+	if m.data != nil {
+		err := syswrap.Munmap(m.data)
+		if err != nil {
+			m.logger.Printf("finalizing generation, munmap: %v\n", err)
+		}
+		m.data = nil
+	}
+	finalizeGeneration(m.id)
+}
+
+// Cancel closes a generation out entirely. It cancels any finalizer,
+// unmaps any data, ends generation tracking, and closes any files.
+// It does each of these separately whether or not the others need to be done,
+// or succeed. It's used to handle failures from newGeneration; it makes sure
+// the generation isn't holding any resources and doesn't need to be cleaned
+// up otherwise.
+//
+// Mostly a helper function because there's several cases where newGeneration
+// might fail.
+func (m *mmapGeneration) Cancel() {
+	if m.data != nil {
+		_ = syswrap.Munmap(m.data)
+		m.data = nil
+	}
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error cancelling generation %s: %v", m.id, err)
+	}
+	runtime.SetFinalizer(m, nil)
+	m.dead = true
+	m.deadSince = time.Now()
+	cancelGeneration(m.id)
+}
+
+// newGeneration creates a new generation using the given file path. It
+// then calls the provided setup function with the allocated storage, a
+// file handle, the new generation, and a flag indicatting whether the storage
+// is memory-mapped. If the setup function returns a non-nil error, the
+// generation is cleaned up, and newGeneration fails. The setup function
+// also returns a boolean indicating whether it used the mapping; if it
+// didn't, newGeneration discards the mapping and returns a nil generation.
+//
+// If generationDebug is enabled, we track the generation even if no mapping
+// is actually in use, so we can verify that the tracking is working.
+//
+// On failure, newGeneration returns nil values for generation and func,
+// and an error. On success, the func returned is the close func to use
+// when the generation is no longer needed by the caller.
+func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
+	m := mmapGeneration{path: path, logger: logger}
+	if existing != nil {
+		m.generation = existing.Generation() + 1
+		// we might keep a previous generation around just for its generation count.
+		if !existing.Dead() {
+			defer existing.Done()
+		}
+	}
+	shouldClose, err := m.openFile()
+	if err != nil {
+		return nil, err
+	}
+	m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
+	// possibly assign new generation ID if this one's been used, which can
+	// happen with reopens, especially during testing.
+	m.id = registerGeneration(m.id)
+	// if debugging, we always want the finalizer on so we notice if a
+	// generation is finalized without being closed. for non-debugging
+	// use, we only need it when the generation is closed.
+	if generationDebug {
+		runtime.SetFinalizer(&m, generationFinalizer)
+	}
+	// Mmap the underlying file so it can be zero copied.
+	var mapped bool
+	var data []byte
+	fi, err := m.file.Stat()
+	if err == nil && fi.Size() > 0 {
+		data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+		if err == syswrap.ErrMaxMapCountReached {
+			// I have no idea where/how to display this message.
+			m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
+		} else if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "mmap failed")
+		} else {
+			mapped = true
+		}
+	}
+	if data == nil && readData {
+		data, err = ioutil.ReadAll(m.file)
+		if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "failure file readall")
+		}
+	}
+	// if we got here, data's the expected data, so let's try to use it
+	mappedAny, err := setup(data, m.file, &m, mapped)
+
+	// if the setup failed, we unmap data if we previously mapped it,
+	// and exit. Note that having no data, or having only trivial
+	// data (like a zero-container Roaring file) isn't "failed".
+	if err != nil {
+		m.Cancel()
+		// Unless, that is, we think the file probably ought to
+		// be truncated: For instance, if a bitmap has a corrupted
+		// ops log, we could truncate that part of it and retry.
+		if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
+			m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
+			// close this generation, then try again. once.
+			m.retries++
+			err := os.Truncate(m.path, err.SuggestedLength())
+			if err != nil {
+				m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
+			}
+			return newGeneration(&m, path, readData, setup, logger)
+		}
+		return nil, err
+	}
+
+	if mapped {
+		// when generationDebug is on, we want to track this even
+		// if it's not being used.
+		if generationDebug || mappedAny {
+			// Advise the kernel that the mmap is accessed randomly.
+			// We don't care much about errors with this.
+			_ = madvise(data, syscall.MADV_RANDOM)
+			// store the data, so we can unmap it when this generation
+			// gets finalized.
+			m.data = data
+		} else {
+			// unmap the data and don't stash the pointer in this
+			// generation. It's not being used. This generation
+			// doesn't need to exist, yay.
+			unmapErr := syswrap.Munmap(data)
+			if unmapErr != nil {
+				m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
+			}
+		}
+	}
+	// shouldClose comes from underlying syswrap.OpenFile, which checks
+	// a count of open files to hint at us when we need to start closing
+	// files to preserve open file descriptor limit.
+	if shouldClose {
+		err := m.closeFile()
+		if err != nil {
+			m.logger.Printf("closing file to preserve open files failed: %v\n", err)
+		}
+	}
+	// It's possible that the generation has no actual data to track,
+	// because nothing's mapped, in which case there won't be any bitmap
+	// sources following this, just the fragment source. (Bitmaps won't
+	// be attached to the source unless they're actually mapped to it,
+	// or generationDebug is true). That's okay. We pay a tiny cost
+	// for the finalizer, but we also get higher confidence that it really
+	// does get cleaned up.
+	return &m, nil
+}

generation_debug.go

@@ -0,0 +1,160 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+const generationDebug = true
+
+type lifespan struct {
+	from, to, finalized time.Time
+}
+
+var knownGenerations map[string]lifespan
+var knownGenerationLock sync.Mutex
+
+var timeZero time.Time
+
+func registerGeneration(id string) string {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	if knownGenerations == nil {
+		knownGenerations = make(map[string]lifespan)
+	}
+	newSpan := lifespan{from: time.Now()}
+	origId := id
+
+	// if you have more than 65k of the same file open, maybe you have bigger
+	// problems than this.
+	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
+		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
+		if span.finalized != timeZero {
+			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
+				id, suffix, span.from, span.to, span.finalized)
+		} else {
+			if span.to != timeZero {
+				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
+			} else {
+				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
+			}
+		}
+		id = origId + suffix
+	}
+	fmt.Printf("new generation %s\n", id)
+	knownGenerations[id] = newSpan
+	return id
+}
+
+func endGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("ending generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero || span.to != timeZero {
+		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
+		panic(oops)
+	}
+	span.to = time.Now()
+	knownGenerations[id] = span
+}
+
+// cancelGeneration marks the generation as finalized. In principle it's
+// only used in cases where we just started a generation but something
+// went wrong. it's not fancier than this because of the weird cases
+// where the same generation shows up again, such as when closing and
+// reopening an index so we don't know about previous instances of the
+// same files.
+func cancelGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if exists {
+		span.finalized = time.Now()
+		span.to = span.finalized
+		knownGenerations[id] = span
+	}
+}
+
+func finalizeGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("finalizing generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero {
+		var oops string
+		if span.to != timeZero {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
+		} else {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
+		}
+		panic(oops)
+	}
+	span.finalized = time.Now()
+	knownGenerations[id] = span
+}
+
+func reportGenerations() []string {
+	runtime.GC()
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	var surviving []string
+	times := make([]int64, 0, len(knownGenerations))
+	for id, span := range knownGenerations {
+		if span.to == timeZero {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
+			} else {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
+			}
+		} else {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
+			} else {
+				times = append(times, int64(span.finalized.Sub(span.to)))
+			}
+		}
+	}
+	if len(times) > 0 {
+		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
+		var total int64
+		for _, d := range times {
+			total += d
+		}
+		var mean, median, p90, p99, worst int64
+		mean = total / int64(len(times))
+		median = times[len(times)/2]
+		p90 = times[(len(times)*9)/10]
+		p99 = times[(len(times)*99)/100]
+		worst = times[len(times)-1]
+		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+	}
+	return surviving
+}

generation_nodebug.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package pilosa
+
+const generationDebug = false
+
+func registerGeneration(id string) string {
+	return id
+}
+
+func endGeneration(id string) {
+}
+
+func cancelGeneration(id string) {
+}
+
+func finalizeGeneration(id string) {
+}
+
+//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
+func reportGenerations() []string { //nolint:unused,deadcode
+	return nil
+}

generation_test.go

@@ -0,0 +1,67 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationparanoia
+
+package pilosa
+
+import (
+	"runtime"
+	"testing"
+	"unsafe"
+)
+
+func TestGenerationPanic(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "none")
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	_ = f.Snapshot()
+	_ = f.row(0)
+	var prevData []byte
+
+	if f.gen.(*mmapGeneration).data == nil {
+		t.Fatalf("generation code didn't create a mapping, apparently?")
+	}
+	prevData = f.gen.(*mmapGeneration).data
+	f.mu.Lock()
+	_ = f.snapshotQueue.Immediate(f)
+	f.mu.Unlock()
+	runtime.GC()
+	for i := 0; i < (f.MaxOpN / 2); i++ {
+		_, _ = f.setBit(0, uint64(i*32)+23)
+	}
+	f.mu.Lock()
+	f.snapshotQueue.Await(f)
+	f.mu.Unlock()
+	runtime.GC()
+	newData := f.gen.(*mmapGeneration).data
+	if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
+		t.Fatalf("test can't run usefully, didn't get new data pointer")
+	}
+
+	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+		prevData[0] = 0x3c
+		return nil
+	})
+	if err == nil {
+		t.Fatalf("expected a panic to get caught, but nothing happened")
+	}
+	if err.Error() != "invalid memory access during transaction" {
+		t.Fatalf("expected \"invalid memory access during transaction\", got %q", err.Error())
+	}
+}

go.mod

@@ -3,7 +3,6 @@ module github.com/pilosa/pilosa/v2
 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -13,17 +12,19 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
-	github.com/golang/protobuf v1.3.1
+	github.com/golang/protobuf v1.3.3
 	github.com/google/go-cmp v0.2.0
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
+	github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.18.12+incompatible
@@ -40,8 +41,9 @@ require (
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
 	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/grpc v1.28.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 )
 
-go 1.11
+go 1.13

go.sum

@@ -1,3 +1,4 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
@@ -18,8 +19,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -28,6 +32,9 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -39,10 +46,17 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -80,6 +94,14 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 h1:XOImsA5XhGklFj8Y0TxSm1qWZzEwYxom2JOXiu9GMq0=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@@ -101,6 +123,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -153,18 +177,29 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -180,12 +215,29 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=

handler.go

@@ -16,6 +16,9 @@ package pilosa
 
 import (
 	"encoding/json"
+
+	"github.com/pilosa/pilosa/v2/tracing"
+	"github.com/pkg/errors"
 )
 
 // QueryRequest represent a request to process a query.
@@ -42,12 +45,19 @@ type QueryRequest struct {
 	// If true, indicates that query is part of a larger distributed query.
 	// If false, this request is on the originating node.
 	Remote bool
+
+	// Should we profile this query?
+	Profile bool
+
+	// Additional data associated with the query, in cases where there's
+	// row-style inputs for precomputed values.
+	EmbeddedData []*Row
 }
 
 // QueryResponse represent a response from a processed query.
 type QueryResponse struct {
 	// Result for each top-level query call.
-	// Can be a Bitmap, Pairs, or uint64.
+	// Can be a Bitmap, Pairs, or uint64. // TODO: this comment is out of date.
 	Results []interface{}
 
 	// Set of column attribute objects matching IDs returned in Result.
@@ -55,6 +65,9 @@ type QueryResponse struct {
 
 	// Error during parsing or execution.
 	Err error
+
+	// Profiling data, if any
+	Profile *tracing.Profile
 }
 
 // MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
@@ -68,9 +81,11 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(struct {
 		Results        []interface{}    `json:"results"`
 		ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
+		Profile        *tracing.Profile `json:"profile,omitempty"`
 	}{
 		Results:        resp.Results,
 		ColumnAttrSets: resp.ColumnAttrSets,
+		Profile:        resp.Profile,
 	})
 }
 
@@ -97,12 +112,63 @@ var NopHandler Handler = nopHandler{}
 // ImportValueRequest describes the import request structure
 // for a value (BSI) import.
 type ImportValueRequest struct {
-	Index      string
-	Field      string
-	Shard      uint64
-	ColumnIDs  []uint64
-	ColumnKeys []string
-	Values     []int64
+	Index string
+	Field string
+	// if Shard is MaxUint64 (an impossible shard value), this
+	// indicates that the column IDs may come from multiple shards.
+	Shard        uint64
+	ColumnIDs    []uint64
+	ColumnKeys   []string
+	Values       []int64
+	FloatValues  []float64
+	StringValues []string
+}
+
+func (ivr *ImportValueRequest) Len() int           { return len(ivr.ColumnIDs) }
+func (ivr *ImportValueRequest) Less(i, j int) bool { return ivr.ColumnIDs[i] < ivr.ColumnIDs[j] }
+func (ivr *ImportValueRequest) Swap(i, j int) {
+	ivr.ColumnIDs[i], ivr.ColumnIDs[j] = ivr.ColumnIDs[j], ivr.ColumnIDs[i]
+	if len(ivr.Values) > 0 {
+		ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
+	} else if len(ivr.FloatValues) > 0 {
+		ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
+	} else if len(ivr.StringValues) > 0 {
+		ivr.StringValues[i], ivr.StringValues[j] = ivr.StringValues[j], ivr.StringValues[i]
+	}
+}
+
+// Validate ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) Validate() error {
+	if ivr.Index == "" || ivr.Field == "" {
+		return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
+	}
+	if len(ivr.ColumnIDs) != 0 && len(ivr.ColumnKeys) != 0 {
+		return errors.Errorf("must pass either column ids or keys, but not both")
+	}
+	var valueSetCount int
+	if len(ivr.Values) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.FloatValues) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.StringValues) != 0 {
+		valueSetCount++
+	}
+	if valueSetCount > 1 {
+		return errors.Errorf("must pass ints, floats, or strings but not multiple")
+	}
+	return nil
+}
+
+// ImportColumnAttrsRequest describes the import request structure
+// for a ColumnAttr import
+type ImportColumnAttrsRequest struct {
+	AttrKey   string
+	ColumnIDs []uint64
+	AttrVals  []string
+	Shard     int64
+	Index     string
 }
 
 // ImportRequest describes the import request structure
@@ -118,11 +184,19 @@ type ImportRequest struct {
 	Timestamps []int64
 }
 
+const (
+	RequestActionSet       = "set"
+	RequestActionClear     = "clear"
+	RequestActionOverwrite = "overwrite"
+)
+
 // ImportRoaringRequest describes the import request structure
 // for an import containing roaring-encoded data.
 type ImportRoaringRequest struct {
-	Clear bool
-	Views map[string][]byte
+	Clear  bool
+	Action string // [set, clear, overwrite]
+	Block  int
+	Views  map[string][]byte
 }
 
 // ImportResponse is the structured response of an import.
@@ -160,3 +234,17 @@ type TranslateKeysRequest struct {
 type TranslateKeysResponse struct {
 	IDs []uint64
 }
+
+// TranslateIDsRequest describes the structure of a request
+// for a batch of id translations.
+type TranslateIDsRequest struct {
+	Index string
+	Field string
+	IDs   []uint64
+}
+
+// TranslateIDsResponse is the structured response of a id
+// translation request.
+type TranslateIDsResponse struct {
+	Keys []string
+}

holder.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -50,6 +51,9 @@ const (
 type Holder struct {
 	mu sync.RWMutex
 
+	// Partition count used by translation.
+	partitionN int
+
 	// Indexes by name.
 	indexes map[string]*Index
 
@@ -75,15 +79,45 @@ type Holder struct {
 
 	Logger logger.Logger
 
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
-	// Manages replication from the primary node.
-	primaryTranslateNode     *Node
-	translateStoreReplicator *holderTranslateStoreReplicator
+	// Instantiates new translation stores
+	OpenTranslateStore  OpenTranslateStoreFunc
+	OpenTranslateReader OpenTranslateReaderFunc
 
-	// Instantiates new translation stores for indexes & fields.
-	OpenTranslateStore  OpenTranslateStoreFunc  // local store
-	OpenTranslateReader OpenTranslateReaderFunc // replication
+	// Func to open whatever implementation of transaction store we're using.
+	OpenTransactionStore OpenTransactionStoreFunc
+
+	// transactionManager
+	transactionManager *TransactionManager
+
+	translationSyncer translationSyncer
+
+	// Queue of fields (having a foreign index) which have
+	// opened before their foreign index has opened.
+	foreignIndexFields []*Field
+
+	// opening is set to true while Holder is opening.
+	// It's used to determine if foreign index application
+	// needs to be queued and completed after all indexes
+	// have opened.
+	opening bool
+}
+
+func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
+	return h.transactionManager.Start(ctx, id, timeout, exclusive)
+}
+
+func (h *Holder) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return h.transactionManager.Finish(ctx, id)
+}
+
+func (h *Holder) Transactions(ctx context.Context) (map[string]*Transaction, error) {
+	return h.transactionManager.List(ctx)
+}
+
+func (h *Holder) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
+	return h.transactionManager.Get(ctx, id)
 }
 
 // lockedChan looks a little ridiculous admittedly, but exists for good reason.
@@ -113,10 +147,11 @@ func (lc *lockedChan) Recv() {
 }
 
 // NewHolder returns a new instance of Holder.
-func NewHolder() *Holder {
+func NewHolder(partitionN int) *Holder {
 	return &Holder{
-		indexes: make(map[string]*Index),
-		closing: make(chan struct{}),
+		partitionN: partitionN,
+		indexes:    make(map[string]*Index),
+		closing:    make(chan struct{}),
 
 		opened: lockedChan{ch: make(chan struct{})},
 
@@ -127,14 +162,21 @@ func NewHolder() *Holder {
 
 		cacheFlushInterval: defaultCacheFlushInterval,
 
-		Logger: logger.NopLogger,
-
 		OpenTranslateStore: OpenInMemTranslateStore,
+
+		OpenTransactionStore: OpenInMemTransactionStore,
+
+		translationSyncer: NopTranslationSyncer,
+
+		Logger: logger.NopLogger,
 	}
 }
 
 // Open initializes the root data directory for the holder.
 func (h *Holder) Open() error {
+	h.opening = true
+	defer func() { h.opening = false }()
+
 	// Reset closing in case Holder is being reopened.
 	h.closing = make(chan struct{})
 
@@ -152,6 +194,13 @@ func (h *Holder) Open() error {
 		return ErrCannotOpenV1TranslateFile
 	}
 
+	tstore, err := h.OpenTransactionStore(h.Path)
+	if err != nil {
+		return errors.Wrap(err, "opening transaction store")
+	}
+	h.transactionManager = NewTransactionManager(tstore)
+	h.transactionManager.Log = h.Logger
+
 	// Open path to read all index directories.
 	f, err := os.Open(h.Path)
 	if err != nil {
@@ -167,7 +216,7 @@ func (h *Holder) Open() error {
 	// Run snapshots asynchronously. The snapshotQueue will have a background
 	// task associated with it which flushes it and waits until this channel
 	// is closed, so we should always close this channel when done.
-	h.snapshotQueue = newSnapshotQueue(100, 2, h.Logger)
+	h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger)
 
 	for _, fi := range fis {
 		// Skip files or hidden directories.
@@ -196,6 +245,14 @@ func (h *Holder) Open() error {
 		h.indexes[index.Name()] = index
 		h.mu.Unlock()
 	}
+
+	// If any fields were opened before their foreign index
+	// was opened, it's safe to process those now since all index
+	// opens have completed by this point.
+	if err := h.processForeignIndexFields(); err != nil {
+		return errors.Wrap(err, "processing foreign index fields")
+	}
+
 	h.Logger.Printf("open holder: complete")
 
 	// Periodically flush cache.
@@ -203,8 +260,37 @@ func (h *Holder) Open() error {
 	go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
 
 	h.Stats.Open()
+	h.snapshotQueue.ScanHolder(h)
 
 	h.opened.Close()
+
+	return nil
+}
+
+// checkForeignIndex is a check before applying a foreign
+// index to a field; if the index is not yet available,
+// (because holder is still opening and may not have opened
+// the index yet), this method queues it up to be processed
+// once all indexes have been opened.
+func (h *Holder) checkForeignIndex(f *Field) error {
+	if h.opening {
+		if fi := h.Index(f.options.ForeignIndex); fi == nil {
+			h.foreignIndexFields = append(h.foreignIndexFields, f)
+			return nil
+		}
+	}
+	return f.applyForeignIndex()
+}
+
+// processForeignIndexFields applies a foreign index to any
+// fields which were opened before their foreign index.
+func (h *Holder) processForeignIndexFields() error {
+	for _, f := range h.foreignIndexFields {
+		if err := f.applyForeignIndex(); err != nil {
+			return errors.Wrap(err, "applying foreign index")
+		}
+	}
+	h.foreignIndexFields = h.foreignIndexFields[:0] // reset
 	return nil
 }
 
@@ -222,8 +308,7 @@ func (h *Holder) Close() error {
 		}
 	}
 	if h.snapshotQueue != nil {
-		close(h.snapshotQueue)
-		// assuming the snapshotQueueWorker has already started, this is safe.
+		h.snapshotQueue.Stop()
 		h.snapshotQueue = nil
 	}
 
@@ -232,13 +317,6 @@ func (h *Holder) Close() error {
 	h.opened.ch = make(chan struct{})
 	h.opened.mu.Unlock()
 
-	h.mu.Lock()
-	if h.translateStoreReplicator != nil {
-		h.translateStoreReplicator.Close()
-		h.translateStoreReplicator = nil
-	}
-	h.mu.Unlock()
-
 	return nil
 }
 
@@ -301,7 +379,10 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap {
 func (h *Holder) Schema() []*IndexInfo {
 	var a []*IndexInfo
 	for _, index := range h.Indexes() {
-		di := &IndexInfo{Name: index.Name()}
+		di := &IndexInfo{
+			Name:    index.Name(),
+			Options: index.Options(),
+		}
 		for _, field := range index.Fields() {
 			fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
 			for _, view := range field.views() {
@@ -350,7 +431,7 @@ func (h *Holder) applySchema(schema *Schema) error {
 		}
 		// Create fields that don't exist.
 		for _, f := range index.Fields {
-			field, err := idx.createFieldIfNotExists(f.Name, f.Options)
+			field, err := idx.createFieldIfNotExists(f.Name, &f.Options)
 			if err != nil {
 				return errors.Wrap(err, "creating field")
 			}
@@ -398,7 +479,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 	defer h.mu.Unlock()
 
 	// Ensure index doesn't already exist.
-	if h.indexes[name] != nil {
+	if h.index(name) != nil {
 		return nil, newConflictError(ErrIndexExists)
 	}
 	return h.createIndex(name, opt)
@@ -407,21 +488,15 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 // CreateIndexIfNotExists returns an index by name.
 // The index is created if it does not already exist.
 func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error) {
-	h.mu.RLock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 
-	// Find index in cache first.
-	if index := h.indexes[name]; index != nil {
-		h.mu.RUnlock()
+	// Return index if it exists.
+	if index := h.index(name); index != nil {
 		return index, nil
 	}
 
-	h.mu.RUnlock()
-
-	index, err := h.CreateIndex(name, opt)
-	if _, ok := err.(ConflictError); err != nil && !ok {
-		return nil, err
-	}
-	return index, nil
+	return h.createIndex(name, opt)
 }
 
 func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
@@ -429,11 +504,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 		return nil, errors.New("index name required")
 	}
 
-	// Return index if it exists.
-	if index := h.index(name); index != nil {
-		return index, nil
-	}
-
 	// Otherwise create a new index.
 	index, err := h.newIndex(h.IndexPath(name), name)
 	if err != nil {
@@ -443,23 +513,27 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err := index.Open(); err != nil {
+	if err = index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
-	} else if err := index.saveMeta(); err != nil {
+	}
+	if err = index.saveMeta(); err != nil {
 		return nil, errors.Wrap(err, "meta")
 	}
 
 	// Update options.
 	h.indexes[index.Name()] = index
 
-	// Restart replication.
-	go h.refreshTranslateStoreReplicator()
+	// Since this is a new index, we need to kick off
+	// its translation sync.
+	if err := h.translationSyncer.Reset(); err != nil {
+		return nil, errors.Wrap(err, "resetting translation sync")
+	}
 
 	return index, nil
 }
 
 func (h *Holder) newIndex(path, name string) (*Index, error) {
-	index, err := NewIndex(path, name)
+	index, err := NewIndex(path, name, h.partitionN)
 	if err != nil {
 		return nil, err
 	}
@@ -469,8 +543,9 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
 	index.newAttrStore = h.NewAttrStore
 	index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data"))
 	index.snapshotQueue = h.snapshotQueue
-	index.holder = h
 	index.OpenTranslateStore = h.OpenTranslateStore
+	index.translationSyncer = h.translationSyncer
+	index.holder = h
 	return index, nil
 }
 
@@ -498,7 +573,10 @@ func (h *Holder) DeleteIndex(name string) error {
 	// Remove reference.
 	delete(h.indexes, name)
 
-	return nil
+	// I'm not sure if calling Reset() here is necessary
+	// since closing the index stops its translation
+	// sync processes.
+	return h.translationSyncer.Reset()
 }
 
 // Field returns the field for an index and name.
@@ -667,243 +745,6 @@ func (h *Holder) logStartup() error {
 	return nil
 }
 
-// TranslateStore returns store for the given index or field.
-func (h *Holder) TranslateStore(index, field string) (TranslateStore, error) {
-	if field == "" {
-		idx := h.Index(index)
-		if idx == nil {
-			return nil, ErrIndexNotFound
-		}
-		return idx.TranslateStore(), nil
-	}
-
-	f := h.Field(index, field)
-	if f == nil {
-		return nil, ErrFieldNotFound
-	}
-	return f.TranslateStore(), nil
-}
-
-// TranslateOffsetMap returns a map of offsets for all indexes & fields.
-func (h *Holder) TranslateOffsetMap() (TranslateOffsetMap, error) {
-	m := make(TranslateOffsetMap)
-	for _, idx := range h.Indexes() {
-		id, err := idx.TranslateStore().MaxID()
-		if err != nil {
-			return nil, err
-		}
-		m.SetIndexOffset(idx.Name(), id+1)
-
-		for _, field := range idx.Fields() {
-			id, err := field.TranslateStore().MaxID()
-			if err != nil {
-				return nil, err
-			}
-			m.SetFieldOffset(idx.Name(), field.Name(), id+1)
-		}
-	}
-	return m, nil
-}
-
-func (h *Holder) setTranslateStoreReadOnly(v bool) {
-	for _, idx := range h.Indexes() {
-		idx.TranslateStore().SetReadOnly(v)
-		for _, field := range idx.Fields() {
-			field.TranslateStore().SetReadOnly(v)
-		}
-	}
-}
-
-func (h *Holder) setPrimaryTranslateStore(node *Node) error {
-	if node != nil && h.OpenTranslateReader == nil {
-		return nil
-	}
-	h.mu.Lock()
-	h.primaryTranslateNode = node.Clone()
-	h.mu.Unlock()
-
-	go h.refreshTranslateStoreReplicator()
-	return nil
-}
-
-func (h *Holder) refreshTranslateStoreReplicator() {
-	h.mu.RLock()
-	node := h.primaryTranslateNode
-	h.mu.RUnlock()
-
-	var nodeURL string
-	if node != nil {
-		u := node.URI.URL()
-		nodeURL = u.String()
-	}
-
-	// Stop existing replication, if running.
-	h.mu.Lock()
-	if h.translateStoreReplicator != nil {
-		h.translateStoreReplicator.Close()
-		h.translateStoreReplicator = nil
-	}
-	h.mu.Unlock()
-
-	// Set all stores read only mode based on if we have a primary.
-	h.setTranslateStoreReadOnly(node != nil)
-
-	// Start replication monitor, if needed.
-	h.mu.Lock()
-	defer h.mu.Unlock()
-	if nodeURL != "" {
-		h.translateStoreReplicator = newHolderTranslateStoreReplicator(h, nodeURL)
-		h.translateStoreReplicator.logger = h.Logger
-		if err := h.translateStoreReplicator.Open(); err != nil {
-			h.Logger.Printf("cannot open translate store replicator: %s", err)
-		}
-	}
-}
-
-// TranslateEntryReader returns a reader that merges all index & field reader
-// that are specified in the offsets map.
-func (h *Holder) TranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
-	// Ensure all readers are cleaned up if any error.
-	var a []TranslateEntryReader
-	defer func() {
-		if err != nil {
-			for i := range a {
-				a[i].Close()
-			}
-		}
-	}()
-
-	// Fetch all readers.
-	for indexName, m := range offsets {
-		for fieldName, offset := range m {
-			var store TranslateStore
-
-			idx := h.Index(indexName)
-			if idx == nil {
-				return nil, ErrIndexNotFound
-			}
-
-			// Fetch from index or field store.
-			if fieldName == "" {
-				store = idx.TranslateStore()
-			} else {
-				f := idx.Field(fieldName)
-				if f == nil {
-					return nil, ErrFieldNotFound
-				}
-				store = f.TranslateStore()
-			}
-
-			// Generate reader and append to multireader.
-			r, err := store.EntryReader(ctx, uint64(offset))
-			if err != nil {
-				return nil, errors.Wrap(err, "translate reader")
-			}
-			a = append(a, r)
-		}
-	}
-
-	return NewMultiTranslateEntryReader(ctx, a), nil
-}
-
-// holderTranslateStoreReplicator manages the replication of translation store
-// data from a primary store to the local replica. Continually tries to
-// reconnect on disconnect.
-type holderTranslateStoreReplicator struct {
-	ctx    context.Context
-	cancel func()
-	wg     sync.WaitGroup
-
-	holder  *Holder
-	nodeURL string
-
-	logger logger.Logger
-}
-
-func newHolderTranslateStoreReplicator(h *Holder, nodeURL string) *holderTranslateStoreReplicator {
-	r := &holderTranslateStoreReplicator{
-		holder:  h,
-		nodeURL: nodeURL,
-		logger:  logger.NopLogger,
-	}
-	r.ctx, r.cancel = context.WithCancel(context.Background())
-	return r
-}
-
-// Open starts the background monitoring goroutine.
-func (r *holderTranslateStoreReplicator) Open() error {
-	r.wg.Add(1)
-	go func() { defer r.wg.Done(); r.monitor() }()
-	return nil
-}
-
-// Close stops the replicator.
-func (r *holderTranslateStoreReplicator) Close() error {
-	r.cancel()
-	return nil
-}
-
-// monitor runs in a background goroutine and continually tries to connect and
-// stream translate changes from the primary store.
-func (r *holderTranslateStoreReplicator) monitor() {
-	for {
-		select {
-		case <-r.ctx.Done():
-			return
-		default:
-			if err := r.replicate(); err != nil {
-				r.logger.Printf("cannot replicate: nodeURL=%s err=%s", r.nodeURL, err)
-			}
-			time.Sleep(1 * time.Second)
-		}
-	}
-}
-
-func (r *holderTranslateStoreReplicator) replicate() error {
-	// Determine the offsets of every index & field store.
-	offsets, err := r.holder.TranslateOffsetMap()
-	if err != nil {
-		return err
-	} else if len(offsets) == 0 {
-		return nil
-	}
-
-	// Begin streaming from remote primary.
-	rd, err := r.holder.OpenTranslateReader(r.ctx, r.nodeURL, offsets)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	for {
-		var entry TranslateEntry
-		if err := rd.ReadEntry(&entry); err != nil {
-			return err
-		}
-
-		// Find appropriate store.
-		var store TranslateStore
-		if entry.Field == "" {
-			idx := r.holder.Index(entry.Index)
-			if idx == nil {
-				return ErrIndexNotFound
-			}
-			store = idx.TranslateStore()
-		} else {
-			f := r.holder.Field(entry.Index, entry.Field)
-			if f == nil {
-				return ErrFieldNotFound
-			}
-			store = f.TranslateStore()
-		}
-
-		// Apply replication to store.
-		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
-			return err
-		}
-	}
-}
-
 // holderSyncer is an active anti-entropy tool that compares the local holder
 // with a remote holder based on block checksums and resolves differences.
 type holderSyncer struct {
@@ -914,6 +755,9 @@ type holderSyncer struct {
 	Node    *Node
 	Cluster *cluster
 
+	// Translation sync handling.
+	readers []TranslateEntryReader
+
 	// Stats
 	Stats stats.StatsClient
 
@@ -988,10 +832,10 @@ func (s *holderSyncer) SyncHolder() error {
 					}
 				}
 			}
-			s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0)
+			s.Stats.Timing(MetricSyncFieldDurationSeconds, time.Since(tf), 1.0)
 			tf = time.Now() // reset tf
 		}
-		s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0)
+		s.Stats.Timing(MetricSyncIndexDurationSeconds, time.Since(ti), 1.0)
 		ti = time.Now() // reset ti
 	}
 
@@ -1015,7 +859,7 @@ func (s *holderSyncer) syncIndex(index string) error {
 	if err != nil {
 		return errors.Wrap(err, "getting blocks")
 	}
-	s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
+	s.Stats.CountWithCustomTags(MetricColumnAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag})
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
@@ -1027,7 +871,7 @@ func (s *holderSyncer) syncIndex(index string) error {
 		} else if len(m) == 0 {
 			continue
 		}
-		s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID})
+		s.Stats.CountWithCustomTags(MetricColumnAttrDiff, int64(len(m)), 1.0, []string{indexTag, node.ID})
 
 		// Update local copy.
 		if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
@@ -1062,7 +906,7 @@ func (s *holderSyncer) syncField(index, name string) error {
 	if err != nil {
 		return errors.Wrap(err, "getting blocks")
 	}
-	s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag})
+	s.Stats.CountWithCustomTags(MetricRowAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag, fieldTag})
 
 	// Sync with every other host.
 	for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
@@ -1076,7 +920,7 @@ func (s *holderSyncer) syncField(index, name string) error {
 		} else if len(m) == 0 {
 			continue
 		}
-		s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
+		s.Stats.CountWithCustomTags(MetricRowAttrDiff, int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
 
 		// Update local copy.
 		if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@@ -1115,10 +959,11 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
 
 	// Sync fragments together.
 	fs := fragmentSyncer{
-		Fragment: frag,
-		Node:     s.Node,
-		Cluster:  s.Cluster,
-		Closing:  s.Closing,
+		Fragment:  frag,
+		Node:      s.Node,
+		Cluster:   s.Cluster,
+		FieldType: f.Type(),
+		Closing:   s.Closing,
 	}
 	if err := fs.syncFragment(); err != nil {
 		return errors.Wrap(err, "syncing fragment")
@@ -1127,6 +972,263 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
 	return nil
 }
 
+// resetTranslationSync reinitializes streaming sync of translation data.
+func (s *holderSyncer) resetTranslationSync() error {
+	// Stop existing streams.
+	if err := s.stopTranslationSync(); err != nil {
+		return errors.Wrap(err, "stop translation sync")
+	}
+
+	// Set read-only flag for all translation stores.
+	s.setTranslateReadOnlyFlags()
+
+	// Connect to each node that has a primary for which we are a replica.
+	if err := s.initializeIndexTranslateReplication(); err != nil {
+		return errors.Wrap(err, "initialize index translate replication")
+	}
+
+	// Connect to coordinator to stream field data.
+	if err := s.initializeFieldTranslateReplication(); err != nil {
+		return errors.Wrap(err, "initialize field translate replication")
+	}
+	return nil
+}
+
+////////////////////////////////////////////////////////////
+
+// translationSyncer provides an interface allowing a function
+// to notify the server that an action has occurred which requires
+// the translation sync process to be reset. In general, this
+// includes anything which modifies schema (add/remove index, etc),
+// or anything that changes the cluster topology (add/remove node).
+// I originally considered leveraging the broadcaster since that was
+// already in place and provides similar event messages, but the
+// broadcaster is really meant for notifiying other nodes, while
+// this is more akin to an internal message bus. In fact, I think
+// a future iteration on this may be to make it more generic so
+// it can act as an internal message bus where one of the messages
+// being published is "translationSyncReset".
+type translationSyncer interface {
+	Reset() error
+}
+
+// NopTranslationSyncer represents a translationSyncer that doesn't do anything.
+var NopTranslationSyncer translationSyncer = &nopTranslationSyncer{}
+
+type nopTranslationSyncer struct{}
+
+// Reset is a no-op implementation of translationSyncer Reset method.
+func (nopTranslationSyncer) Reset() error { return nil }
+
+// activeTranslationSyncer represents a translationSyncer that resets
+// the server's translation syncer.
+type activeTranslationSyncer struct {
+	ch chan struct{}
+}
+
+// newActiveTranslationSyncer returns a new instance of activeTranslationSyncer.
+func newActiveTranslationSyncer(ch chan struct{}) *activeTranslationSyncer {
+	return &activeTranslationSyncer{
+		ch: ch,
+	}
+}
+
+// Reset resets the server's translation syncer.
+func (a *activeTranslationSyncer) Reset() error {
+	a.ch <- struct{}{}
+	return nil
+}
+
+////////////////////////////////////////////////////////////
+
+// stopTranslationSync closes and waits for all outstanding translation readers
+// to complete. This should be called before reconnecting to the cluster in case
+// of a cluster resize or schema change.
+func (s *holderSyncer) stopTranslationSync() error {
+	var g errgroup.Group
+	for i := range s.readers {
+		rd := s.readers[i]
+		g.Go(func() error {
+			return rd.Close()
+		})
+	}
+	return g.Wait()
+}
+
+// setTranslateReadOnlyFlags updates all translation stores to enable or disable
+// writing new translation keys. Index stores are writable if the node owns the
+// partition. Field stores are writable if the node is the coordinator.
+func (s *holderSyncer) setTranslateReadOnlyFlags() {
+	isCoordinator := s.Cluster.isCoordinator()
+
+	for _, index := range s.Holder.Indexes() {
+		// There is a race condition here:
+		// if Indexes() returns idx1, and then in another
+		// process, holder.DeleteIndex(idx1) is called,
+		// then the next step trying to get TranslateStore(partitionID)
+		// for an index that is closed (and therefore its transateStores
+		// no longer exist) will fail with a nil pointer error.
+		// For now, I just checked that the translateStore hasn't been
+		// set to nil before trying to use it, but another option may
+		// be to prevent the translateStores from being zeroed out
+		// while this process is active. Checking for nil as we do
+		// really obviates the need for the RLock around the for loop.
+
+		// Obtain a read lock on index to prevent Index.Close() from
+		// destroying the Index.translateStores map before this is
+		// done using it.
+		index.mu.RLock()
+		for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
+			ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID)
+			if ts := index.TranslateStore(partitionID); ts != nil {
+				ts.SetReadOnly(!ownsPartition)
+			}
+		}
+		index.mu.RUnlock()
+
+		for _, field := range index.Fields() {
+			field.TranslateStore().SetReadOnly(!isCoordinator)
+		}
+	}
+}
+
+// initializeIndexTranslateReplication connects to each node that is the
+// primary for a partition that we are a replica of.
+func (s *holderSyncer) initializeIndexTranslateReplication() error {
+	for _, node := range s.Cluster.Nodes() {
+		// Skip local node.
+		if node.ID == s.Node.ID {
+			continue
+		}
+
+		// Build a map of partition offsets to stream from.
+		m := make(TranslateOffsetMap)
+		for _, index := range s.Holder.Indexes() {
+			if !index.Keys() {
+				continue
+			}
+
+			for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
+				partitionNodes := s.Cluster.partitionNodes(partitionID)
+				isPrimary := partitionNodes[0].ID == node.ID                 // remote is primary?
+				isReplica := Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica?
+				if !isPrimary || !isReplica {
+					continue
+				}
+
+				store := index.TranslateStore(partitionID)
+				offset, err := store.MaxID()
+				if err != nil {
+					return errors.Wrapf(err, "cannot determine max id for %q", index.Name())
+				}
+				m.SetIndexPartitionOffset(index.Name(), partitionID, offset)
+			}
+		}
+
+		// Skip if no replication required.
+		if len(m) == 0 {
+			continue
+		}
+
+		// Connect to remote node and begin streaming.
+		rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
+		if err != nil {
+			return err
+		}
+		s.readers = append(s.readers, rd)
+
+		go func() { defer rd.Close(); s.readIndexTranslateReader(rd) }()
+	}
+
+	return nil
+}
+
+// initializeFieldTranslateReplication connects the coordinator to stream field data.
+func (s *holderSyncer) initializeFieldTranslateReplication() error {
+	// Skip if coordinator.
+	if s.Cluster.isCoordinator() {
+		return nil
+	}
+
+	// Build a map of partition offsets to stream from.
+	m := make(TranslateOffsetMap)
+	for _, index := range s.Holder.Indexes() {
+		for _, field := range index.Fields() {
+			store := field.TranslateStore()
+			offset, err := store.MaxID()
+			if err != nil {
+				return errors.Wrapf(err, "cannot determine max id for %q/%q", index.Name(), field.Name())
+			}
+			m.SetFieldOffset(index.Name(), field.Name(), offset)
+		}
+	}
+
+	// Skip if no replication required.
+	if len(m) == 0 {
+		return nil
+	}
+
+	// Connect to coordinator and begin streaming.
+	coordinator := s.Cluster.coordinatorNode()
+	rd, err := s.Holder.OpenTranslateReader(context.Background(), coordinator.URI.String(), m)
+	if err != nil {
+		return err
+	}
+	s.readers = append(s.readers, rd)
+
+	go func() { defer rd.Close(); s.readFieldTranslateReader(rd) }()
+
+	return nil
+}
+
+func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
+	for {
+		var entry TranslateEntry
+		if err := rd.ReadEntry(&entry); err != nil {
+			s.Holder.Logger.Printf("cannot read index translate entry: %s", err)
+			return
+		}
+
+		// Find appropriate store.
+		idx := s.Holder.Index(entry.Index)
+		if idx == nil {
+			s.Holder.Logger.Printf("index not found: %q", entry.Index)
+			return
+		}
+
+		// Apply replication to store.
+		store := idx.TranslateStore(s.Cluster.keyPartition(entry.Index, entry.Key))
+		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
+			s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key)
+			return
+		}
+	}
+}
+
+func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) {
+	for {
+		var entry TranslateEntry
+		if err := rd.ReadEntry(&entry); err != nil {
+			s.Holder.Logger.Printf("cannot read field translate entry: %s", err)
+			return
+		}
+
+		// Find appropriate store.
+		f := s.Holder.Field(entry.Index, entry.Field)
+		if f == nil {
+			s.Holder.Logger.Printf("field not found: %s/%s", entry.Index, entry.Field)
+			return
+		}
+
+		// Apply replication to store.
+		store := f.TranslateStore()
+		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
+			s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key)
+			return
+		}
+	}
+}
+
 // holderCleaner removes fragments and data files that are no longer used.
 type holderCleaner struct {
 	Node *Node
@@ -1162,6 +1264,15 @@ func (c *holderCleaner) CleanHolder() error {
 
 		// Get the fragments registered in memory.
 		for _, field := range index.Fields() {
+			// deletedShards is used to track which shards for the field
+			// were deleted. Any shards that get deleted from this node
+			// get added to remoteAvailableShards. This is done because
+			// the CleanHolder process is cleaning up shards which got
+			// moved to other nodes. Because those shards still exist
+			// (just no longer on this particular node), this node still
+			// needs to consider each of them as an available shard in
+			// the cluster.
+			var deletedShards []uint64
 			for _, view := range field.views() {
 				for _, fragment := range view.allFragments() {
 					fragShard := fragment.shard
@@ -1173,6 +1284,12 @@ func (c *holderCleaner) CleanHolder() error {
 					if err := view.deleteFragment(fragShard); err != nil {
 						return errors.Wrap(err, "deleting fragment")
 					}
+					deletedShards = append(deletedShards, fragShard)
+				}
+			}
+			if len(deletedShards) > 0 {
+				if err := field.AddRemoteAvailableShards(roaring.NewBitmap(deletedShards...)); err != nil {
+					return errors.Wrap(err, "adding remote available shards")
 				}
 			}
 		}

holder_internal_test.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2/roaring"
 )
@@ -39,7 +40,7 @@ func (h *tHolder) Close() error {
 // Note that the holder must be Closed first.
 func (h *tHolder) Reopen() error {
 	path, logger := h.Path, h.Holder.Logger
-	h.Holder = NewHolder()
+	h.Holder = NewHolder(DefaultPartitionN)
 	h.Holder.Path = path
 	h.Holder.Logger = logger
 	return h.Holder.Open()
@@ -51,7 +52,7 @@ func newHolder() *tHolder {
 		panic(err)
 	}
 
-	h := &tHolder{Holder: NewHolder()}
+	h := &tHolder{Holder: NewHolder(DefaultPartitionN)}
 	h.Path = path
 	return h
 }
@@ -98,6 +99,7 @@ func TestHolder_Optn(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
+		availableShardFileFlushDuration.Set(100 * time.Millisecond)
 		h := newHolder()
 		defer h.Close()
 
@@ -182,6 +184,7 @@ func TestHolder_Optn(t *testing.T) {
 
 // Ensure holder can clean up orphaned fragments.
 func TestHolderCleaner_CleanHolder(t *testing.T) {
+	availableShardFileFlushDuration.Set(100 * time.Millisecond) //shorten the default time to force a file write
 	cluster := NewTestCluster(2)
 
 	// Create a local holder.
@@ -223,6 +226,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
 	if err != nil {
 		t.Fatalf("adding remote shards: %v", err)
 	}
+	time.Sleep(2 * availableShardFileFlushDuration.Get())
 
 	// Keep replication the same and ensure we get the expected results.
 	cluster.ReplicaN = 2
@@ -302,7 +306,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
 
 // Ensure holder can reopen.
 func TestHolderCleaner_Reopen(t *testing.T) {
-	h := NewHolder()
+	h := NewHolder(DefaultPartitionN)
 	h.Path = "path"
 	err := h.Open()
 	if err != nil {

holder_test.go

@@ -17,6 +17,7 @@ package pilosa_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,7 +26,9 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 func TestHolder_Open(t *testing.T) {
@@ -197,7 +200,110 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
+	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		h := test.MustOpenHolder()
+		defer h.Close()
 
+		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := h.Reopen(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ForeignIndex", func(t *testing.T) {
+		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else {
+				_, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("nonexistent"))
+				if err == nil {
+					t.Fatalf("expected error: %s", pilosa.ErrForeignIndexNotFound)
+				} else if errors.Cause(err) != pilosa.ErrForeignIndexNotFound {
+					t.Fatalf("expected error: %s, but got: %s", pilosa.ErrForeignIndexNotFound, err)
+				}
+			}
+		})
+
+		// Foreign index zzz is opened after foo/bar.
+		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("zzz")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Foreign index aaa is opened before foo/bar.
+		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("aaa")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Try to re-create existing index
+		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err = h.CreateIndex("aaa", pilosa.IndexOptions{}); err == nil {
+				t.Fatalf("expected: ConflictError, got: nil")
+			} else if _, ok := err.(pilosa.ConflictError); !ok {
+				t.Fatalf("expected: ConflictError, got: %s", err)
+			}
+
+			idx2, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if idx1 != idx2 {
+				t.Fatalf("expected the same indexes, got: %s and %s", idx1.Name(), idx2.Name())
+			}
+		})
+	})
 }
 
 func TestHolder_HasData(t *testing.T) {
@@ -303,10 +409,14 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	if err != nil {
 		t.Fatalf("creating field f0: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field z in y: %v", err)
 	}
+	_, err = c[0].API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
+	if err != nil {
+		t.Fatalf("creating field b in y: %v", err)
+	}
 
 	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
 	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
@@ -321,6 +431,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 
 	// Set a bit to create the fragment.
 	hldr0.SetBit("y", "z", 0, 0)
+	hldr0.SetBit("y", "b", 0, 0) // rowID = 0 means false
 
 	// Set data on the remote holder.
 	hldr1.SetBit("i", "f", 0, 4000)
@@ -331,6 +442,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
 	hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
 
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+4) // true
+	hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
+	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
+
 	err = c[0].Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
@@ -365,6 +480,119 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 		if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
 			t.Errorf("unexpected columns(%d/y/z): %+v", i, a)
 		}
+
+		if a := hldr.Row("y", "b", 0).Columns(); !reflect.DeepEqual(a, []uint64{0, (3 * ShardWidth) + 5}) {
+			t.Errorf("unexpected false columns(%d/y/b): %+v", i, a)
+		}
+		if a := hldr.Row("y", "b", 1).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 7}) {
+			t.Errorf("unexpected true columns(%d/y/b): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder can sync with a remote holder and respects
+// the row boundaries of the block.
+func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c[0].Config.Cluster.ReplicaN = 3
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 3
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	blockEdge := uint64(pilosa.HashBlockSize)
+
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+
+	// Set data on the local holder.
+	hldr0.SetBit("i", "f", blockEdge-1, 10)
+	hldr0.SetBit("i", "f", blockEdge, 20)
+
+	// Set the same data on one of the replicas
+	// so that we have a quorum.
+	hldr1.SetBit("i", "f", blockEdge-1, 10)
+	hldr1.SetBit("i", "f", blockEdge, 20)
+
+	// Leave the third replica empty to force a block merge.
+	//
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", blockEdge-1).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
+			t.Errorf("unexpected columns(%d/block 0): %+v", i, a)
+		}
+		if a := hldr.Row("i", "f", blockEdge).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d/block 1): %+v", i, a)
+		}
+	}
+}
+
+// Ensure holder correctly handles clears during block sync.
+func TestHolderSyncer_Clears(t *testing.T) {
+	c := test.MustNewCluster(t, 3)
+	c[0].Config.Cluster.ReplicaN = 3
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 3
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+
+	// Set data on the local holder that should be cleared
+	// because it's the only instance of this value.
+	hldr0.SetBit("i", "f", 0, 30)
+
+	// Set similar data on the replicas, but
+	// different from what's on local. This should end
+	// up being set on all replicas
+	hldr1.SetBit("i", "f", 0, 20)
+	hldr2.SetBit("i", "f", 0, 20)
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+
+	// Verify data is the same on all nodes.
+	for i, hldr := range []*test.Holder{hldr0, hldr1, hldr2} {
+		if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
+			t.Errorf("unexpected columns(%d): %+v", i, a)
+		}
 	}
 }
 
@@ -419,3 +647,112 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 		}
 	}
 }
+
+// Ensure holder can sync integer views with a remote holder.
+func TestHolderSyncer_IntField(t *testing.T) {
+	t.Run("BasicSync", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c[0].Config.Cluster.ReplicaN = 2
+		c[0].Config.AntiEntropy.Interval = 0
+		c[1].Config.Cluster.ReplicaN = 2
+		c[1].Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+
+		// Set data on the local holder for node0.
+		hldr0.SetValue("i", "f", 1, 1)
+
+		// Set data on node1.
+		hldr1.SetValue("i", "f", 2, 2)
+
+		err = c[0].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
+				t.Errorf("unexpected value(node%d/0): %d, exists: %v", i, a, exists)
+			}
+			if a, exists := hldr.Value("i", "f", 2); exists {
+				t.Errorf("unexpected value(node%d/1): %d, exists: %v", i, a, exists)
+			}
+		}
+	})
+
+	t.Run("MultiShard", func(t *testing.T) {
+		c := test.MustNewCluster(t, 2)
+		c[0].Config.Cluster.ReplicaN = 2
+		c[0].Config.AntiEntropy.Interval = 0
+		c[1].Config.Cluster.ReplicaN = 2
+		c[1].Config.AntiEntropy.Interval = 0
+		err := c.Start()
+		if err != nil {
+			t.Fatalf("starting cluster: %v", err)
+		}
+		defer c.Close()
+
+		_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index i: %v", err)
+		}
+		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatalf("creating field f: %v", err)
+		}
+
+		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+
+		// Set data on the local holder for node0.
+		hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
+		hldr0.SetValue("i", "f", 3*pilosa.ShardWidth, 32)
+		hldr0.SetValue("i", "f", 4*pilosa.ShardWidth, math.MinInt32)
+		hldr0.SetValue("i", "f", 7*pilosa.ShardWidth, math.MinInt32)
+
+		// Set data on node1.
+		hldr1.SetValue("i", "f", 0*pilosa.ShardWidth, 2)
+		hldr1.SetValue("i", "f", 2*pilosa.ShardWidth, 22)
+		hldr1.SetValue("i", "f", 4*pilosa.ShardWidth, math.MaxInt32)
+		hldr1.SetValue("i", "f", 7*pilosa.ShardWidth, math.MaxInt32)
+
+		// Primary for shards (for index "i"):
+		// node0: [0,3,7]
+		// node1: [1,2,4]
+
+		err = c[0].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 0: %v", err)
+		}
+		err = c[1].Server.SyncData()
+		if err != nil {
+			t.Fatalf("syncing node 1: %v", err)
+		}
+
+		// Verify data is the same on both nodes.
+		for i, hldr := range []*test.Holder{hldr0, hldr1} {
+			if a := hldr.Range("i", "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+			if a := hldr.Range("i", "f", pql.LT, 0); !reflect.DeepEqual(a.Columns(), []uint64{7 * pilosa.ShardWidth}) {
+				t.Errorf("unexpected columns(node%d/0): %d", i, a.Columns())
+			}
+		}
+	})
+}

http/client.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/encoding/proto"
@@ -392,6 +393,13 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits
 
 	// Get the coordinator node; all bits are sent to the
 	// primary translate store (i.e. coordinator).
+	// TODO... is that right^^?
+	// RESPONSE: It looks like in ctl/import.go, we could change the
+	// logic in ImportCommand.importBits() to only use ImportK
+	// when useRowKeys = true. It's no longer necessary to
+	// send column key translations to the coordinator (although
+	// it should still work). As far as I know, the only thing
+	// that uses ImportK is the pilosa import sub-command.
 	nodes, err := c.Nodes(ctx)
 	if err != nil {
 		return fmt.Errorf("getting nodes: %s", err)
@@ -553,6 +561,35 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 	return nil
 }
 
+// ImportValue2 is a simplified ImportValue method which just uses the
+// ImportValueRequest instead of splitting up ImportValue and
+// ImportValueK... it also supports importing float values. The idea
+// being that (assuming it works) this will become the default (and be
+// renamed) for 2.0, and we can deprecate the other methods.
+func (c *InternalClient) ImportValue2(ctx context.Context, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.NewImportValue")
+	defer span.Finish()
+
+	buf, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Errorf("marshal import request: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, req.Index, req.Shard)
+	if err != nil {
+		return errors.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, req.Index, req.Field, buf, options); err != nil {
+			return errors.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+	return nil
+}
+
 // ImportValueK bulk imports keyed field values to a host.
 func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
@@ -667,6 +704,56 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	return nil
 }
 
+// ImportColumnAttrs does bulk import of column attrs
+func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	url := fmt.Sprintf("%s/index/%s/import-column-attrs", uri, index)
+
+	// Marshal data to protobuf.
+	data, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Wrap(err, "marshal import-column-attrs request")
+	}
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("Content-Type", "application/x-protobuf")
+	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	dec := json.NewDecoder(resp.Body)
+	rbody := &pilosa.ImportResponse{}
+	err = dec.Decode(rbody)
+	// Decode can return EOF when no error occurred. helpful!
+	if err != nil && err != io.EOF {
+		return errors.Wrap(err, "decoding response body")
+	}
+	if rbody.Err != "" {
+		return errors.Wrap(errors.New(rbody.Err), "importing roaring")
+	}
+
+	return nil
+}
+
 // ExportCSV bulk exports data for a single shard from a host to CSV format.
 func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
@@ -791,18 +878,28 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
 	}
 
 	// convert pilosa.FieldOptions to fieldOptions
+	//
+	// TODO this kind of sucks because it's one more place that needs
+	// changes when we change anything with field options (and there
+	// are a lot of places already). It's not clear to me that this is
+	// providing a lot of value, but I think this kind of validation
+	// should probably happen in the field anyway??
 	fieldOpt := fieldOptions{
 		Type: opt.Type,
 		Keys: &opt.Keys,
 	}
-	if fieldOpt.Type == "set" {
+	if fieldOpt.Type == pilosa.FieldTypeSet {
 		fieldOpt.CacheType = &opt.CacheType
 		fieldOpt.CacheSize = &opt.CacheSize
-	} else if fieldOpt.Type == "int" {
+	} else if fieldOpt.Type == pilosa.FieldTypeInt {
 		fieldOpt.Min = &opt.Min
 		fieldOpt.Max = &opt.Max
-	} else if fieldOpt.Type == "time" {
+	} else if fieldOpt.Type == pilosa.FieldTypeTime {
 		fieldOpt.TimeQuantum = &opt.TimeQuantum
+	} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
+		fieldOpt.Min = &opt.Min
+		fieldOpt.Max = &opt.Max
+		fieldOpt.Scale = &opt.Scale
 	}
 
 	// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
@@ -1026,21 +1123,284 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Connection", "keep-alive")
 
 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
 		return errors.Wrap(err, "executing request")
 	}
-	return errors.Wrap(resp.Body.Close(), "closing response body")
+	defer resp.Body.Close()
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	return errors.Wrap(err, "draining SendMessage response body")
+}
+
+// TranslateKeysNode sends a key translation request to a specific node.
+func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string) ([]uint64, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{
+		Index: index,
+		Field: field,
+		Keys:  keys,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateKeysRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/keys")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateKeysResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.IDs, nil
+}
+
+// TranslateIDsNode sends an id translation request to a specific node.
+func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{
+		Index: index,
+		Field: field,
+		IDs:   ids,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateIDsRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/ids")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateIDsResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.Keys, nil
+}
+
+func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transactions")
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating transactions request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	trnsMap := make(map[string]*pilosa.Transaction)
+	err = json.NewDecoder(resp.Body).Decode(&trnsMap)
+	return trnsMap, errors.Wrap(err, "json decoding")
+}
+
+func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.StartTransaction")
+	defer span.Finish()
+	buf, err := json.Marshal(&pilosa.Transaction{
+		ID:        id,
+		Timeout:   timeout,
+		Exclusive: exclusive,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshalling payload")
+	}
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating post transaction request")
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+	if resp.StatusCode == 409 {
+		err = pilosa.ErrTransactionExclusive
+	} else if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FinishTransaction")
+	defer span.Finish()
+
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id+"/finish")
+	req, err := http.NewRequest("POST", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating finish transaction request")
+	}
+
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetTransaction")
+	defer span.Finish()
+
+	// We're using the defaultURI here because this is only used by
+	// tests, and we want to test requests against all hosts. A robust
+	// client implementation would ensure that these requests go to
+	// the coordinator.
+	u := uriPathToURL(c.defaultURI, "/transaction/"+id)
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating get transaction request")
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing request")
+	}
+	defer func() {
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+	tr := &TransactionResponse{}
+	err = json.NewDecoder(resp.Body).Decode(tr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding response")
+	}
+
+	if tr.Error != "" {
+		err = errors.New(tr.Error)
+	}
+	return tr.Transaction, err
+}
+
+type executeOpts struct {
+	// giveRawResponse instructs executeRequest not to process the
+	// respStatusCode and try to extract errors or whatever.
+	giveRawResponse bool
+}
+
+type executeRequestOption func(*executeOpts)
+
+func giveRawResponse(b bool) executeRequestOption {
+	return func(eo *executeOpts) {
+		eo.giveRawResponse = b
+	}
 }
 
 // executeRequest executes the given request and checks the Response. For
 // responses with non-2XX status, the body is read and closed, and an error is
 // returned. If the error is nil, the caller must ensure that the response body
 // is closed.
-func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) {
+func (c *InternalClient) executeRequest(req *http.Request, opts ...executeRequestOption) (*http.Response, error) {
+	eo := &executeOpts{}
+	for _, opt := range opts {
+		opt(eo)
+	}
+
 	tracing.GlobalTracer.InjectHTTPHeaders(req)
+	req.Close = false
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		if resp != nil {
@@ -1048,6 +1408,9 @@ func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, erro
 		}
 		return nil, errors.Wrap(err, "getting response")
 	}
+	if eo.giveRawResponse {
+		return resp, nil
+	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		defer resp.Body.Close()
 		buf, err := ioutil.ReadAll(resp.Body)
@@ -1278,3 +1641,40 @@ func nodePathToURL(node *pilosa.Node, path string) url.URL {
 		Path:   path,
 	}
 }
+
+// RetrieveTranslatePartitionFromURI returns a ReadCloser which contains the data of the
+// specified translate partition from the specified node. Caller *must* close the returned
+// ReadCloser or risk leaking goroutines/tcp connections.
+func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pilosa.URI) (io.ReadCloser, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveTranslatePartitionFromURI")
+	defer span.Finish()
+
+	node := &pilosa.Node{
+		URI: uri,
+	}
+
+	u := nodePathToURL(node, "/internal/translate/data")
+	u.RawQuery = url.Values{
+		"index":     {index},
+		"partition": {strconv.FormatInt(int64(partition), 10)},
+	}.Encode()
+
+	// Build request.
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			return nil, pilosa.ErrFragmentNotFound
+		}
+		return nil, err
+	}
+
+	return resp.Body, nil
+}

http/client_test.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	gohttp "net/http"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -31,6 +33,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/server"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 // Test distributed TopN Row count across 3 nodes.
@@ -116,15 +119,15 @@ func TestClient_MultiNode(t *testing.T) {
 	// Rebuild the RankCache.
 	// We have to do this to avoid the 10-second cache invalidation delay
 	// built into cache.Invalidate()
-	err = c[0].RecalculateCaches()
+	err = c[0].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[1].RecalculateCaches()
+	err = c[1].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[2].RecalculateCaches()
+	err = c[2].RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
@@ -147,7 +150,8 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	// Test must return exactly N results.
-	if len(result.Results[0].([]pilosa.Pair)) != topN {
+	pairsField := result.Results[0].(*pilosa.PairsField)
+	if len(pairsField.Pairs) != topN {
 		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
 	}
 	p := []pilosa.Pair{
@@ -157,7 +161,7 @@ func TestClient_MultiNode(t *testing.T) {
 		{ID: 99, Count: 7}}
 
 	// Valdidate the Top 4 result counts.
-	if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
+	if !reflect.DeepEqual(pairsField.Pairs, p) {
 		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
 	}
 
@@ -292,22 +296,29 @@ func TestClient_Export(t *testing.T) {
 		buf := bytes.NewBuffer(nil)
 		bw := bufio.NewWriter(buf)
 
-		// Send export request.
-		if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", 0, bw); err != nil {
-			t.Fatal(err)
+		// Send export request for every partition.
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil {
+				t.Fatal(err)
+			}
 		}
 
 		got := buf.String()
 
-		// Expected output.
-		exp := ""
-		for _, bit := range data {
-			exp += fmt.Sprintf("%d,%s\n", bit.RowID, bit.ColumnKey)
-		}
+		// Expected output is not sorted because of key sharding.
+		exp := "" +
+			"2,col200\n" +
+			"2,col201\n" +
+			"2,col202\n" +
+			"2,col203\n" +
+			"1,col103\n" +
+			"1,col102\n" +
+			"1,col101\n" +
+			"1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 
@@ -327,21 +338,28 @@ func TestClient_Export(t *testing.T) {
 		bw := bufio.NewWriter(buf)
 
 		// Send export request.
-		if err := c.ExportCSV(context.Background(), "keyed", "keyedf", 0, bw); err != nil {
-			t.Fatal(err)
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil {
+				t.Fatal(err)
+			}
 		}
 
 		got := buf.String()
 
-		// Expected output.
-		exp := ""
-		for _, bit := range data {
-			exp += fmt.Sprintf("%s,%s\n", bit.RowKey, bit.ColumnKey)
-		}
+		// Expected output is unsorted because of key sharding.
+		exp := "" +
+			"row2,col200\n" +
+			"row2,col201\n" +
+			"row2,col202\n" +
+			"row2,col203\n" +
+			"row1,col103\n" +
+			"row1,col102\n" +
+			"row1,col101\n" +
+			"row1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 }
@@ -394,6 +412,60 @@ func TestClient_Import(t *testing.T) {
 	}
 }
 
+// Ensure client can bulk import column attrs.
+func TestClient_ImportColumnAttrs(t *testing.T) {
+	cluster := test.MustNewCluster(t, 2)
+	for _, c := range cluster {
+		c.Config.Cluster.ReplicaN = 2
+	}
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer cluster.Close()
+
+	ctx := context.Background()
+	_, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+	_, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+
+	attrKey := "k"
+	// Send import request.
+	host := cluster[0].URL()
+	c := MustNewClient(host, http.GetHTTPClient(nil))
+	colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
+	if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data.
+	pql := "Options(Row(f=0), columnAttrs=true)"
+	res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res.ColumnAttrSets) != 5 {
+		t.Fatal("incorrect number of column attrs set")
+	}
+
+	for _, v := range res.ColumnAttrSets {
+		attrVal := attrFun(v.ID)
+		if attrVal != v.Attrs[attrKey] {
+			t.Fatal(err)
+		}
+	}
+
+}
+
 // Ensure client can bulk import data.
 func TestClient_ImportRoaring(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
@@ -546,18 +618,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -573,18 +645,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(unkeyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{ID: 1, Count: 3},
 				{ID: 2, Count: 2},
 				{ID: 3, Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -600,18 +672,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd.MustRecalculateCaches(t)
-			resp := cmd.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "unkeyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 	})
@@ -645,18 +717,18 @@ func TestClient_ImportKeys(t *testing.T) {
 				t.Fatal(err)
 			}
 			cmd0.MustRecalculateCaches(t)
-			resp := cmd0.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd0.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf0)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -677,18 +749,18 @@ func TestClient_ImportKeys(t *testing.T) {
 			time.Sleep(500 * time.Millisecond)
 
 			cmd1.MustRecalculateCaches(t)
-			resp := cmd1.MustQuery(t, &pilosa.QueryRequest{
+			resp := cmd1.QueryAPI(t, &pilosa.QueryRequest{
 				Index: "keyed",
 				Query: "TopN(keyedf1)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %#v", pairs)
+				t.Fatalf("unexpected topn result: %#v", pairs.Pairs)
 			}
 		})
 	})
@@ -777,6 +849,72 @@ func TestClient_ImportKeys(t *testing.T) {
 	})
 }
 
+func TestClient_ImportIDs(t *testing.T) {
+	// Ensure that running a query between two imports does
+	// not affect the result set. It turns out, this is caused
+	// by the fragment.rowCache failing to be cleared after an
+	// importValue. This ensures that the rowCache is cleared
+	// after an import.
+	t.Run("ImportRangeImport", func(t *testing.T) {
+		cluster := test.MustRunCluster(t, 1)
+		defer cluster.Close()
+		cmd := cluster[0]
+		host := cmd.URL()
+		holder := cmd.Server.Holder()
+		hldr := test.Holder{Holder: holder}
+
+		idxName := "i"
+		fldName := "f"
+
+		// Load bitmap into cache to ensure cache gets updated.
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{Keys: false})
+		_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-10000, 10000))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Send import request.
+		c := MustNewClient(host, http.GetHTTPClient(nil))
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 2, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		queryRequest := &pilosa.QueryRequest{
+			Query:  fmt.Sprintf(`Row(%s>0)`, fldName),
+			Remote: false,
+		}
+
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+
+		// Send import request.
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 1000, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2, 1000}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+	})
+}
+
 // Ensure client can bulk import value data.
 func TestClient_ImportValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
@@ -998,6 +1136,271 @@ func TestClient_FragmentBlocks(t *testing.T) {
 	}
 }
 
+func TestClient_CreateDecimalField(t *testing.T) {
+	cluster := test.MustRunCluster(t, 1)
+	defer cluster.Close()
+	cmd := cluster[0]
+
+	c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
+
+	index := "cdf"
+	err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	field := "dfield"
+	err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeDecimal, Scale: 1, Min: pql.NewDecimal(-1000, 0), Max: pql.NewDecimal(1000, 0)})
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	fld, err := cmd.API.Field(context.Background(), index, field)
+	if err != nil {
+		t.Fatalf("getting field: %v", err)
+	}
+	if fld.Options().Scale != 1 {
+		t.Fatalf("expected Scale 1, got: %+v", fld.Options())
+	}
+
+	err = c.ImportValue2(context.Background(), &pilosa.ImportValueRequest{Index: index, Field: field, ColumnIDs: []uint64{1, 2, 3}, Shard: 0, FloatValues: []float64{1.1, 2.2, 3.3}}, &pilosa.ImportOptions{})
+	if err != nil {
+		t.Fatalf("importing float values: %v", err)
+	}
+
+	// Integer predicate.
+	resp, err := c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicate.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.1)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Integer predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1<dfield<3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<=dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<=3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>=2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+}
+
+func TestClientTransactions(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	client0 := MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
+	client1 := MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
+
+	// can create, list, get, and finish a transaction
+	var expDeadline time.Time
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	if trnsMap, err := client0.Transactions(context.Background()); err != nil {
+		t.Errorf("listing transactions: %v", err)
+	} else {
+		if len(trnsMap) != 1 {
+			t.Errorf("unexpected trnsMap: %+v", trnsMap)
+		}
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trnsMap["blah"])
+	}
+
+	if trns, err := client0.GetTransaction(context.Background(), "blah"); err != nil {
+		t.Fatalf("error getting transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	if trns, err := client0.FinishTransaction(context.Background(), "blah"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// can create exclusive transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blahe", time.Minute, true); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// cannot start new transaction - correct error and exclusive transaction are returned
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); errors.Cause(err) != pilosa.ErrTransactionExclusive {
+		t.Fatalf("shouldn't be able to start transaction while an exclusive is running, but got: %+v, %v", trns, err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish exclusive transaction
+	if trns, err := client0.FinishTransaction(context.Background(), "blahe"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: true, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// start new transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// try to start same transaction
+	if trns, err := client0.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionExists.Error()) {
+		t.Fatalf("expected ErrTransactionExists, but got: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// start an exclusive transaction which can't go active
+	if trns, err := client0.StartTransaction(context.Background(), "blahe", time.Minute, true); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: false, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish exclusive transaction that never went active
+	if trns, err := client0.FinishTransaction(context.Background(), "blahe"); err != nil {
+		t.Fatalf("error finishing transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: "blahe", Timeout: time.Minute, Active: false, Exclusive: true, Deadline: expDeadline},
+			trns)
+	}
+
+	// finish non-existent transaction
+	if trns, err := client0.FinishTransaction(context.Background(), "zzz"); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionNotFound.Error()) {
+		t.Fatalf("unexpected error finishing nonexistent transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// get non-existent transaction
+	if trns, err := client0.GetTransaction(context.Background(), "xxx"); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrTransactionNotFound.Error()) {
+		t.Fatalf("unexpected error getting nonexistent transaction: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// non-coordinator
+	if trns, err := client1.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
+		!strings.Contains(err.Error(), pilosa.ErrNodeNotCoordinator.Error()) {
+		t.Fatalf("unexpected error starting on non-coordinator: %v", err)
+	} else {
+		test.CompareTransactions(t,
+			nil,
+			trns)
+	}
+
+	// start transaction with blank id
+	if trns, err := client0.StartTransaction(context.Background(), "", time.Minute, false); err != nil {
+		t.Fatalf("error starting transaction: %v", err)
+	} else {
+		expDeadline = time.Now().Add(time.Minute)
+		if len(trns.ID) != 36 {
+			t.Errorf("expected generated UUID, but got '%s'", trns.ID)
+		}
+		test.CompareTransactions(t,
+			&pilosa.Transaction{ID: trns.ID, Timeout: time.Minute, Active: true, Deadline: expDeadline},
+			trns)
+	}
+
+}
+
 // Client represents a test wrapper for pilosa.Client.
 type Client struct {
 	*http.InternalClient
@@ -1021,3 +1424,23 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring
 		},
 	}
 }
+
+func attrFun(id uint64) string {
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pilosa.ImportColumnAttrsRequest {
+	colIDs := make([]uint64, 0, 5)
+	attrVals := make([]string, 0, 5)
+	for n := uint64(0); n < 5; n++ {
+		colIDs = append(colIDs, n)
+		attrVals = append(attrVals, attrFun(n))
+	}
+	return &pilosa.ImportColumnAttrsRequest{
+		Index:     index,
+		Shard:     shard,
+		AttrKey:   attrKey,
+		ColumnIDs: colIDs,
+		AttrVals:  attrVals,
+	}
+}

http/handler.go

@@ -37,6 +37,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -181,13 +182,13 @@ func (h *Handler) populateValidators() {
 	h.validators["GetIndex"] = queryValidationSpecRequired()
 	h.validators["PostIndex"] = queryValidationSpecRequired()
 	h.validators["DeleteIndex"] = queryValidationSpecRequired()
-	h.validators["GetTranslateData"] = queryValidationSpecRequired("offset")
+	h.validators["GetTranslateData"] = queryValidationSpecRequired("index", "partition")
 	h.validators["PostTranslateKeys"] = queryValidationSpecRequired()
 	h.validators["PostField"] = queryValidationSpecRequired()
 	h.validators["DeleteField"] = queryValidationSpecRequired()
 	h.validators["PostImport"] = queryValidationSpecRequired().Optional("clear", "ignoreKeyCheck")
 	h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote", "clear")
-	h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
+	h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns", "profile")
 	h.validators["GetInfo"] = queryValidationSpecRequired()
 	h.validators["RecalculateCaches"] = queryValidationSpecRequired()
 	h.validators["GetSchema"] = queryValidationSpecRequired()
@@ -203,6 +204,33 @@ func (h *Handler) populateValidators() {
 	h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
 	h.validators["GetNodes"] = queryValidationSpecRequired()
 	h.validators["GetShardMax"] = queryValidationSpecRequired()
+	h.validators["GetTransactions"] = queryValidationSpecRequired()
+	h.validators["GetTransaction"] = queryValidationSpecRequired()
+	h.validators["PostTransaction"] = queryValidationSpecRequired()
+	h.validators["PostFinishTransaction"] = queryValidationSpecRequired()
+}
+
+type contextKeyQuery int
+
+const (
+	contextKeyQueryRequest contextKeyQuery = iota
+	contextKeyQueryError
+)
+
+// addQueryContext puts the results of handler.readQueryRequest into the Context for use by
+// both other middleware and any handlers.
+func (h *Handler) addQueryContext(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pathParts := strings.Split(r.URL.Path, "/")
+		if len(pathParts) > 3 && pathParts[3] == "query" {
+			req, err := h.readQueryRequest(r)
+			ctx := context.WithValue(r.Context(), contextKeyQueryRequest, req)
+			ctx = context.WithValue(ctx, contextKeyQueryError, err)
+			next.ServeHTTP(w, r.WithContext(ctx))
+		} else {
+			next.ServeHTTP(w, r)
+		}
+	})
 }
 
 func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@@ -245,13 +273,24 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
 
 		longQueryTime := h.api.LongQueryTime()
 		if longQueryTime > 0 && dur > longQueryTime {
-			h.logger.Printf("%s %s %v", r.Method, r.URL.String(), dur)
-			statsTags = append(statsTags, "slow_query")
+			queryRequest := r.Context().Value(contextKeyQueryRequest)
+
+			var queryString string
+			if req, ok := queryRequest.(*pilosa.QueryRequest); ok {
+				queryString = req.Query
+			}
+
+			h.logger.Printf("%s %s %v %s", r.Method, r.URL.String(), dur, queryString)
+			statsTags = append(statsTags, "slow:true")
+		} else {
+			statsTags = append(statsTags, "slow:false")
 		}
 
 		pathParts := strings.Split(r.URL.Path, "/")
 		if externalPrefixFlag[pathParts[1]] {
-			statsTags = append(statsTags, "external")
+			statsTags = append(statsTags, "where:external")
+		} else {
+			statsTags = append(statsTags, "where:internal")
 		}
 
 		statsTags = append(statsTags, "useragent:"+r.UserAgent())
@@ -265,7 +304,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
 
 		stats := h.api.StatsWithTags(statsTags)
 		if stats != nil {
-			stats.Timing("http.request", dur, 0.1)
+			stats.Timing(pilosa.MetricHTTPRequest, dur, 0.1)
 		}
 	})
 }
@@ -282,10 +321,15 @@ func newRouter(handler *Handler) *mux.Router {
 	router.Handle("/metrics", promhttp.Handler())
 	router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
 	router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
+	router.HandleFunc("/index", handler.handlePostIndex).Methods("POST").Name("PostIndex")
+	router.HandleFunc("/index/", handler.handlePostIndex).Methods("POST").Name("PostIndex")
 	router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
 	router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
 	router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
 	//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
+	router.HandleFunc("/index/{index}/import-column-attrs", handler.handlePostImportColumnAttrs).Methods("POST").Name("PostImportColumnAttrs")
+	router.HandleFunc("/index/{index}/field", handler.handlePostField).Methods("POST").Name("PostField")
+	router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField")
 	router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
 	router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
 	router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
@@ -296,6 +340,12 @@ func newRouter(handler *Handler) *mux.Router {
 	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
 	router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema")
 	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
+	router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
+	router.HandleFunc("/transaction/", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
+	router.HandleFunc("/transaction/{id}", handler.handleGetTransaction).Methods("GET").Name("GetTransaction")
+	router.HandleFunc("/transaction/{id}", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
+	router.HandleFunc("/transaction/{id}/finish", handler.handlePostFinishTransaction).Methods("POST").Name("PostFinishTransaction")
+	router.HandleFunc("/transactions", handler.handleGetTransactions).Methods("GET").Name("GetTransactions")
 	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
 
 	// /internal endpoints are for internal use only; they may change at any time.
@@ -306,14 +356,17 @@ func newRouter(handler *Handler) *mux.Router {
 	router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
 	router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
 	router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
+	router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
 	router.HandleFunc("/internal/translate/data", handler.handlePostTranslateData).Methods("POST").Name("PostTranslateData")
 	router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys")
+	router.HandleFunc("/internal/translate/ids", handler.handlePostTranslateIDs).Methods("POST").Name("PostTranslateIDs")
 	router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
 	router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE")
 	router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
 	router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
 
 	router.Use(handler.queryArgValidator)
+	router.Use(handler.addQueryContext)
 	router.Use(handler.extractTracing)
 	router.Use(handler.collectStats)
 	return router
@@ -384,16 +437,15 @@ func (r *successResponse) write(w http.ResponseWriter, err error) {
 
 	// Write the response.
 	if statusCode == 0 {
+		w.Header().Set("Content-Type", "application/json")
 		_, err := w.Write(msg)
 		if err != nil {
 			r.h.logger.Printf("error writing response: %v", err)
-			http.Error(w, string(msg), http.StatusInternalServerError)
 			return
 		}
 		_, err = w.Write([]byte("\n"))
 		if err != nil {
 			r.h.logger.Printf("error writing newline after response: %v", err)
-			http.Error(w, string(msg), http.StatusInternalServerError)
 			return
 		}
 	} else {
@@ -427,6 +479,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	w.Header().Set("Content-Type", "application/json")
 	schema := h.api.Schema(r.Context())
 	if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here?
 		h.logger.Printf("write schema response error: %s", err)
@@ -465,6 +518,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
 		Nodes:   h.api.Hosts(r.Context()),
 		LocalID: h.api.Node().ID,
 	}
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(status); err != nil {
 		h.logger.Printf("write status response error: %s", err)
 	}
@@ -476,6 +530,7 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	info := h.api.Info()
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(info); err != nil {
 		h.logger.Printf("write info response error: %s", err)
 	}
@@ -493,9 +548,14 @@ type getStatusResponse struct {
 
 // handlePostQuery handles /query requests.
 func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
-	// Parse incoming request.
-	req, err := h.readQueryRequest(r)
-	if err != nil {
+
+	// Read previouly parsed request from context
+	qreq := r.Context().Value(contextKeyQueryRequest)
+	qerr := r.Context().Value(contextKeyQueryError)
+	req, ok := qreq.(*pilosa.QueryRequest)
+	err, _ := qerr.(error)
+
+	if err != nil || !ok {
 		w.WriteHeader(http.StatusBadRequest)
 		e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err})
 		if e != nil {
@@ -550,6 +610,7 @@ func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(getShardsMaxResponse{
 		Standard: h.api.MaxShards(r.Context()),
 	}); err != nil {
@@ -575,6 +636,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
 	indexName := mux.Vars(r)["index"]
 	for _, idx := range h.api.Schema(r.Context()) {
 		if idx.Name == indexName {
+			w.Header().Set("Content-Type", "application/json")
 			if err := json.NewEncoder(w).Encode(idx); err != nil {
 				h.logger.Printf("write response error: %s", err)
 			}
@@ -682,7 +744,11 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
-	indexName := mux.Vars(r)["index"]
+	indexName, ok := mux.Vars(r)["index"]
+	if !ok {
+		http.Error(w, "index name is required", http.StatusBadRequest)
+		return
+	}
 
 	resp := successResponse{h: h}
 
@@ -729,6 +795,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
 	}
 
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{
 		Attrs: attrs,
 	}); err != nil {
@@ -750,8 +817,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
-	indexName := mux.Vars(r)["index"]
-	fieldName := mux.Vars(r)["field"]
+
+	indexName, ok := mux.Vars(r)["index"]
+	if !ok {
+		http.Error(w, "index name is required", http.StatusBadRequest)
+		return
+	}
+
+	fieldName, ok := mux.Vars(r)["field"]
+	if !ok {
+		http.Error(w, "field name is required", http.StatusBadRequest)
+		return
+	}
 
 	resp := successResponse{h: h}
 
@@ -778,14 +855,37 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 		fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize))
 	case pilosa.FieldTypeInt:
 		if req.Options.Min == nil {
-			min := int64(math.MinInt64)
+			min := pql.NewDecimal(int64(math.MinInt64), 0)
 			req.Options.Min = &min
 		}
 		if req.Options.Max == nil {
-			max := int64(math.MaxInt64)
+			max := pql.NewDecimal(int64(math.MaxInt64), 0)
 			req.Options.Max = &max
 		}
-		fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
+		fos = append(fos, pilosa.OptFieldTypeInt(req.Options.Min.ToInt64(0), req.Options.Max.ToInt64(0)))
+	case pilosa.FieldTypeDecimal:
+		scale := int64(0)
+		if req.Options.Scale != nil {
+			scale = *req.Options.Scale
+		}
+		if req.Options.Min == nil {
+			min := pql.NewDecimal(int64(math.MinInt64), scale)
+			req.Options.Min = &min
+		}
+		if req.Options.Max == nil {
+			max := pql.NewDecimal(int64(math.MaxInt64), scale)
+			req.Options.Max = &max
+		}
+		var minmax []pql.Decimal
+		if req.Options.Min != nil {
+			minmax = []pql.Decimal{
+				*req.Options.Min,
+			}
+			if req.Options.Max != nil {
+				minmax = append(minmax, *req.Options.Max)
+			}
+		}
+		fos = append(fos, pilosa.OptFieldTypeDecimal(scale, minmax...))
 	case pilosa.FieldTypeTime:
 		fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView))
 	case pilosa.FieldTypeMutex:
@@ -798,6 +898,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 			fos = append(fos, pilosa.OptFieldKeys())
 		}
 	}
+	if req.Options.ForeignIndex != nil {
+		fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex))
+	}
 
 	_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
 	if _, ok := err.(pilosa.BadRequestError); ok {
@@ -817,11 +920,13 @@ type fieldOptions struct {
 	Type           string              `json:"type,omitempty"`
 	CacheType      *string             `json:"cacheType,omitempty"`
 	CacheSize      *uint32             `json:"cacheSize,omitempty"`
-	Min            *int64              `json:"min,omitempty"`
-	Max            *int64              `json:"max,omitempty"`
+	Min            *pql.Decimal        `json:"min,omitempty"`
+	Max            *pql.Decimal        `json:"max,omitempty"`
+	Scale          *int64              `json:"scale,omitempty"`
 	TimeQuantum    *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
 	Keys           *bool               `json:"keys,omitempty"`
 	NoStandardView bool                `json:"noStandardView,omitempty"`
+	ForeignIndex   *string             `json:"foreignIndex,omitempty"`
 }
 
 func (o *fieldOptions) validate() error {
@@ -849,6 +954,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type set"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("set field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeInt:
 		if o.CacheType != nil {
@@ -857,6 +964,20 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
+		} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
+			return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
+		}
+	case pilosa.FieldTypeDecimal:
+		if o.Scale == nil {
+			return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument"))
+		} else if o.CacheType != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int"))
+		} else if o.CacheSize != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
+		} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
+			return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeTime:
 		if o.CacheType != nil {
@@ -869,6 +990,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type time"))
 		} else if o.TimeQuantum == nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("time field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeMutex:
 		if o.CacheType == nil {
@@ -883,6 +1006,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("mutex field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeBool:
 		if o.CacheType != nil {
@@ -897,6 +1022,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
 		} else if o.Keys != nil {
 			return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key"))
 		}
 	default:
 		return errors.Errorf("invalid field type: %s", o.Type)
@@ -919,6 +1046,104 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) {
 	resp.write(w, err)
 }
 
+func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+	trnsMap, err := h.api.Transactions(r.Context())
+	if err != nil {
+		switch errors.Cause(err) {
+		case pilosa.ErrNodeNotCoordinator:
+			http.Error(w, err.Error(), http.StatusBadRequest)
+		default:
+			http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError)
+		}
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(trnsMap); err != nil {
+		h.logger.Printf("encoding GetTransactions response: %s", err)
+	}
+}
+
+type TransactionResponse struct {
+	Transaction *pilosa.Transaction `json:"transaction,omitempty"`
+	Error       string              `json:"error,omitempty"`
+}
+
+func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *pilosa.Transaction) {
+	if err != nil {
+		switch errors.Cause(err) {
+		case pilosa.ErrNodeNotCoordinator, pilosa.ErrTransactionExists:
+			w.WriteHeader(http.StatusBadRequest)
+		case pilosa.ErrTransactionExclusive:
+			w.WriteHeader(http.StatusConflict)
+		case pilosa.ErrTransactionNotFound:
+			w.WriteHeader(http.StatusNotFound)
+		default:
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+	}
+
+	var errString string
+	if err != nil {
+		errString = err.Error()
+	}
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(
+		TransactionResponse{Error: errString, Transaction: trns})
+	if err != nil {
+		h.logger.Printf("encoding transaction response: %v", err)
+	}
+
+}
+
+func (h *Handler) handleGetTransaction(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+	id := mux.Vars(r)["id"]
+	trns, err := h.api.GetTransaction(r.Context(), id, false)
+	h.doTransactionResponse(w, err, trns)
+}
+
+func (h *Handler) handlePostTransaction(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+	reqTrns := &pilosa.Transaction{}
+	if err := json.NewDecoder(r.Body).Decode(reqTrns); err != nil || reqTrns.Timeout == 0 {
+		if err == nil {
+			http.Error(w, "timeout is required and cannot be 0", http.StatusBadRequest)
+		} else {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+		}
+		return
+	}
+
+	id, ok := mux.Vars(r)["id"]
+	if !ok {
+		id = reqTrns.ID
+	}
+	trns, err := h.api.StartTransaction(r.Context(), id, reqTrns.Timeout, reqTrns.Exclusive, false)
+
+	h.doTransactionResponse(w, err, trns)
+}
+
+func (h *Handler) handlePostFinishTransaction(w http.ResponseWriter, r *http.Request) {
+	if !validHeaderAcceptJSON(r.Header) {
+		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
+		return
+	}
+	id := mux.Vars(r)["id"]
+	trns, err := h.api.FinishTransaction(r.Context(), id, false)
+	h.doTransactionResponse(w, err, trns)
+}
+
 // handleDeleteRemoteAvailableShard handles DELETE /field/{field}/available-shards/{shardID} request.
 func (h *Handler) handleDeleteRemoteAvailableShard(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
@@ -963,6 +1188,7 @@ func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request
 	}
 
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{
 		Attrs: attrs,
 	}); err != nil {
@@ -1021,9 +1247,20 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
 		return nil, errors.New("invalid shard argument")
 	}
 
+	// Optional profiling
+	profile := false
+	profileString := q.Get("profile")
+	if profileString != "" {
+		profile, err = strconv.ParseBool(q.Get("profile"))
+		if err != nil {
+			return nil, fmt.Errorf("invalid profile argument: '%s' (should be true/false)", profileString)
+		}
+	}
+
 	return &pilosa.QueryRequest{
 		Query:           query,
 		Shards:          shards,
+		Profile:         profile,
 		ColumnAttrs:     q.Get("columnAttrs") == "true",
 		ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
 		ExcludeColumns:  q.Get("excludeColumns") == "true",
@@ -1101,7 +1338,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Unmarshal request based on field type.
-	if field.Type() == pilosa.FieldTypeInt {
+	if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
 		// Field type: Int
 		// Marshal into request object.
 		req := &pilosa.ImportValueRequest{}
@@ -1120,7 +1357,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	} else {
-		// Field type: Set, Time
+		// Field type: set, time, mutex
 		// Marshal into request object.
 		req := &pilosa.ImportRequest{}
 		if err := h.api.Serializer.Unmarshal(body, req); err != nil {
@@ -1142,7 +1379,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	// Marshal response object.
 	buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
 	if e != nil {
-		http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError)
+		http.Error(w, "marshal import response", http.StatusInternalServerError)
 		return
 	}
 
@@ -1211,6 +1448,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Write to response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(nodes); err != nil {
 		h.logger.Printf("json write error: %s", err)
 	}
@@ -1227,6 +1465,7 @@ func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) {
 	nodes := h.api.Hosts(r.Context())
 
 	// Write to response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(nodes); err != nil {
 		h.logger.Printf("json write error: %s", err)
 	}
@@ -1280,6 +1519,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
 	}
 
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{
 		Blocks: blocks,
 	}); err != nil {
@@ -1312,12 +1552,34 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+// handleGetTranslateData handles GET /internal/translate/data requests.
+func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
+	// Read partition parameter.
+	q := r.URL.Query()
+	partition, err := strconv.ParseUint(q.Get("partition"), 10, 32)
+	if err != nil {
+		http.Error(w, "partition required", http.StatusBadRequest)
+		return
+	}
+	// Retrieve partition data from holder.
+	p, err := h.api.TranslateData(r.Context(), q.Get("index"), int(partition))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	// Stream translate partition to response body.
+	if _, err := p.WriteTo(w); err != nil {
+		h.logger.Printf("error streaming translation data: %s", err)
+	}
+}
+
 // handleGetVersion handles /version requests.
 func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
 	err := json.NewEncoder(w).Encode(struct {
 		Version string `json:"version"`
 	}{
@@ -1377,6 +1639,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r
 		return
 	}
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(setCoordinatorResponse{
 		Old: oldNode,
 		New: newNode,
@@ -1419,6 +1682,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht
 	}
 
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(removeNodeResponse{
 		Remove: removeNode,
 	}); err != nil {
@@ -1455,6 +1719,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re
 		}
 	}
 	// Encode response.
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{
 		Info: msg,
 	}); err != nil {
@@ -1486,13 +1751,13 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
 		http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
 		return
 	}
-
 	err := h.api.ClusterMessage(r.Context(), r.Body)
 	if err != nil {
 		// TODO this was the previous behavior, but perhaps not everything is a bad request
 		http.Error(w, err.Error(), http.StatusBadRequest)
 	}
 
+	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil {
 		h.logger.Printf("response encoding error: %s", err)
 	}
@@ -1507,7 +1772,6 @@ func (h *Handler) handlePostTranslateData(w http.ResponseWriter, r *http.Request
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-
 	// Stream all translation data.
 	rd, err := h.api.GetTranslateEntryReader(r.Context(), offsets)
 	if errors.Cause(err) == pilosa.ErrNotImplemented {
@@ -1601,6 +1865,50 @@ func GetHTTPClient(t *tls.Config) *http.Client {
 	return &http.Client{Transport: transport}
 }
 
+// handlePostImportColumnAttrs
+func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) {
+	// Verify that request is only communicating over protobufs.
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
+		return
+	} else if r.Header.Get("Accept") != "application/x-protobuf" {
+		http.Error(w, "Not acceptable", http.StatusNotAcceptable)
+		return
+	}
+
+	opts := []pilosa.ImportOption{}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	req := &pilosa.ImportColumnAttrsRequest{}
+	if err := h.api.Serializer.Unmarshal(body, req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Marshal response object.
+	buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
+	if e != nil {
+		http.Error(w, "marshal import-column-attrs response", http.StatusInternalServerError)
+		return
+	}
+
+	// Write response.
+	_, err = w.Write(buf)
+	if err != nil {
+		h.logger.Printf("writing import-column-attrs response: %v", err)
+	}
+}
+
 // handlPostRoaringImport
 func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
 	// Verify that request is only communicating over protobufs.
@@ -1611,7 +1919,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 		http.Error(w, "Not acceptable", http.StatusNotAcceptable)
 		return
 	}
-
 	indexName := mux.Vars(r)["index"]
 	fieldName := mux.Vars(r)["field"]
 
@@ -1649,7 +1956,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 		http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest)
 		return
 	}
-
 	resp := &pilosa.ImportResponse{}
 	// TODO give meaningful stats for import
 	err = h.api.ImportRoaring(ctx, indexName, fieldName, shard, remote, req)
@@ -1665,7 +1971,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 	// Marshal response object.
 	buf, err := h.api.Serializer.Marshal(resp)
 	if err != nil {
-		http.Error(w, fmt.Sprintf("marshal import response: %v", err), http.StatusInternalServerError)
+		http.Error(w, fmt.Sprintf("marshal import-roaring response: %v", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -1687,11 +1993,34 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	buf, err := h.api.TranslateKeys(r.Body)
+	buf, err := h.api.TranslateKeys(r.Context(), r.Body)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError)
 	}
 
+	// Write response.
+	_, err = w.Write(buf)
+	if err != nil {
+		h.logger.Printf("writing translate keys response: %v", err)
+		return
+	}
+}
+
+func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) {
+	// Verify that request is only communicating over protobufs.
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
+		return
+	} else if r.Header.Get("Accept") != "application/x-protobuf" {
+		http.Error(w, "Not acceptable", http.StatusNotAcceptable)
+		return
+	}
+	buf, err := h.api.TranslateIDs(r.Context(), r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("translate ids: %v", err), http.StatusInternalServerError)
+		return
+	}
+
 	// Write response.
 	_, err = w.Write(buf)
 	if err != nil {

http/handler_internal_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 )
 
 // Test custom UnmarshalJSON for postIndexRequest object
@@ -99,8 +100,8 @@ func stringPtr(s string) *string {
 	return &s
 }
 
-func int64Ptr(i int64) *int64 {
-	return &i
+func decimalPtr(d pql.Decimal) *pql.Decimal {
+	return &d
 }
 
 // Test fieldOption validation.
@@ -135,10 +136,10 @@ func TestFieldOptionValidation(t *testing.T) {
 		// FieldType: Int
 		{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
 		{json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"},
-		{json: `{"options": {"type": "int", "min": 0, "max": 1000}}`, expected: postFieldRequest{Options: fieldOptions{
+		{json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{
 			Type: pilosa.FieldTypeInt,
-			Min:  int64Ptr(0),
-			Max:  int64Ptr(1000),
+			Min:  decimalPtr(pql.NewDecimal(0, 0)),
+			Max:  decimalPtr(pql.NewDecimal(1001, 0)),
 		}}},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"},
 		{json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"},

http/handler_test.go

@@ -15,11 +15,13 @@
 package http_test
 
 import (
+	"encoding/json"
 	"net"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/http"
+	"github.com/pilosa/pilosa/v2/test"
 )
 
 func TestHandlerOptions(t *testing.T) {
@@ -40,3 +42,39 @@ func TestHandlerOptions(t *testing.T) {
 		t.Fatalf("expected error making handler without options, got nil")
 	}
 }
+
+func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
+	tests := []struct {
+		name string
+		tr   *http.TransactionResponse
+	}{
+		{
+			name: "nil transaction",
+			tr:   &http.TransactionResponse{},
+		},
+		{
+			name: "empty transaction",
+			tr:   &http.TransactionResponse{Transaction: &pilosa.Transaction{}},
+		},
+	}
+
+	for _, tst := range tests {
+		t.Run(tst.name, func(t *testing.T) {
+			data, err := json.Marshal(tst.tr)
+			if err != nil {
+				t.Fatalf("marshaling: %v", err)
+			}
+
+			mytr := &http.TransactionResponse{}
+			err = json.Unmarshal(data, mytr)
+			if err != nil {
+				t.Fatalf("unmarshalling: %v", err)
+			}
+
+			if mytr.Error != tst.tr.Error {
+				t.Errorf("errors mismatch:exp/got \n%v\n%v", tst.tr.Error, mytr.Error)
+			}
+			test.CompareTransactions(t, tst.tr.Transaction, mytr.Transaction)
+		})
+	}
+}

index.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
@@ -44,6 +45,9 @@ type Index struct {
 	trackExistence bool
 	existenceFld   *Field
 
+	// Partitions used by translation.
+	partitionN int
+
 	// Fields by name.
 	fields map[string]*Field
 
@@ -52,32 +56,36 @@ type Index struct {
 	// Column attribute storage and cache.
 	columnAttrs AttrStore
 
-	translateStore TranslateStore
-
 	broadcaster broadcaster
 	Stats       stats.StatsClient
 
 	logger        logger.Logger
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
-	// Used for notifying holder when a field is added.
+	// Passed to field for foreign-index lookup.
 	holder *Holder
 
-	// Instantiates new translation stores for fields.
+	// Per-partition translation stores
+	translateStores map[int]TranslateStore
+
+	translationSyncer translationSyncer
+
+	// Instantiates new translation stores
 	OpenTranslateStore OpenTranslateStoreFunc
 }
 
 // NewIndex returns a new instance of Index.
-func NewIndex(path, name string) (*Index, error) {
+func NewIndex(path, name string, partitionN int) (*Index, error) {
 	err := validateName(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "validating name")
 	}
 
 	return &Index{
-		path:   path,
-		name:   name,
-		fields: make(map[string]*Field),
+		path:       path,
+		name:       name,
+		partitionN: partitionN,
+		fields:     make(map[string]*Field),
 
 		newAttrStore: newNopAttrStore,
 		columnAttrs:  nopStore,
@@ -87,6 +95,10 @@ func NewIndex(path, name string) (*Index, error) {
 		logger:         logger.NopLogger,
 		trackExistence: true,
 
+		translateStores: make(map[int]TranslateStore),
+
+		translationSyncer: NopTranslationSyncer,
+
 		OpenTranslateStore: OpenInMemTranslateStore,
 	}, nil
 }
@@ -97,15 +109,22 @@ func (i *Index) Name() string { return i.name }
 // Path returns the path the index was initialized with.
 func (i *Index) Path() string { return i.path }
 
+// TranslateStorePath returns the translation database path for a partition.
+func (i *Index) TranslateStorePath(partitionID int) string {
+	return filepath.Join(i.path, translateStoreDir, strconv.Itoa(partitionID))
+}
+
+// TranslateStore returns the translation store for a given partition.
+func (i *Index) TranslateStore(partitionID int) TranslateStore {
+	return i.translateStores[partitionID]
+}
+
 // Keys returns true if the index uses string keys.
 func (i *Index) Keys() bool { return i.keys }
 
 // ColumnAttrStore returns the storage for column attributes.
 func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs }
 
-// TranslateStore returns the underlying translation store for the index.
-func (i *Index) TranslateStore() TranslateStore { return i.translateStore }
-
 // Options returns all options for this index.
 func (i *Index) Options() IndexOptions {
 	i.mu.RLock()
@@ -149,9 +168,27 @@ func (i *Index) Open() (err error) {
 		return errors.Wrap(err, "opening attrstore")
 	}
 
-	// Instantiate & open translation store.
-	if i.translateStore, err = i.OpenTranslateStore(filepath.Join(i.path, "keys"), i.name, ""); err != nil {
-		return errors.Wrap(err, "opening translate store")
+	i.logger.Debugf("open translate store for index: %s", i.name)
+
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := 0; partitionID < i.partitionN; partitionID++ {
+		partitionID := partitionID
+
+		g.Go(func() error {
+			store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.partitionN)
+			if err != nil {
+				return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			i.translateStores[partitionID] = store
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return err
 	}
 
 	return nil
@@ -197,6 +234,10 @@ fileLoop:
 					return errors.Wrapf(ErrName, "'%s'", fi.Name())
 				}
 
+				// Pass holder through to the field for use in looking
+				// up a foreign index.
+				fld.holder = i.holder
+
 				if err := fld.Open(); err != nil {
 					return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
 				}
@@ -213,7 +254,7 @@ fileLoop:
 
 // openExistenceField gets or creates the existence field and associates it to the index.
 func (i *Index) openExistenceField() error {
-	f, err := i.createFieldIfNotExists(existenceFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
+	f, err := i.createFieldIfNotExists(existenceFieldName, &FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
 	if err != nil {
 		return errors.Wrap(err, "creating existence field")
 	}
@@ -223,7 +264,8 @@ func (i *Index) openExistenceField() error {
 
 // loadMeta reads meta data for the index, if any.
 func (i *Index) loadMeta() error {
-	var pb internal.IndexMeta
+	// TrackExistence is by default true
+	pb := &internal.IndexMeta{TrackExistence: true}
 
 	// Read data from meta file.
 	buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
@@ -232,14 +274,18 @@ func (i *Index) loadMeta() error {
 	} else if err != nil {
 		return errors.Wrap(err, "reading")
 	} else {
-		if err := proto.Unmarshal(buf, &pb); err != nil {
+		if err := proto.Unmarshal(buf, pb); err != nil {
 			return errors.Wrap(err, "unmarshalling")
 		}
 	}
 
 	// Copy metadata fields.
-	i.keys = pb.Keys
-	i.trackExistence = pb.TrackExistence
+	if pb == nil {
+		i.trackExistence = true
+	} else {
+		i.trackExistence = pb.TrackExistence
+	}
+	i.keys = pb.GetKeys()
 
 	return nil
 }
@@ -271,6 +317,14 @@ func (i *Index) Close() error {
 	// Close the attribute store.
 	i.columnAttrs.Close()
 
+	// Close partitioned translation stores.
+	for _, store := range i.translateStores {
+		if err := store.Close(); err != nil {
+			return errors.Wrap(err, "closing translation store")
+		}
+	}
+	i.translateStores = make(map[int]TranslateStore)
+
 	// Close all fields.
 	for _, f := range i.fields {
 		if err := f.Close(); err != nil {
@@ -279,12 +333,6 @@ func (i *Index) Close() error {
 	}
 	i.fields = make(map[string]*Field)
 
-	if i.translateStore != nil {
-		if err := i.translateStore.Close(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -299,10 +347,10 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
 
 	b := roaring.NewBitmap()
 	for _, f := range i.fields {
-		b = b.Union(f.AvailableShards())
+		b.UnionInPlace(f.AvailableShards())
 	}
 
-	i.Stats.Gauge("maxShard", float64(b.Max()), 1.0)
+	i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0)
 	return b
 }
 
@@ -362,13 +410,10 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 		return nil, newConflictError(ErrFieldExists)
 	}
 
-	// Apply functional options.
-	fo := FieldOptions{}
-	for _, opt := range opts {
-		err := opt(&fo)
-		if err != nil {
-			return nil, errors.Wrap(err, "applying option")
-		}
+	// Apply and validate functional options.
+	fo, err := newFieldOptions(opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
 	}
 
 	return i.createField(name, fo)
@@ -389,19 +434,16 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field
 		return f, nil
 	}
 
-	// Apply functional options.
-	fo := FieldOptions{}
-	for _, opt := range opts {
-		err := opt(&fo)
-		if err != nil {
-			return nil, errors.Wrap(err, "applying option")
-		}
+	// Apply and validate functional options.
+	fo, err := newFieldOptions(opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
 	}
 
 	return i.createField(name, fo)
 }
 
-func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createFieldIfNotExists(name string, opt *FieldOptions) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -413,7 +455,7 @@ func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, e
 	return i.createField(name, opt)
 }
 
-func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
 	if name == "" {
 		return nil, errors.New("field name required")
 	} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
@@ -426,17 +468,17 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 		return nil, errors.Wrap(err, "initializing")
 	}
 
+	// Pass holder through to the field for use in looking
+	// up a foreign index.
+	f.holder = i.holder
+
+	f.setOptions(opt)
+
 	// Open field.
 	if err := f.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 
-	// Apply field options.
-	if err := f.applyOptions(opt); err != nil {
-		f.Close()
-		return nil, errors.Wrap(err, "applying options")
-	}
-
 	if err := f.saveMeta(); err != nil {
 		f.Close()
 		return nil, errors.Wrap(err, "saving meta")
@@ -445,9 +487,9 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 	// Add to index's field lookup.
 	i.fields[name] = f
 
-	// Update replication, if needed.
-	if i.holder != nil {
-		go i.holder.refreshTranslateStoreReplicator()
+	// Kick off the field's translation sync process.
+	if err := i.translationSyncer.Reset(); err != nil {
+		return nil, errors.Wrap(err, "resetting translation syncer")
 	}
 
 	return f, nil
@@ -462,7 +504,9 @@ func (i *Index) newField(path, name string) (*Field, error) {
 	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
-	f.snapshotQueue = i.snapshotQueue
+	if i.snapshotQueue != nil {
+		f.snapshotQueue = i.snapshotQueue
+	}
 	f.OpenTranslateStore = i.OpenTranslateStore
 	return f, nil
 }
@@ -503,7 +547,7 @@ func (i *Index) DeleteField(name string) error {
 	// Remove reference.
 	delete(i.fields, name)
 
-	return nil
+	return i.translationSyncer.Reset()
 }
 
 type indexSlice []*Index

index_internal_test.go

@@ -25,7 +25,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, "i")
+	index, err := NewIndex(path, "i", DefaultPartitionN)
 	if err != nil {
 		panic(err)
 	}

index_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
 	"github.com/pkg/errors"
 )
@@ -185,6 +186,30 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 		*/
 	})
+
+	t.Run("WithKeys", func(t *testing.T) {
+		// Don't allow an int field to be created with keys=true
+		t.Run("IntField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrIntFieldWithKeys {
+				t.Fatal("int field cannot be created with keys=true")
+			}
+		})
+
+		// Don't allow a decimal field to be created with keys=true
+		t.Run("DecimalField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -1}, pql.Decimal{Value: 1}), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrDecimalFieldWithKeys {
+				t.Fatal("decimal field cannot be created with keys=true")
+			}
+		})
+	})
 }
 
 // Ensure index can delete a field.
@@ -217,7 +242,7 @@ func TestIndex_InvalidName(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(path, "ABC")
+	index, err := pilosa.NewIndex(path, "ABC", pilosa.DefaultPartitionN)
 	if err == nil {
 		t.Fatalf("should have gotten an error on index name with caps")
 	}

internal/clustertests/docker-compose.yml

@@ -51,6 +51,6 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command:
-      - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
+      - "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
 networks: 
   pilosanet:

internal/private.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package internal;
 
+import "public.proto";
+
 message IndexMeta {
 	bool Keys = 3;
 	bool TrackExistence = 4;
@@ -12,12 +14,16 @@ message FieldOptions {
 	string CacheType = 3;
 	uint32 CacheSize = 4;
 	string TimeQuantum = 5;
-	int64 Min = 9;
-	int64 Max = 10;
+	int64 OldMin = 9;
+	int64 OldMax = 10;
 	bool Keys = 11;
 	bool NoStandardView = 12;
 	int64 Base = 13;
 	uint64 BitDepth = 14;
+	int64 Scale = 15;
+	string ForeignIndex = 16;
+	Decimal Min = 17;
+	Decimal Max = 18;
 }
 
 message ImportResponse {
@@ -89,6 +95,7 @@ message Schema {
 
 message Index {
 	string Name = 1;
+	IndexMeta Options = 5;
 	repeated Field Fields = 4;
 }
 
@@ -161,6 +168,7 @@ message ResizeInstruction {
 	Node Node = 2;
 	Node Coordinator = 3;
 	repeated ResizeSource Sources = 4;
+	repeated TranslationResizeSource TranslationSources = 8;
 	NodeStatus NodeStatus = 7;
 	ClusterStatus ClusterStatus = 6;
 }
@@ -173,6 +181,12 @@ message ResizeSource {
 	uint64 Shard = 5;
 }
 
+message TranslationResizeSource {
+	Node Node = 1;
+	string Index = 2;
+	int32 PartitionID = 3;
+}
+
 message ResizeInstructionComplete {
 	int64 JobID = 1;
 	Node Node = 2;
@@ -193,3 +207,19 @@ message Topology {
 }
 
 message RecalculateCaches {}
+
+message TransactionMessage {
+	string Action = 1;
+	Transaction Transaction = 2;
+}
+
+message Transaction {
+	string ID = 1;
+	bool Active = 2;
+	bool Exclusive = 3;
+	int64 Timeout = 4;
+	int64 Deadline = 5;
+	TransactionStats Stats = 6;
+}
+
+message TransactionStats {}

internal/public.proto

@@ -6,6 +6,12 @@ message Row {
 	repeated uint64 Columns = 1;
 	repeated string Keys = 3;
 	repeated Attr Attrs = 2;
+	bytes Roaring = 4;
+}
+
+message SignedRow {
+	Row Pos = 1;
+	Row Neg = 2;
 }
 
 message RowIdentifiers {
@@ -19,20 +25,43 @@ message Pair {
 	uint64 Count = 2;
 }
 
-message FieldRow{
+message PairField {
+	Pair Pair = 1;
+	string Field = 2;
+}
+
+message PairsField {
+	repeated Pair Pairs = 1;
+	string Field = 2;
+}
+
+message Int64 {
+	int64 Value = 1;
+}
+
+message FieldRow {
 	string Field = 1;
 	uint64 RowID = 2;
 	string RowKey = 3;
+	Int64 Value = 4;
 }
 
 message GroupCount{
 	repeated FieldRow Group = 1;
 	uint64 Count = 2;
+	int64 Sum = 3;
 }
 
 message ValCount {
 	int64 Val = 1;
 	int64 Count = 2;
+	double FloatVal = 3;
+	Decimal DecimalVal = 4;
+}
+
+message Decimal {
+	int64 Value = 1;
+	int64 Scale = 2;
 }
 
 message ColumnAttrSet {
@@ -61,6 +90,7 @@ message QueryRequest {
 	bool Remote = 5;
 	bool ExcludeRowAttrs = 6;
 	bool ExcludeColumns = 7;
+	repeated Row EmbeddedData = 8;
 }
 
 message QueryResponse {
@@ -79,6 +109,8 @@ message QueryResult {
 	repeated uint64 RowIDs = 7;
 	repeated GroupCount GroupCounts = 8;
 	RowIdentifiers RowIdentifiers = 9;
+	SignedRow SignedRow = 10;
+	PairsField PairsField = 11;
 }
 
 message ImportRequest {
@@ -99,6 +131,8 @@ message ImportValueRequest {
 	repeated uint64 ColumnIDs = 5;
 	repeated string ColumnKeys = 7;
 	repeated int64 Values = 6;
+	repeated double FloatValues = 8;
+	repeated string StringValues = 9;
 }
 
 message TranslateKeysRequest {
@@ -111,6 +145,16 @@ message TranslateKeysResponse {
 	repeated uint64 IDs = 3;
 }
 
+message TranslateIDsRequest {
+	string Index = 1;
+	string Field = 2;
+	repeated uint64 IDs = 3;
+}
+
+message TranslateIDsResponse {
+	repeated string Keys = 3;
+}
+
 message ImportRoaringRequestView {
 	string Name = 1;
 	bytes Data = 2;
@@ -119,4 +163,14 @@ message ImportRoaringRequestView {
 message ImportRoaringRequest {
 	bool Clear = 1;
 	repeated ImportRoaringRequestView views = 2;
-}
+	string Action = 3;
+	uint64 Block = 4;
+}
+
+message ImportColumnAttrsRequest {
+	string Index = 1;
+	int64 Shard = 2;
+	string AttrKey = 3;
+	repeated string AttrVals = 4;
+	repeated uint64 ColumnIDs = 5;
+}

license.exceptions

@@ -0,0 +1,12 @@
+# names of files which we do not expect to have our license header
+./apimethod_string.go
+./pql/pql.peg.go
+./internal/private.pb.go
+./internal/public.pb.go
+./lru/lru.go
+./enterprise/enterprise.go
+./roaring/btree.go
+./roaring/btree_test.go
+./proto/pilosa.pb.go
+./logger/filewriter.go
+./logger/filewriter_test.go

logger/filewriter.go

@@ -0,0 +1,108 @@
+// This file is a modified redistribution of reopen (github.com/client9/reopen),
+// which is governed by the following license notice:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2015 Nick Galbreath
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package logger
+
+import (
+	"os"
+	"sync"
+)
+
+// FileWriter that can also be reopened
+type FileWriter struct {
+	mu   sync.Mutex // ensures close / reopen / write are not called at the same time, protects f
+	f    *os.File
+	mode os.FileMode
+	name string
+}
+
+// Close calls the underlyding File.Close()
+func (f *FileWriter) Close() error {
+	f.mu.Lock()
+	err := f.f.Close()
+	f.mu.Unlock()
+	return err
+}
+
+// mutex free version
+func (f *FileWriter) reopen() error {
+	if f.f != nil {
+		f.f.Close()
+		f.f = nil
+	}
+	newf, err := os.OpenFile(f.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, f.mode)
+	if err != nil {
+		f.f = nil
+		return err
+	}
+	f.f = newf
+
+	return nil
+}
+
+// Reopen the file
+func (f *FileWriter) Reopen() error {
+	f.mu.Lock()
+	err := f.reopen()
+	f.mu.Unlock()
+	return err
+}
+
+// Write implements the stander io.Writer interface
+func (f *FileWriter) Write(p []byte) (int, error) {
+	f.mu.Lock()
+	n, err := f.f.Write(p)
+	f.mu.Unlock()
+	return n, err
+}
+
+// Fd returns the file descriptor of the underlying file.
+func (f *FileWriter) Fd() uintptr {
+	f.mu.Lock()
+	n := f.f.Fd()
+	f.mu.Unlock()
+	return n
+}
+
+// NewFileWriter opens a file for appending and writing and can be reopened.
+// it is a ReopenWriteCloser...
+func NewFileWriter(name string) (*FileWriter, error) {
+	// Standard default mode
+	return NewFileWriterMode(name, 0666)
+}
+
+// NewFileWriterMode opens a Reopener file with a specific permission
+func NewFileWriterMode(name string, mode os.FileMode) (*FileWriter, error) {
+	writer := FileWriter{
+		f:    nil,
+		name: name,
+		mode: mode,
+	}
+	err := writer.reopen()
+	if err != nil {
+		return nil, err
+	}
+	return &writer, nil
+}

logger/filewriter_test.go

@@ -0,0 +1,162 @@
+// This file is a modified redistribution of reopen (github.com/client9/reopen),
+// which is governed by the following license notice:
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2015 Nick Galbreath
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package logger
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+// TestReopenAppend -- make sure we always append to an existing file
+//
+// 1. Create a sample file using normal means
+// 2. Open a ioreopen.File
+//    write line 1
+// 3. call Reopen
+//    write line 2
+// 4. close file
+// 5. read file, make sure it contains line0,line1,line2
+//
+func TestReopenAppend(t *testing.T) {
+	// TODO fix
+	var fname = "/tmp/foo"
+
+	// Step 1 -- Create a sample file using normal means
+	forig, err := os.Create(fname)
+	if err != nil {
+		t.Fatalf("Unable to create initial file %s: %s", fname, err)
+	}
+	_, err = forig.Write([]byte("line0\n"))
+	if err != nil {
+		t.Fatalf("Unable to write initial line %s: %s", fname, err)
+	}
+	err = forig.Close()
+	if err != nil {
+		t.Fatalf("Unable to close initial file: %s", err)
+	}
+
+	// Test that making a new File appends
+	f, err := NewFileWriter(fname)
+	if err != nil {
+		t.Fatalf("Unable to create %s", fname)
+	}
+	_, err = f.Write([]byte("line1\n"))
+	if err != nil {
+		t.Errorf("Got write error1: %s", err)
+	}
+
+	// Test that reopen always appends
+	err = f.Reopen()
+	if err != nil {
+		t.Errorf("Got reopen error %s: %s", fname, err)
+	}
+	_, err = f.Write([]byte("line2\n"))
+	if err != nil {
+		t.Errorf("Got write error2 on %s: %s", fname, err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Errorf("Got closing error for %s: %s", fname, err)
+	}
+
+	out, err := ioutil.ReadFile(fname)
+	if err != nil {
+		t.Fatalf("Unable read in final file %s: %s", fname, err)
+	}
+
+	outstr := string(out)
+	if outstr != "line0\nline1\nline2\n" {
+		t.Errorf("Result was %s", outstr)
+	}
+}
+
+// Test that reopen works when Inode is swapped out
+// 1. Create a sample file using normal means
+// 2. Open a ioreopen.File
+//    write line 1
+// 3. call Reopen
+//    write line 2
+// 4. close file
+// 5. read file, make sure it contains line0,line1,line2
+//
+func TestChangeInode(t *testing.T) {
+	// TODO fix
+	var fname = "/tmp/foo"
+
+	// Step 1 -- Create a empty sample file
+	forig, err := os.Create(fname)
+	if err != nil {
+		t.Fatalf("Unable to create initial file %s: %s", fname, err)
+	}
+	err = forig.Close()
+	if err != nil {
+		t.Fatalf("Unable to close initial file: %s", err)
+	}
+
+	// Test that making a new File appends
+	f, err := NewFileWriter(fname)
+	if err != nil {
+		t.Fatalf("Unable to create %s", fname)
+	}
+	_, err = f.Write([]byte("line1\n"))
+	if err != nil {
+		t.Errorf("Got write error1: %s", err)
+	}
+
+	// Now move file
+	err = os.Rename(fname, fname+".orig")
+	if err != nil {
+		t.Errorf("Renaming error: %s", err)
+	}
+	_, err = f.Write([]byte("after1\n"))
+	if err != nil {
+		t.Errorf("Write error: %s", err)
+	}
+
+	// Test that reopen always appends
+	err = f.Reopen()
+	if err != nil {
+		t.Errorf("Got reopen error %s: %s", fname, err)
+	}
+	_, err = f.Write([]byte("line2\n"))
+	if err != nil {
+		t.Errorf("Got write error2 on %s: %s", fname, err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Errorf("Got closing error for %s: %s", fname, err)
+	}
+
+	out, err := ioutil.ReadFile(fname)
+	if err != nil {
+		t.Fatalf("Unable read in final file %s: %s", fname, err)
+	}
+	outstr := string(out)
+	if outstr != "line2\n" {
+		t.Errorf("Result was %s", outstr)
+	}
+}

logger/logger.go

@@ -105,3 +105,27 @@ func (cl *CaptureLogger) Printf(format string, v ...interface{}) {
 func (cl *CaptureLogger) Debugf(format string, v ...interface{}) {
 	cl.Debugs = append(cl.Debugs, fmt.Sprintf(format, v...))
 }
+
+// Logfer is a thing that has only a Logf() method, like for instance,
+// testing.T or testing.B.
+type Logfer interface {
+	Logf(format string, v ...interface{})
+}
+
+// LogfLogger is a logger that wraps something that has a Logf interface
+// and makes it act like our logger.
+type LogfLogger struct {
+	wrapped Logfer
+}
+
+func (ll *LogfLogger) Printf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func (ll *LogfLogger) Debugf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func NewLogfLogger(l Logfer) *LogfLogger {
+	return &LogfLogger{wrapped: l}
+}

metrics.go

@@ -0,0 +1,61 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+const (
+	MetricCreateIndex                     = "create_index_total"
+	MetricDeleteIndex                     = "delete_index_total"
+	MetricCreateField                     = "create_field_total"
+	MetricDeleteField                     = "delete_field_total"
+	MetricDeleteAvailableShard            = "delete_available_shard_total"
+	MetricRecalculateCache                = "recalculate_cache_total"
+	MetricInvalidateCache                 = "invalidate_cache_total"
+	MetricRankCacheLength                 = "rank_cache_length"
+	MetricCacheThresholdReached           = "cache_threshold_reached_total"
+	MetricRow                             = "query_row_total"
+	MetricRowBSI                          = "query_row_bsi_total"
+	MetricSetRowAttrs                     = "query_setrowattrs_total"
+	MetricSetColumnAttrs                  = "query_setcolumnattrs_total"
+	MetricSetBit                          = "set_bit_total"
+	MetricClearBit                        = "clear_bit_total"
+	MetricImportingN                      = "importing_total"
+	MetricImportedN                       = "imported_total"
+	MetricClearingN                       = "clearing_total"
+	MetricClearedN                        = "cleared_total"
+	MetricSnapshotDurationSeconds         = "snapshot_duration_seconds"
+	MetricBlockRepair                     = "block_repair_total"
+	MetricSyncFieldDurationSeconds        = "sync_field_duration_seconds"
+	MetricSyncIndexDurationSeconds        = "sync_index_duration_seconds"
+	MetricColumnAttrStoreBlocks           = "column_attr_store_blocks_total"
+	MetricColumnAttrDiff                  = "column_attr_diff_total"
+	MetricRowAttrStoreBlocks              = "row_attr_store_blocks_total"
+	MetricRowAttrDiff                     = "row_attr_diff_total"
+	MetricHTTPRequest                     = "http_request_duration_seconds"
+	MetricGRPCUnaryQueryDurationSeconds   = "grpc_request_pql_unary_query_duration_seconds"
+	MetricGRPCUnaryFormatDurationSeconds  = "grpc_request_pql_unary_format_duration_seconds"
+	MetricGRPCStreamQueryDurationSeconds  = "grpc_request_pql_stream_query_duration_seconds"
+	MetricGRPCStreamFormatDurationSeconds = "grpc_request_pql_stream_format_duration_seconds"
+	MetricMaxShard                        = "maximum_shard"
+	MetricAntiEntropy                     = "antientropy_total"
+	MetricAntiEntropyDurationSeconds      = "antientropy_duration_seconds"
+	MetricGarbageCollection               = "garbage_collection_total"
+	MetricGoroutines                      = "goroutines"
+	MetricOpenFiles                       = "open_files"
+	MetricHeapAlloc                       = "heap_alloc"
+	MetricHeapInuse                       = "heap_inuse"
+	MetricStackInuse                      = "stack_inuse"
+	MetricMallocs                         = "mallocs"
+	MetricFrees                           = "frees"
+)

mmap_test.go

@@ -0,0 +1,102 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/syswrap"
+)
+
+type cv struct {
+	cols []uint64
+	vals []int64
+}
+
+func forceSnapshotsCheckMapping(t *testing.T) {
+	depth := uint(6)
+	f := mustOpenBSIFragment("i", "f", viewStandard, 0)
+	f.Logger = logger.NewLogfLogger(t)
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	err := f.Snapshot()
+	if err != nil {
+		t.Fatalf("initial snapshot error: %v", err)
+	}
+
+	values := make([]cv, 1024)
+	for i := range values {
+		cols := make([]uint64, 128)
+		vals := make([]int64, 128)
+		for j := range cols {
+			// pick values in the first 16 cols of each of the 16
+			// shards in a default shardwidth, so each set will
+			// probably change some values from the previous one.
+			cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16))
+			vals[j] = int64(rand.Int63n(1 << depth))
+		}
+		values[i] = cv{cols, vals}
+	}
+
+	// modify the original bitmap, until it causes a snapshot, which
+	// then invalidates the other map...
+	for i := 0; i < 32; i++ {
+		cv := values[i%len(values)]
+		// periodically force gc, so if we have a small pool of maps
+		// we'll go in and out of mapping mode
+		if i%5 == 0 {
+			runtime.GC()
+		}
+		err := f.importValue(cv.cols, cv.vals, depth, (i%3 == 1))
+		if err != nil {
+			t.Fatalf("importValue[%d]: %v", i, err)
+		}
+		err = f.Snapshot()
+		if err != nil {
+			t.Fatalf("snapshot[%d]: %v", i, err)
+		}
+	}
+}
+
+// This test should basically never fail, but it might if you were running
+// out of available mmaps. Which you can fake up by adding '&& false' to the test
+// in newGeneration in generation.go. So this is probably useless but it's
+// a failure mode we've been bitten by once...
+func TestMmapBehavior(t *testing.T) {
+	var changed bool
+	var original uint64
+	defer func() {
+		syswrap.SetMaxMapCount(original)
+	}()
+
+	for _, mmapMaxVal := range []uint64{0, 3} {
+		prev := syswrap.SetMaxMapCount(mmapMaxVal)
+		if !changed {
+			original = prev
+			changed = true
+		}
+		t.Run(fmt.Sprintf("maps%d", mmapMaxVal), func(t *testing.T) {
+			forceSnapshotsCheckMapping(t)
+		})
+	}
+}

mock/translator.go

@@ -16,6 +16,7 @@ package mock
 
 import (
 	"context"
+	"io"
 
 	"github.com/pilosa/pilosa/v2"
 )
@@ -25,6 +26,7 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil)
 type TranslateStore struct {
 	CloseFunc         func() error
 	MaxIDFunc         func() (uint64, error)
+	PartitionIDFunc   func() int
 	ReadOnlyFunc      func() bool
 	SetReadOnlyFunc   func(v bool)
 	TranslateKeyFunc  func(key string) (uint64, error)
@@ -43,6 +45,10 @@ func (s *TranslateStore) MaxID() (uint64, error) {
 	return s.MaxIDFunc()
 }
 
+func (s *TranslateStore) PartitionID() int {
+	return s.PartitionIDFunc()
+}
+
 func (s *TranslateStore) ReadOnly() bool {
 	return s.ReadOnlyFunc()
 }
@@ -75,6 +81,16 @@ func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa
 	return s.EntryReaderFunc(ctx, offset)
 }
 
+// TODO: implement this
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	return 0, nil
+}
+
+// TODO: implement this
+func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) {
+	return 0, nil
+}
+
 var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil)
 
 type TranslateEntryReader struct {

pilosa.go

@@ -29,6 +29,8 @@ var (
 	ErrIndexExists   = errors.New("index already exists")
 	ErrIndexNotFound = errors.New("index not found")
 
+	ErrForeignIndexNotFound = errors.New("foreign index not found")
+
 	// ErrFieldRequired is returned when no field is specified.
 	ErrFieldRequired = errors.New("field required")
 	ErrFieldExists   = errors.New("field already exists")
@@ -44,11 +46,12 @@ var (
 	ErrBSIGroupValueTooHigh     = errors.New("bsigroup value too high")
 	ErrInvalidRangeOperation    = errors.New("invalid range operation")
 	ErrInvalidBetweenValue      = errors.New("invalid value for between operation")
+	ErrDecimalOutOfRange        = errors.New("decimal value out of range")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")
 
-	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters")
+	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
 	ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
 
 	// ErrFragmentNotFound is returned when a fragment does not exist.
@@ -65,10 +68,14 @@ var (
 	ErrNodeIDNotExists    = errors.New("node with provided ID does not exist")
 	ErrNodeNotCoordinator = errors.New("node is not the coordinator")
 	ErrResizeNotRunning   = errors.New("no resize job currently running")
+	ErrResizeNoReplicas   = errors.New("not enough data to perform resize (replica factor may need to be increased)")
 
 	ErrNotImplemented            = errors.New("not implemented")
 	ErrFieldsArgumentRequired    = errors.New("fields argument required")
 	ErrExpectedFieldListArgument = errors.New("expected field list argument")
+
+	ErrIntFieldWithKeys     = errors.New("int field cannot be created with 'keys=true' option")
+	ErrDecimalFieldWithKeys = errors.New("decimal field cannot be created with 'keys=true' option")
 )
 
 // apiMethodNotAllowedError wraps an error value indicating that a particular
@@ -118,7 +125,7 @@ func newNotFoundError(err error) NotFoundError {
 }
 
 // Regular expression to validate index and field names.
-var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
+var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
 
 // ColumnAttrSet represents a set of attributes for a vertical column in an index.
 // Can have a set of attributes attached to it.

pilosa_internal_test.go

@@ -21,7 +21,7 @@ import (
 func TestValidateName(t *testing.T) {
 	names := []string{
 		"a", "ab", "ab1", "b-c", "d_e", "exists",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"longbutnottoolongaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa12345689012345689012345678901234567890",
 	}
 	for _, name := range names {
 		if validateName(name) != nil {
@@ -33,7 +33,7 @@ func TestValidateName(t *testing.T) {
 func TestValidateNameInvalid(t *testing.T) {
 	names := []string{
 		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
+		"long123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
 	}
 	for _, name := range names {
 		if validateName(name) == nil {

pql/ast.go

@@ -17,10 +17,13 @@ package pql
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/molecula/ext"
 )
 
 // Query represents a PQL query.
@@ -59,7 +62,7 @@ func (q *Query) lastCallStackElem() *callStackElem {
 
 func (q *Query) addPosNum(key, value string) {
 	q.addField(key)
-	q.addNumVal(value)
+	q.addNumVal(value, false)
 }
 
 func (q *Query) addPosStr(key, value string) {
@@ -84,19 +87,26 @@ func (q *Query) endConditional() {
 	if len(q.conditional) != 5 {
 		panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional))
 	}
-	low, _ := strconv.ParseInt(q.conditional[0], 10, 64)
+	low := parseNum(q.conditional[0], false)
 	field := q.conditional[2]
-	high, _ := strconv.ParseInt(q.conditional[4], 10, 64)
+	high := parseNum(q.conditional[4], false)
 
-	if q.conditional[1] == "<" {
-		low++
-	}
-	if q.conditional[3] == "<" {
-		high--
+	var op Token
+	switch q.conditional[1] + q.conditional[3] {
+	case "<<":
+		op = BTWN_LT_LT
+	case "<=<":
+		op = BTWN_LTE_LT
+	case "<<=":
+		op = BTWN_LT_LTE
+	case "<=<=":
+		op = BETWEEN
+	default:
+		panic(fmt.Sprintf("impossible conditional ops: '%s' and '%s'", q.conditional[1], q.conditional[3]))
 	}
 
 	elem := q.lastCallStackElem()
-	elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
+	elem.call.Args[field] = &Condition{Op: op, Value: []interface{}{low, high}}
 
 	q.conditional = nil
 }
@@ -147,21 +157,12 @@ func (q *Query) addVal(val interface{}) {
 	elem.lastCond = ILLEGAL
 }
 
-func (q *Query) addNumVal(val string) {
+func (q *Query) addNumVal(val string, asFloat bool) {
 	elem := q.lastCallStackElem()
 	if elem == nil || elem.lastField == "" {
 		panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
 	}
-	var ival interface{}
-	var err error
-	if strings.Contains(val, ".") {
-		ival, err = strconv.ParseFloat(val, 64)
-	} else {
-		ival, err = strconv.ParseInt(val, 10, 64)
-	}
-	if err != nil {
-		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
-	}
+	ival := parseNum(val, asFloat)
 	if elem.inList {
 		if elem.lastCond != ILLEGAL {
 			list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{})
@@ -259,11 +260,261 @@ type callStackElem struct {
 	inList    bool
 }
 
-// Call represents a function call in the AST.
+// Some call types may require special handling, which needs to occur
+// before distributing processing to individual shards.
+type CallType byte
+
+const (
+	// Normal calls can be executed per shard.
+	PrecallNone = CallType(iota)
+	// PreCallGlobal indicates a call which must be run globally *before*
+	// distributing the call to other shards. Example: A Distinct query,
+	// where every shard could potentially produce results for any shard,
+	// so you have to produce the results up front.
+	PrecallGlobal
+	// PreCallPerNode indicates a call which needs to be run per-shard
+	// in a way that lets it be done on each shard, but where it should
+	// be done prior to spawning per-shard goroutines. Example:
+	// A cross-index query, where each local shard may or may not need
+	// to get data from a remote node, but batches of shards can
+	// probably be gotten from the same remote node.
+	PrecallPerNode
+)
+
+// Call represents a function call in the AST. The Precomputed field
+// is used by the executor to handle non-standard call types; it does
+// these by actually executing them separately, then replacing them
+// in the call tree with a new call using the special precomputed
+// type, with the Precomputed field set to a map from shards to results.
 type Call struct {
-	Name     string
-	Args     map[string]interface{}
-	Children []*Call
+	Name        string
+	Args        map[string]interface{}
+	Children    []*Call
+	Type        CallType
+	Precomputed map[uint64]interface{}
+}
+
+// callInfo defines the arguments allowed for a particular PQL call, and
+// possibly things about its semantics. If allowUnknown is true, unfamiliar
+// non-reserved names are allowed on the assumption that they're field names.
+// Otherwise, only those names explicitly listed are allowed. Reserved args
+// (those with a leading underscore) are never allowed unless explicitly
+// present.
+//
+// The prototypes map maps from argument names to a value. If the value is
+// non-nil, the argument will be checked for type-matching. So, for instance,
+// `x: 10` would indicate that x must be an int.
+type callInfo struct {
+	allowUnknown bool
+	prototypes   map[string]interface{}
+	callType     CallType
+}
+
+// We want to be able to accept either a string or int64 for
+// field names. Special-case type:
+type stringOrInt64Type struct{}
+
+var stringOrInt64 stringOrInt64Type
+
+var allowUnderField = callInfo{
+	allowUnknown: true,
+	prototypes: map[string]interface{}{
+		"_field": "",
+	},
+}
+
+var allowField = callInfo{
+	allowUnknown: false,
+	prototypes: map[string]interface{}{
+		"field": "",
+	},
+}
+
+var callInfoByFunc = map[string]callInfo{
+	// the easy cases: things that take arbitrary inputs, because they're
+	// taking field=value cases
+	"Bitmap": {allowUnknown: true},
+	"Count":  {allowUnknown: true},
+	"Row":    {allowUnknown: true},
+	"Range":  {allowUnknown: true},
+
+	"Distinct": {allowUnknown: true},
+
+	// allow only "field=X" cases with string field names
+	"Max": allowField,
+	"Min": allowField,
+	"Sum": allowField,
+
+	// only take other calls, should never have "args"
+	"Difference": {allowUnknown: false},
+	"Intersect":  {allowUnknown: false},
+	"Not":        {allowUnknown: false},
+	"All": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"limit":  int64(0),
+			"offset": int64(0),
+		},
+	},
+	"ClearRow": {allowUnknown: true},
+	"Store":    {allowUnknown: true},
+	"MinRow":   allowField,
+	"MaxRow":   allowField,
+	"Rows": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"_field":   "",
+			"field":    "",
+			"limit":    int64(0),
+			"column":   nil,
+			"previous": nil,
+			"from":     nil,
+			"to":       nil,
+		},
+	},
+	"Shift": {allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"n": int64(0),
+		},
+	},
+	"Union": {allowUnknown: false},
+	"Xor":   {allowUnknown: false},
+
+	// things that take _field
+	"TopN": allowUnderField,
+	// special cases:
+	"Clear": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"GroupBy": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"filter":    nil,
+			"limit":     int64(0),
+			"previous":  nil,
+			"aggregate": nil,
+			"having":    nil,
+		},
+	},
+	"Options": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"excludeRowAttrs": true,
+			"excludeColumns":  true,
+			"columnAttrs":     true,
+			"shards":          nil,
+		},
+	},
+	"Set": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col":       stringOrInt64,
+			"_timestamp": "",
+		},
+	},
+	"Precomputed": {
+		allowUnknown: true,
+	},
+	"SetBit": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"SetRowAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_row":   stringOrInt64,
+		},
+	},
+	"SetColumnAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_col":   stringOrInt64,
+		},
+	},
+	"IncludesColumn": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"column": stringOrInt64,
+		},
+	},
+}
+
+// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
+// arg validation.
+func RegisterPluginFuncs(ops []ext.BitmapOp) {
+	for _, op := range ops {
+		// ignore overlap for now. This should change.
+		if _, ok := callInfoByFunc[op.Name]; ok {
+			continue
+		}
+		ci := callInfo{allowUnknown: true}
+		if len(op.Reserved) > 0 {
+			// mark these as valid/known reserved words
+			ci.prototypes = make(map[string]interface{})
+			for _, res := range op.Reserved {
+				ci.prototypes[res] = nil
+			}
+		}
+		t := op.Func.BitmapOpType()
+		if t.Precall == ext.OpPrecallGlobal {
+			ci.callType = PrecallGlobal
+		}
+		callInfoByFunc[op.Name] = ci
+	}
+}
+
+// CheckCallInfo tries to validate that arguments are correct and valid for the
+// given call. It does not guarantee checking all possible errors; for instance,
+// if an argument is a field name, CheckCallInfo can't validate that the field
+// exists. It also updates with information like whether the call is expected
+// to require precalling.
+func (c *Call) CheckCallInfo() error {
+	valid, ok := callInfoByFunc[c.Name]
+	if !ok {
+		return fmt.Errorf("no arg validation for '%s'", c.Name)
+	}
+	c.Type = valid.callType
+	for k, v := range c.Args {
+		acceptable, ok := valid.prototypes[k]
+		if !ok && !valid.allowUnknown {
+			return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
+		}
+		if !ok && strings.HasPrefix(k, "_") {
+			return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
+		}
+		if acceptable == nil {
+			continue
+		}
+		// if the types are identical, that's fine
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
+			continue
+		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
+			switch v.(type) {
+			case string, int64:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
+					c.String(), k, v)
+			}
+		}
+		return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
+			c.String(), k, v, acceptable)
+	}
+	// call-specific checking
+	for _, child := range c.Children {
+		if err := child.CheckCallInfo(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // FieldArg determines which key-value pair contains the field and rowID,
@@ -283,13 +534,36 @@ func IsReservedArg(name string) bool {
 		return true
 	}
 	switch name {
-	case "from", "to":
+	case "from", "to", "index":
 		return true
 	default:
 		return false
 	}
 }
 
+// CallIndex handles guessing whether we've been asked to apply this to a
+// different index. An empty string means "no".
+func (c *Call) CallIndex() string {
+	if index, ok := c.Args["_index"]; ok {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	if index, ok := c.Args["index"]; ok && index != "" {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	return ""
+}
+
+// Arg is for reading the value at key from call.Args.
+// If the key is not in Call.Args, the value of the returned bool will be false.
+func (c *Call) Arg(key string) (interface{}, bool) {
+	v, ok := c.Args[key]
+	return v, ok
+}
+
 // BoolArg is for reading the value at key from call.Args as a bool. If the
 // key is not in Call.Args, the value of the returned bool will be false, and
 // the error will be nil. The value is assumed to be a bool. An error is
@@ -416,6 +690,11 @@ func (c *Call) Clone() *Call {
 			other.Children[i] = c.Children[i].Clone()
 		}
 	}
+	// @seebs "...it should be safe,
+	// because nothing should be writing to Precomputed
+	// once it's gotten created in the first place."
+	other.Precomputed = c.Precomputed
+
 	return other
 }
 
@@ -455,7 +734,7 @@ func (c *Call) String() string {
 		// the equal sign in the string representation.
 		switch v := c.Args[key].(type) {
 		case *Condition:
-			fmt.Fprintf(&buf, "%v %s", key, v.String())
+			fmt.Fprintf(&buf, "%s", v.StringWithSubj(key))
 		default:
 			fmt.Fprintf(&buf, "%v=%s", key, formatValue(v))
 		}
@@ -477,6 +756,36 @@ func (c *Call) HasConditionArg() bool {
 	return false
 }
 
+// TranslateInfo returns the relevant translation fields.
+func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) {
+	switch c.Name {
+	case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow":
+		// Positional args in new PQL syntax require special handling here.
+		fieldName, _ = c.FieldArg()
+		return "_" + columnLabel, fieldName, fieldName
+	case "SetRowAttrs":
+		// Positional args in new PQL syntax require special handling here.
+		return "", "_" + rowLabel, c.ArgString("_field")
+	case "Rows":
+		return "column", "previous", c.ArgString("_field")
+	case "IncludesColumn":
+		return "column", "", ""
+	case "GroupBy":
+		return "", "", ""
+	default:
+		return "col", "row", c.ArgString("_field")
+	}
+}
+
+func (c *Call) ArgString(key string) string {
+	value, ok := c.Args[key]
+	if !ok {
+		return ""
+	}
+	s, _ := value.(string)
+	return s
+}
+
 // Condition represents an operation & value.
 // When used in an argument map it represents a binary expression.
 type Condition struct {
@@ -486,13 +795,85 @@ type Condition struct {
 
 // String returns the string representation of the condition.
 func (cond *Condition) String() string {
-	return fmt.Sprintf("%s %s", cond.Op.String(), formatValue(cond.Value))
+	return fmt.Sprintf("%s%s", cond.Op.String(), formatValue(cond.Value))
 }
 
-// IntSliceValue reads cond.Value as a slice of uint64.
-// If the value is a slice of uint64 it will convert
-// it to []int64. Otherwise, if it is not a []int64 it will return an error.
-func (cond *Condition) IntSliceValue() ([]int64, error) {
+// StringWithSubj returns the string representation of the condition
+// including the provided subject.
+func (cond *Condition) StringWithSubj(subj string) string {
+	switch cond.Op {
+	case EQ, NEQ, LT, LTE, GT, GTE:
+		return fmt.Sprintf("%s%s", subj, cond.String())
+	case BETWEEN, BTWN_LT_LTE, BTWN_LTE_LT, BTWN_LT_LT:
+		val, ok := cond.StringSliceValue()
+		if !ok || len(val) < 2 {
+			return ""
+		}
+		if cond.Op == BETWEEN {
+			return fmt.Sprintf("%s<=%s<=%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LTE {
+			return fmt.Sprintf("%s<%s<=%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LTE_LT {
+			return fmt.Sprintf("%s<=%s<%s", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LT {
+			return fmt.Sprintf("%s<%s<%s", val[0], subj, val[1])
+		}
+	}
+	return ""
+}
+
+func (cond *Condition) Uint64Value() (uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		if tval >= 0 {
+			return uint64(tval), true
+		}
+	case uint64:
+		return tval, true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Uint64SliceValue() ([]uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]uint64, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = uint64(tv)
+			case uint64:
+				ret[i] = tv
+			default:
+				return nil, false
+			}
+		}
+		return ret, true
+	}
+
+	return nil, false
+}
+
+func (cond *Condition) Int64Value() (int64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		return tval, true
+	case uint64:
+		// TODO: consider overflow?
+		return int64(tval), true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Int64SliceValue() ([]int64, bool) {
 	val := cond.Value
 
 	switch tval := val.(type) {
@@ -505,17 +886,48 @@ func (cond *Condition) IntSliceValue() ([]int64, error) {
 			case uint64:
 				ret[i] = int64(tv)
 			default:
-				return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv)
+				return nil, false
 			}
 		}
-		return ret, nil
-	default:
-		return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval)
+		return ret, true
 	}
+
+	return nil, false
+}
+
+// StringSliceValue returns the value(s) of the conditional
+// as a slice of strings. For example, if cond.Value is
+// []int64{-10,20}, this will return []string{"-10","20"}.
+// It also returns a bool indicating that the conversion
+// succeeded.
+func (cond *Condition) StringSliceValue() ([]string, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]string, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = strconv.FormatInt(tv, 10)
+			case uint64:
+				ret[i] = strconv.FormatUint(tv, 10)
+			case Decimal:
+				ret[i] = tv.String()
+			default:
+				return nil, false
+			}
+		}
+		return ret, true
+	}
+
+	return nil, false
 }
 
 func formatValue(v interface{}) string {
 	switch v := v.(type) {
+	case nil:
+		return "null"
 	case string:
 		return fmt.Sprintf("%q", v)
 	case []interface{}:
@@ -560,3 +972,21 @@ func joinUint64Slice(a []uint64) string {
 	}
 	return "[" + strings.Join(other, ",") + "]"
 }
+
+func parseNum(val string, asFloat bool) interface{} {
+	var ival interface{}
+	var err error
+	if strings.Contains(val, ".") {
+		if asFloat {
+			ival, err = strconv.ParseFloat(val, 64)
+		} else {
+			ival, err = ParseDecimal(val)
+		}
+	} else {
+		ival, err = strconv.ParseInt(val, 10, 64)
+	}
+	if err != nil {
+		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
+	}
+	return ival
+}

pql/ast_test.go

@@ -15,7 +15,6 @@
 package pql_test
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
@@ -37,33 +36,32 @@ func TestCall_String(t *testing.T) {
 				"field0": &pql.Condition{Op: pql.GTE, Value: 10},
 			},
 		}
-		if s := c.String(); s != `Range(field0 >= 10, other="f")` {
+		if s := c.String(); s != `Range(field0>=10, other="f")` {
 			t.Fatalf("unexpected string: %s", s)
 		}
 	})
 }
 
-// Ensure condition can handle values for BETWEEN operator.
-func TestCondition_Value(t *testing.T) {
-	t.Run("Between Values", func(t *testing.T) {
-		for _, tt := range []struct {
-			val []interface{}
-			exp []int64
-		}{
-			{[]interface{}{int64(4), int64(8)}, []int64{4, 8}},
-			{[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}},
-			{[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}},
-		} {
-			c := &pql.Condition{
-				Op:    pql.BETWEEN,
-				Value: tt.val,
-			}
-			v, err := c.IntSliceValue()
-			if err != nil {
-				t.Fatal(err)
-			} else if !reflect.DeepEqual(v, tt.exp) {
-				t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v)
-			}
+// Ensure condition string with subject is correct.
+func TestCondition_StringWithSubj(t *testing.T) {
+	subj := "subj"
+	for _, tt := range []struct {
+		op  pql.Token
+		val interface{}
+		exp string
+	}{
+		{pql.BETWEEN, []interface{}{int64(4), int64(8)}, "4<=subj<=8"},
+		{pql.BETWEEN, []interface{}{uint64(5), uint64(9)}, "5<=subj<=9"},
+		{pql.BETWEEN, []interface{}{pql.Decimal{Value: -401, Scale: 2}, pql.Decimal{Value: 802, Scale: 1}}, "-4.01<=subj<=80.2"},
+		{pql.EQ, nil, "subj==null"},
+		{pql.NEQ, nil, "subj!=null"},
+	} {
+		c := &pql.Condition{
+			Op:    tt.op,
+			Value: tt.val,
 		}
-	})
+		if sws := c.StringWithSubj(subj); sws != tt.exp {
+			t.Fatalf("invalid between string. expected: %s, got %s", tt.exp, sws)
+		}
+	}
 }

pql/decimal.go

@@ -0,0 +1,547 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pql
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// pow10 is a map used to avoid the float64 required by math.Pow10()
+var pow10 = map[int64]int64{
+	0:  1,
+	1:  10,
+	2:  100,
+	3:  1000,
+	4:  10000,
+	5:  100000,
+	6:  1000000,
+	7:  10000000,
+	8:  100000000,
+	9:  1000000000,
+	10: 10000000000,
+	11: 100000000000,
+	12: 1000000000000,
+	13: 10000000000000,
+	14: 100000000000000,
+	15: 1000000000000000,
+	16: 10000000000000000,
+	17: 100000000000000000,
+	18: 1000000000000000000,
+	//19: 10000000000000000000,
+}
+
+// Pow10 is a function which can be used in place of math.Pow10()
+// to avoid the float64 logic. Note that only powers 0-18 are
+// currently supported; anything else will return 0, which is
+// probably going to result in incorrect values.
+func Pow10(p int64) int64 {
+	return pow10[p]
+}
+
+// Decimal represents a decimal value; the intention
+// is to avoid relying on float64, and the primary
+// purpose is to have a predictable way to encode such
+// values used in query strings.
+// Scale is the number of digits to the right of the
+// decimal point.
+// Precision is currently not considered; precision, for
+// our purposes is implied to be the complete, known value.
+type Decimal struct {
+	Value int64
+	Scale int64
+}
+
+// NewDecimal returns a Decimal based on the provided arguments.
+func NewDecimal(value, scale int64) Decimal {
+	return Decimal{
+		Value: value,
+		Scale: scale,
+	}
+}
+
+// MinMax returns the minimum and maximum values
+// supported by the provided scale.
+func MinMax(scale int64) (Decimal, Decimal) {
+	min := NewDecimal(math.MinInt64, scale)
+	max := NewDecimal(math.MaxInt64, scale)
+	return min, max
+}
+
+// LessThan returns true if d < d2.
+func (d Decimal) LessThan(d2 Decimal) bool {
+	return d.lessThan(d2, false)
+}
+
+// LessThanOrEqualTo returns true if d <= d2.
+func (d Decimal) LessThanOrEqualTo(d2 Decimal) bool {
+	return d.lessThan(d2, true)
+}
+
+// GreaterThan returns true if d > d2.
+func (d Decimal) GreaterThan(d2 Decimal) bool {
+	return d.greaterThan(d2, false)
+}
+
+// GreaterThanOrEqualTo returns true if d >= d2.
+func (d Decimal) GreaterThanOrEqualTo(d2 Decimal) bool {
+	return d.greaterThan(d2, true)
+}
+
+// EqualTo returns true if d == d2.
+func (d Decimal) EqualTo(d2 Decimal) bool {
+	if d.Scale == d2.Scale {
+		return d.Value == d2.Value
+	}
+
+	quotientD := quotient(d)
+	quotientD2 := quotient(d2)
+	if quotientD != quotientD2 {
+		return false
+	}
+	remainderD, remainderD2 := remainder(d), remainder(d2)
+	if d.Scale < d2.Scale {
+		scaleDiff := d2.Scale - d.Scale
+		return (remainderD * pow10[scaleDiff]) == remainderD2
+	}
+	scaleDiff := d.Scale - d2.Scale
+	return remainderD == (remainderD2 * pow10[scaleDiff])
+}
+
+func (d Decimal) lessThan(d2 Decimal, eq bool) bool {
+	if d.Scale == d2.Scale {
+		if eq {
+			return d.Value <= d2.Value
+		}
+		return d.Value < d2.Value
+	}
+
+	quotientD, quotientD2 := quotient(d), quotient(d2)
+	if quotientD < quotientD2 {
+		return true
+	} else if quotientD == quotientD2 {
+		remainderD, remainderD2 := remainder(d), remainder(d2)
+		if d.Scale < d2.Scale {
+			scaleDiff := d2.Scale - d.Scale
+			if eq {
+				return (remainderD * pow10[scaleDiff]) <= remainderD2
+			}
+			return (remainderD * pow10[scaleDiff]) < remainderD2
+		}
+		scaleDiff := d.Scale - d2.Scale
+		if eq {
+			return remainderD <= (remainderD2 * pow10[scaleDiff])
+		}
+		return remainderD < (remainderD2 * pow10[scaleDiff])
+	}
+
+	return false
+}
+
+func (d Decimal) greaterThan(d2 Decimal, eq bool) bool {
+	if d.Scale == d2.Scale {
+		if eq {
+			return d.Value >= d2.Value
+		}
+		return d.Value > d2.Value
+	}
+
+	quotientD, quotientD2 := quotient(d), quotient(d2)
+	if quotientD > quotientD2 {
+		return true
+	} else if quotientD == quotientD2 {
+		remainderD, remainderD2 := remainder(d), remainder(d2)
+		if d.Scale < d2.Scale {
+			scaleDiff := d2.Scale - d.Scale
+			if eq {
+				return (remainderD * pow10[scaleDiff]) >= remainderD2
+			}
+			return (remainderD * pow10[scaleDiff]) > remainderD2
+		}
+		scaleDiff := d.Scale - d2.Scale
+		if eq {
+			return remainderD >= (remainderD2 * pow10[scaleDiff])
+		}
+		return remainderD > (remainderD2 * pow10[scaleDiff])
+	}
+
+	return false
+}
+
+// SupportedByScale returns true if d can be represented
+// by a decimal based on scale.
+// For example:
+// scale = 2:
+// min: -92233720368547758.08
+// max:  92233720368547758.07
+// would not support: NewDecimal(9223372036854775807, 0)
+func (d Decimal) SupportedByScale(scale int64) bool {
+	min, max := MinMax(scale)
+
+	if d.GreaterThanOrEqualTo(min) && d.LessThanOrEqualTo(max) {
+		return true
+	}
+	return false
+}
+
+// IsValid returns true if the decimal does not break
+// any assumption or resrictions on input.
+func (d Decimal) IsValid() bool {
+	if d.Scale < -18 || d.Scale > 19 {
+		return false
+	}
+	return true
+}
+
+// ToInt64 returns d as an int64 adjusted to the
+// provided scale.
+func (d Decimal) ToInt64(scale int64) int64 {
+	var ret int64
+	scaleDiff := scale - d.Scale
+	if scaleDiff == 0 {
+		ret = d.Value
+	} else if scaleDiff < 0 {
+		ret = d.Value / Pow10(-1*scaleDiff)
+	} else {
+		ret = d.Value * Pow10(scaleDiff)
+	}
+	return ret
+}
+
+// Float64 returns d as a float64.
+// TODO: this could potentially lose precision; we should audit
+// its use and protect against unexpected results.
+func (d Decimal) Float64() float64 {
+	var ret float64
+	if d.Scale == 0 {
+		ret = float64(d.Value)
+	} else {
+		ret = float64(d.Value) / math.Pow10(int(d.Scale))
+	}
+	return ret
+}
+
+// String returns the string representation of the decimal.
+func (d Decimal) String() string {
+	var s string
+
+	var neg bool
+	sval := fmt.Sprintf("%d", d.Value)
+
+	// Strip the negative sign off for now, and
+	// re-apply it at the end.
+	if sval[0] == '-' {
+		neg = true
+		sval = sval[1:]
+	}
+
+	if d.Scale == 0 {
+		s = sval
+	} else if d.Scale < 0 {
+		s = sval + strings.Repeat("0", int(-1*d.Scale))
+	} else {
+		var bufLen int
+		if int(d.Scale) < len(sval) {
+			bufLen = len(sval) + 1
+		} else {
+			bufLen = int(d.Scale) + 2
+		}
+
+		buf := make([]byte, bufLen)
+		j := 0
+		for i := range buf {
+			z := len(buf) - 1 - i // index into buf from the end
+			if i == int(d.Scale) {
+				buf[z] = '.'
+				continue
+			}
+			if len(sval) > j {
+				buf[z] = sval[len(sval)-1-j]
+				j++
+			} else {
+				buf[z] = '0'
+			}
+		}
+		s = string(buf)
+	}
+
+	if neg {
+		return "-" + s
+	}
+	return s
+}
+
+const (
+	stateSign         = "sign"
+	stateLeadingZeros = "zeros"
+	stateMantissa     = "mantissa"
+)
+
+// ParseDecimal parses a string into a Decimal.
+func ParseDecimal(s string) (Decimal, error) {
+	var sign bool
+	var value int64
+	var scale int64
+	var err error
+
+	// General steps:
+	// - Trim leading whitespace/zeros
+	// - Get the sign value
+	// - Trim leading zeros
+	// - Push characters into a buffer
+	// - Track position of decimal point
+	// - Trim trailing zeros of buffer
+	// - value = buffer -> int
+	// - scale = len(buffer) - tracked position
+
+	var decimalPos int = -1
+	var pos int
+	mantissa := make([]byte, len(s))
+
+	state := stateSign
+	var foundLeadingZero bool
+	for i := 0; i < len(s); i++ {
+		switch state {
+		case stateSign:
+			switch s[i] {
+			case ' ':
+				continue
+			case '-':
+				sign = true
+				fallthrough
+			case '+':
+				state = stateLeadingZeros
+			default:
+				state = stateLeadingZeros
+				i--
+			}
+		case stateLeadingZeros:
+			switch s[i] {
+			case '0':
+				foundLeadingZero = true
+				continue
+			default:
+				state = stateMantissa
+				i--
+			}
+		case stateMantissa:
+			switch s[i] {
+			case '.':
+				if decimalPos == -1 {
+					decimalPos = pos
+				} else {
+					return Decimal{}, errors.Errorf("invalid decimal string: %s", s)
+				}
+				continue
+			default:
+				mantissa[pos] = s[i]
+				pos++
+			}
+		}
+	}
+
+	// If we've gotten here and state is still in stateSign or
+	// it's in stateLeadingZeros without finding any zeros,
+	// it means no value was provided.
+	if state == stateSign || (state == stateLeadingZeros && !foundLeadingZero) {
+		return Decimal{}, errors.New("decimal string is empty")
+	}
+
+	// Trim trailing zeros/spaces of mantissa
+	// for any portion that would have been to the
+	// right of the decimal.
+	trimZeroCnt := 0
+	trimSpaceCnt := 0
+	for i := len(mantissa) - 1; i >= 0; i-- {
+		switch mantissa[i] {
+		case uint8(0), uint8(32): // nil/space
+			trimSpaceCnt++
+			continue
+		case uint8(48): // zero
+			trimZeroCnt++
+			continue
+		}
+		break
+	}
+	mantissa = mantissa[:len(mantissa)-trimSpaceCnt-trimZeroCnt]
+
+	// Based on where (or if) the decimal was found,
+	// calculate scale.
+	if decimalPos == -1 {
+		scale = -1 * int64(trimZeroCnt)
+	} else {
+		scale = int64(len(mantissa) - decimalPos)
+	}
+
+	// If mantissa is empty, treat it as "0".
+	if len(mantissa) == 0 {
+		mantissa = []byte{'0'}
+		sign = false
+		scale = 0
+	}
+
+	// If the mantissa can't be represented by an int64, but it contains
+	// enough decimal places such that we can sacrifice precision, then
+	// we do that. This is an attempt to be compatible with the way
+	// `strconv.ParseFloat` works.
+	if m, s, ok := reducePrecision(sign, mantissa, scale); ok {
+		mantissa = m
+		scale = s
+	} else {
+		return Decimal{}, errors.Errorf("value out of range: %s", mantissa)
+	}
+
+	// We have to use ParseUint here (as opposed to ParseInt) because
+	// math.MinInt64 is a valid value, but its absolute value is not.
+	// So this allows us to handle that one value without overflow, and
+	// then we check for the uint bounds in the next step.
+	uvalue, err := strconv.ParseUint(string(mantissa), 10, 64)
+	if err != nil {
+		return Decimal{}, errors.Wrap(err, "converting mantissa string to uint64")
+	}
+
+	if (sign && uvalue > -1*math.MinInt64) || (!sign && uvalue > math.MaxInt64) {
+		return Decimal{}, errors.New("value out of range")
+	}
+	value = int64(uvalue)
+
+	if sign {
+		value *= -1
+	}
+
+	return Decimal{
+		Value: value,
+		Scale: scale,
+	}, nil
+}
+
+// reducePrecision takes a []byte mantissa and scale, and if possible
+// will adjust the mantissa (by reducing precision) until it can be
+// represented by an int64. The returned bool indicates whether the
+// reduction was successful.
+func reducePrecision(sign bool, mantissa []byte, scale int64) ([]byte, int64, bool) {
+	// Trim leading zeros before considering length.
+	var zeroIdx int
+	for i := range mantissa {
+		if mantissa[i] == '0' {
+			zeroIdx++
+		} else {
+			break
+		}
+	}
+	mantissa = mantissa[zeroIdx:]
+
+	// If we zero out the mantissa to an empty
+	// string, that means it's value should be 0.
+	if len(mantissa) == 0 {
+		mantissa = []byte{'0'}
+		return mantissa, scale, true
+	}
+
+	lenMantissa := len(mantissa)
+	maxStr := "9223372036854775807"
+	if sign {
+		maxStr = "9223372036854775808"
+	}
+
+	if lenMantissa <= 18 || (lenMantissa == 19 && string(mantissa) <= maxStr) {
+		return mantissa, scale, true
+	}
+
+	// If we don't have any decimal places to sacrifice,
+	// we can't change anything.
+	if scale <= 0 {
+		return mantissa, scale, false
+	}
+
+	return reducePrecision(sign, mantissa[:len(mantissa)-1], scale-1)
+}
+
+func quotient(d Decimal) int64 {
+	if d.Scale == 0 {
+		return d.Value
+	} else if d.Scale > 0 && d.Scale < 19 {
+		return d.Value / pow10[d.Scale]
+	} else if d.Scale < 0 && d.Scale > -19 {
+		return d.Value * pow10[-1*d.Scale]
+	}
+	return 0
+}
+
+func remainder(d Decimal) int64 {
+	if d.Scale >= 0 && d.Scale < 19 {
+		return d.Value % pow10[d.Scale]
+	}
+	return 0
+}
+
+// UnmarshalJSON is a custom unmarshaller for the Decimal
+// type. The intention is to avoid the use of float64
+// anywhere, so this unmarhaller parses the decimal out
+// of the byte string.
+func (d *Decimal) UnmarshalJSON(data []byte) error {
+	o, err := ParseDecimal(string(data))
+	if err != nil {
+		return errors.Wrapf(err, "parsing decimal: %s", string(data))
+	}
+	d.Value = o.Value
+	d.Scale = o.Scale
+
+	return nil
+}
+
+// MarshalJSON is a custom marshaller for the Decimal type.
+func (d Decimal) MarshalJSON() ([]byte, error) {
+	return []byte(d.String()), nil
+}
+
+// UnmarshalYAML is a custom unmarshaller for the Decimal
+// type.
+func (d *Decimal) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var data string
+	if err := unmarshal(&data); err != nil {
+		return err
+	}
+
+	o, err := ParseDecimal(data)
+	if err != nil {
+		return errors.Wrapf(err, "parsing decimal: %s", data)
+	}
+	d.Value = o.Value
+	d.Scale = o.Scale
+
+	return nil
+}
+
+// MarshalYAML is a custom marshaller for the Decimal type.
+func (d Decimal) MarshalYAML() (interface{}, error) {
+	// TODO: I don't love that this results in a quoted string
+	// in the yaml document:
+	//
+	// min: "-100.05"
+	//
+	// It would be nice if we could get that to result in:
+	//
+	// min: -100.05
+	//
+	// Note that we _can_ do that by casting the output as
+	// float64 (for certain cases), but the whole point of
+	// Decimal is to avoid using float64.
+	return d.String(), nil
+}

pql/decimal_test.go

@@ -0,0 +1,262 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pql_test
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/pql"
+)
+
+// Ensure call can be converted into a string.
+func TestDecimal(t *testing.T) {
+	t.Run("Parse", func(t *testing.T) {
+		tests := []struct {
+			s      string
+			exp    pql.Decimal
+			expErr string
+		}{
+			{"0", pql.Decimal{0, 0}, ""},
+			{"-0", pql.Decimal{0, 0}, ""},
+			{"0.0", pql.Decimal{0, 0}, ""},
+			{"-0.00", pql.Decimal{0, 0}, ""},
+			{"123.4567", pql.Decimal{1234567, 4}, ""},
+			{"  123.4567", pql.Decimal{1234567, 4}, ""},
+			{"  123.4567  ", pql.Decimal{1234567, 4}, ""},
+			{"123.456700", pql.Decimal{1234567, 4}, ""},
+			{"00123.4567", pql.Decimal{1234567, 4}, ""},
+			{"+123.4567", pql.Decimal{1234567, 4}, ""},
+			{"-123.4567", pql.Decimal{-1234567, 4}, ""},
+			{"-00123.4567", pql.Decimal{-1234567, 4}, ""},
+			{"-12.25", pql.Decimal{-1225, 2}, ""},
+
+			{"123", pql.Decimal{123, 0}, ""},
+			{"-12300", pql.Decimal{-123, -2}, ""},
+			{"+012300", pql.Decimal{123, -2}, ""},
+			{"12300", pql.Decimal{123, -2}, ""},
+			{"12300.", pql.Decimal{123, -2}, ""},
+			{"12300.0", pql.Decimal{123, -2}, ""},
+			{"123.0", pql.Decimal{123, 0}, ""},
+
+			{".123", pql.Decimal{123, 3}, ""},
+			{"0.123", pql.Decimal{123, 3}, ""},
+			{"0.001230", pql.Decimal{123, 5}, ""},
+			{" 0.001230 ", pql.Decimal{123, 5}, ""},
+			{"-0.001230 ", pql.Decimal{-123, 5}, ""},
+
+			// int64 edges.
+			{".000009223372036854775807", pql.Decimal{9223372036854775807, 24}, ""},
+			{"-.000009223372036854775808", pql.Decimal{-9223372036854775808, 24}, ""},
+			{"92233720368547.75807", pql.Decimal{9223372036854775807, 5}, ""},
+			{"-92233720368547.75807", pql.Decimal{-9223372036854775807, 5}, ""},
+			{"9223372036854775807000", pql.Decimal{9223372036854775807, -3}, ""},
+			{"-9223372036854775807000", pql.Decimal{-9223372036854775807, -3}, ""},
+
+			// precision adjustment
+			{"2.666666666666666667", pql.Decimal{2666666666666666667, 18}, ""},
+			{"2.6666666666666666667", pql.Decimal{2666666666666666666, 18}, ""},
+			{"2.6666666666666666666667", pql.Decimal{2666666666666666666, 18}, ""},
+			{"-9.223372036854775808", pql.Decimal{-9223372036854775808, 18}, ""},
+			{"-9.223372036854775809", pql.Decimal{-922337203685477580, 17}, ""},
+			{"9.223372036854775807", pql.Decimal{9223372036854775807, 18}, ""},
+			{"9.223372036854775808", pql.Decimal{922337203685477580, 17}, ""},
+
+			// Error cases.
+			{"", pql.Decimal{}, "decimal string is empty"},
+			{"-", pql.Decimal{}, "decimal string is empty"},
+			{"*0.123", pql.Decimal{}, "invalid syntax"},
+			{"abc", pql.Decimal{}, "invalid syntax"},
+			{"0.12.3", pql.Decimal{}, "invalid decimal string"},
+			{"--12300", pql.Decimal{}, "invalid syntax"},
+
+			// These are no longer error cases since we introduced precision adjustment.
+			//{"922337203685477580.9", pql.Decimal{}, "value out of range"},
+			//{"-922337203685477580.9", pql.Decimal{}, "value out of range"},
+			{"9223372036854775808000", pql.Decimal{}, "value out of range"},
+			{"-9223372036854775809000", pql.Decimal{}, "value out of range"},
+		}
+		for i, test := range tests {
+			dec, err := pql.ParseDecimal(test.s)
+			if test.expErr != "" {
+				if err == nil || !strings.Contains(err.Error(), test.expErr) {
+					t.Fatalf("test %d expected error to contain: %s, but got: %v", i, test.expErr, err)
+				}
+			} else if err != nil {
+				t.Fatalf("test %d parsing string `%s`: %s", i, test.s, err)
+			} else if dec != test.exp {
+				t.Fatalf("test %d expected: %v, but got: %v", i, test.exp, dec)
+			}
+		}
+	})
+
+	t.Run("ToInt64", func(t *testing.T) {
+		tests := []struct {
+			dec   pql.Decimal
+			scale int64
+			exp   int64
+		}{
+			{pql.Decimal{0, 0}, 0, 0},  // 0 : 0
+			{pql.Decimal{0, 0}, 1, 0},  // 0 : 0.0
+			{pql.Decimal{0, 0}, -1, 0}, // 0 : 0
+
+			{pql.Decimal{1234567, 4}, 5, 12345670}, // 123.4567 : 123.45670
+			{pql.Decimal{1234567, 4}, 4, 1234567},  // 123.4567 : 123.4567
+			{pql.Decimal{1234567, 4}, 3, 123456},   // 123.4567 : 123.456
+
+			{pql.Decimal{-1234567, 4}, 5, -12345670}, // -123.4567 : -123.45670
+			{pql.Decimal{-1234567, 4}, 4, -1234567},  // -123.4567 : -123.4567
+			{pql.Decimal{-1234567, 4}, 3, -123456},   // -123.4567 : -123.456
+
+			{pql.Decimal{123, -2}, 5, 1230000000}, // 12300 : 12300.00000
+			{pql.Decimal{123, -2}, -1, 1230},      // 12300 : 1230
+			{pql.Decimal{123, 1}, -1, 1},          // 12.3 : 1
+			{pql.Decimal{123, 1}, -2, 0},          // 12.3 : 0
+		}
+		for i, test := range tests {
+			v := test.dec.ToInt64(test.scale)
+			if v != test.exp {
+				t.Fatalf("test %d expected: %d, but got: %d", i, test.exp, v)
+			}
+		}
+	})
+
+	t.Run("String", func(t *testing.T) {
+		tests := []struct {
+			s   string
+			exp string
+		}{
+			{"123.4567", "123.4567"},
+			{"  123.4567", "123.4567"},
+			{"  123.4567  ", "123.4567"},
+			{"123.456700", "123.4567"},
+			{"00123.4567", "123.4567"},
+			{"+123.4567", "123.4567"},
+			{"-123.4567", "-123.4567"},
+			{"-00123.4567", "-123.4567"},
+			{"-12.25", "-12.25"},
+
+			{"123", "123"},
+			{"-12300", "-12300"},
+			{"+012300", "12300"},
+			{"12300", "12300"},
+
+			{"12300.", "12300"},
+			{"12300.0", "12300"},
+			{"123.0", "123"},
+
+			{"0.123", "0.123"},
+			{"0.001230", "0.00123"},
+			{" 0.001230 ", "0.00123"},
+			{"-0.001230 ", "-0.00123"},
+		}
+		for i, test := range tests {
+			dec, err := pql.ParseDecimal(test.s)
+			if err != nil {
+				t.Fatalf("parsing string `%s`: %s", test.s, err)
+			}
+			if str := dec.String(); str != test.exp {
+				t.Fatalf("test %d expected: %s, but got: %s", i, test.exp, str)
+			}
+		}
+	})
+
+	t.Run("Comparisons", func(t *testing.T) {
+		tests := []struct {
+			d1     pql.Decimal
+			d2     pql.Decimal
+			expLT  bool
+			expLTE bool
+			expGT  bool
+			expGTE bool
+			expEQ  bool
+		}{
+			{pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), false, true, false, true, true},
+			{pql.NewDecimal(0, 0), pql.NewDecimal(10, 0), true, true, false, false, false},
+			{pql.NewDecimal(10, 0), pql.NewDecimal(0, 0), false, false, true, true, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(123456, 3), false, true, false, true, true},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(123456, 4), false, false, true, true, false},
+			{pql.NewDecimal(123456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(1233456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+
+			{pql.NewDecimal(0, 0), pql.NewDecimal(-10, 0), false, false, true, true, false},
+			{pql.NewDecimal(-10, 0), pql.NewDecimal(0, 0), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(-123456, 3), false, true, false, true, true},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(-123456, 4), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-1233456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+
+			{pql.NewDecimal(10, 0), pql.NewDecimal(-10, 0), false, false, true, true, false},
+			{pql.NewDecimal(-10, 0), pql.NewDecimal(10, 0), true, true, false, false, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-123456, 3), pql.NewDecimal(123456, 4), true, true, false, false, false},
+			{pql.NewDecimal(123456, 3), pql.NewDecimal(-123456, 4), false, false, true, true, false},
+			{pql.NewDecimal(-123456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(123456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+			{pql.NewDecimal(-1233456, 4), pql.NewDecimal(123456, 3), true, true, false, false, false},
+			{pql.NewDecimal(1233456, 4), pql.NewDecimal(-123456, 3), false, false, true, true, false},
+
+			{pql.NewDecimal(9223372036854775807, 0), pql.NewDecimal(9223372036854775807, 0), false, true, false, true, true},
+			{pql.NewDecimal(9223372036854775807, 2), pql.NewDecimal(9223372036854775807, 0), true, true, false, false, false},
+			{pql.NewDecimal(9223372036854775807, 19), pql.NewDecimal(9223372036854775807, 0), true, true, false, false, false},
+
+			{pql.NewDecimal(-9223372036854775808, 0), pql.NewDecimal(-9223372036854775808, 0), false, true, false, true, true},
+			{pql.NewDecimal(-9223372036854775808, 0), pql.NewDecimal(-9223372036854775807, 0), true, true, false, false, false},
+			{pql.NewDecimal(-9223372036854775808, 2), pql.NewDecimal(-9223372036854775808, 0), false, false, true, true, false},
+			{pql.NewDecimal(-9223372036854775808, 19), pql.NewDecimal(-9223372036854775807, 0), false, false, true, true, false},
+		}
+		for i, test := range tests {
+			if got := test.d1.LessThan(test.d2); got != test.expLT {
+				t.Fatalf("test LT %d expected %s < %s to be %v, but got: %v", i, test.d1, test.d2, test.expLT, got)
+			}
+			if got := test.d1.LessThanOrEqualTo(test.d2); got != test.expLTE {
+				t.Fatalf("test LTE %d expected %s <= %s to be %v, but got: %v", i, test.d1, test.d2, test.expLTE, got)
+			}
+			if got := test.d1.GreaterThan(test.d2); got != test.expGT {
+				t.Fatalf("test GT %d expected %s > %s to be %v, but got: %v", i, test.d1, test.d2, test.expGT, got)
+			}
+			if got := test.d1.GreaterThanOrEqualTo(test.d2); got != test.expGTE {
+				t.Fatalf("test GTE %d expected %s >= %s to be %v, but got: %v", i, test.d1, test.d2, test.expGTE, got)
+			}
+			if got := test.d1.EqualTo(test.d2); got != test.expEQ {
+				t.Fatalf("test EQ %d expected %s == %s to be %v, but got: %v", i, test.d1, test.d2, test.expEQ, got)
+			}
+		}
+	})
+
+	t.Run("JSON", func(t *testing.T) {
+		t.Run("Unmarshal", func(t *testing.T) {
+			tests := []struct {
+				json string
+				exp  pql.Decimal
+			}{
+				{"1234.56", pql.NewDecimal(123456, 2)},
+			}
+			for i, test := range tests {
+				b := []byte(test.json)
+				dec := &pql.Decimal{}
+				if err := json.Unmarshal(b, &dec); err != nil {
+					panic(err)
+				}
+				if !reflect.DeepEqual(*dec, test.exp) {
+					t.Fatalf("test %d expected: %T, but got: %T", i, test.exp, dec)
+				}
+			}
+		})
+	})
+}

pql/parser.go

@@ -59,7 +59,9 @@ func (p *parser) Parse() (*Query, error) {
 	p.PQL = PQL{
 		Buffer: string(buf),
 	}
-	p.Init()
+	if err := p.Init(); err != nil {
+		return nil, errors.Wrap(err, "initializing")
+	}
 	err = p.PQL.Parse()
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing")
@@ -82,6 +84,14 @@ func (p *parser) Parse() (*Query, error) {
 			panic(v)
 		}
 	}
+	for _, call := range p.Query.Calls {
+		if call == nil {
+			return nil, fmt.Errorf("unexpected nil Call in query's call list")
+		}
+		if err := call.CheckCallInfo(); err != nil {
+			return nil, err
+		}
+	}
 
 	return &p.Query, nil
 }

pql/parser_test.go

@@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with only arguments.
 	t.Run("ArgumentsOnly", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
+		q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key":    "value",
 					"foo":    "bar",
@@ -96,19 +96,19 @@ func TestParser_Parse(t *testing.T) {
 		}
 	})
 
-	// Parse with float arguments.
-	t.Run("WithFloatArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
+	// Parse with decimal arguments.
+	t.Run("WithDecimalArgs", func(t *testing.T) {
+		q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
-					"key": 12.25,
-					"foo": 13.167,
-					"bar": 2.,
-					"baz": 0.9,
+					"key": pql.Decimal{1225, 2},
+					"foo": pql.Decimal{13167, 3},
+					"bar": pql.Decimal{2, 0},
+					"baz": pql.Decimal{9, 1},
 				},
 			},
 		) {
@@ -118,14 +118,14 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithNegativeArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
+		q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
-					"key": -12.25,
+					"key": pql.Decimal{-1225, 2},
 					"foo": int64(-13),
 				},
 			},
@@ -173,15 +173,15 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with condition arguments.
 	t.Run("WithCondition", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
+		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": "foo",
-					"x":   &pql.Condition{Op: pql.EQ, Value: 12.25},
+					"x":   &pql.Condition{Op: pql.EQ, Value: pql.Decimal{1225, 2}},
 					"y":   &pql.Condition{Op: pql.GTE, Value: int64(100)},
 					"z":   &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
 					"m":   &pql.Condition{Op: pql.NEQ, Value: nil},
@@ -191,5 +191,4 @@ func TestParser_Parse(t *testing.T) {
 			t.Fatalf("unexpected call: %#v", q.Calls[0])
 		}
 	})
-
 }

pql/pql.peg

@@ -6,22 +6,27 @@ type PQL Peg {
 
 
 Calls <- sp (Call sp)* !.
-Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
-       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
-       / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
-       / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
-       / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
-       / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
+Call <-  'Set' {p.startCall("Set")} open col comma dargs (comma timestamp)? close {p.endCall()}
+       / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma fargs close {p.endCall()}
+       / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma fargs close {p.endCall()}
+       / 'Clear' {p.startCall("Clear")} open col comma dargs close {p.endCall()}
+       / 'ClearRow' {p.startCall("ClearRow")} open darg close {p.endCall()}
+       / 'Store' {p.startCall("Store")} open Call comma darg close {p.endCall()}
        / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
        / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
-       / 'Range' {p.startCall("Range")} open field sp '=' sp value comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
+       / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
        / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
-allargs <- Call (comma Call)* (comma args)? / args / sp
-args <- arg (comma args)? sp
-arg <- (   field sp '=' sp value
-         / field sp COND sp value
-         / conditional
-         )
+allargs <- Call (comma Call)* (comma dargs)? / dargs / sp
+fargs <- farg (comma fargs)? sp
+farg <- ( field sp '=' sp fvalue
+        / field sp COND sp fvalue
+        / conditional
+        )
+dargs <- darg (comma dargs)? sp
+darg <- ( field sp '=' sp dvalue
+        / field sp COND sp dvalue
+        / conditional
+        )
 COND <- ( '><' { p.addBTWN() }
         / '<=' { p.addLTE() }
         / '>=' { p.addGTE() }
@@ -32,30 +37,47 @@ COND <- ( '><' { p.addBTWN() }
         )
 
 conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
-condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
+condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])}
 condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
 condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
 
-value <- ( item
-         / lbrack { p.startList() } list rbrack { p.endList() }
+dvalue <- ( ditem
+         / lbrack { p.startList() } dlist rbrack { p.endList() }
          )
-list <- item (comma list)?
-item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
+fvalue <- ( fitem
+          / lbrack { p.startList() } flist rbrack { p.endList() }
+          )
+dlist <- ditem (comma dlist)?
+flist <- fitem (comma flist)?
+ditem <- ( itema
+         / decimal
+         / itemb
+         )
+fitem <- ( itema
+         / float
+         / itemb
+         )
+itema <- ( 'null' &(comma / sp close) { p.addVal(nil) }
          / 'true' &(comma / sp close) { p.addVal(true) }
          / 'false' &(comma / sp close) { p.addVal(false) }
          / timestampfmt { p.addVal(buffer[begin:end]) }
-         / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
-         / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
-         / < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
+         )
+itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
          / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
          / < '"'  doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }
          / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
          )
+float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) }
+         / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) }
+         )
+decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false) }
+           / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) }
+           )
 
 doublequotedstring <- ( '\\"' / '\\\\' / [^"] )*
 singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )*
 
-fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' / '-' )*
+fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
 field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
 reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
 posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }

pql/pqlpeg_test.go

@@ -24,7 +24,9 @@ import (
 func TestPEG(t *testing.T) {
 	p := PQL{Buffer: `
 SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
-	p.Init()
+	if err := p.Init(); err != nil {
+		t.Fatalf("initialization error: %v", err)
+	}
 	err := p.Parse()
 	if err != nil {
 		t.Fatalf("parse error: %v", err)
@@ -32,7 +34,9 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 	p.Execute()
 
 	p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
-	p.Init()
+	if err := p.Init(); err != nil {
+		t.Fatalf("initialization error: %v", err)
+	}
 	err = p.Parse()
 	if err == nil {
 		t.Fatalf("should have been an error because of the interior unescaped double quote")
@@ -42,11 +46,11 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 	if err != nil {
 		t.Fatalf("should have parsed: %v", err)
 	}
-	if q.String() != `TopN(Bitmap(id == "other"), _field="blah", field="f", n=0)` {
+	if q.String() != `TopN(Bitmap(id=="other"), _field="blah", field="f", n=0)` {
 		t.Fatalf("Failed, got: %s", q)
 	}
 
-	_, err = ParseString("C(a=falsen0)")
+	_, err = ParseString("Row(a=falsen0)")
 	if err != nil {
 		t.Fatalf("falsen0 should have been parsed as a string")
 	}
@@ -55,8 +59,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 	if err != nil {
 		t.Fatalf("should have parsed: %v", err)
 	}
-
-	if q.String() != `Bitmap(did == "other", row=4)` {
+	if q.String() != `Bitmap(did=="other", row=4)` {
 		t.Fatalf("got %s", q)
 	}
 
@@ -109,15 +112,15 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 2},
 		{
 			name:   "SetWithArbCall",
-			input:  "Set(1, a=4)Blerg(z=ha)",
+			input:  "Set(1, a=4)Row(z=ha)",
 			ncalls: 2},
 		{
 			name:   "SetArbSet",
-			input:  "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
+			input:  "Set(1, a=4)Row(z=ha)Set(2, z=99)",
 			ncalls: 3},
 		{
 			name:   "ArbSetArb",
-			input:  "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
+			input:  "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
 			ncalls: 3},
 		{
 			name:   "SetStringArg",
@@ -161,11 +164,11 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 1},
 		{
 			name:   "double quoted args",
-			input:  `B(a="zm''e")`,
+			input:  `Row(a="zm''e")`,
 			ncalls: 1},
 		{
 			name:   "single quoted args",
-			input:  `B(a='zm""e')`,
+			input:  `Row(a='zm""e')`,
 			ncalls: 1},
 		{
 			name:   "SetRowAttrs",
@@ -320,7 +323,7 @@ func TestPEGErrors(t *testing.T) {
 			input: "Set(, 1, a=4)"},
 		{
 			name:  "StartinCommaArb",
-			input: "Zeeb(, a=4)"},
+			input: "Row(, a=4)"},
 		{
 			name:  "SetRowAttrs0args",
 			input: "SetRowAttrs(blah, 9)"},
@@ -533,8 +536,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(4), int64(8)},
+						Op:    BTWN_LTE_LT,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -545,8 +548,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(8)},
+						Op:    BTWN_LT_LT,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -569,8 +572,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(9)},
+						Op:    BTWN_LT_LTE,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -585,11 +588,11 @@ func TestPQLDeepEquality(t *testing.T) {
 			}},
 		{
 			name: "Weird dash",
-			call: "Sum(field-=f)",
+			call: "Count(dashy-=f)",
 			exp: &Call{
-				Name: "Sum",
+				Name: "Count",
 				Args: map[string]interface{}{
-					"field-": "f",
+					"dashy-": "f",
 				},
 			}},
 		{
@@ -672,8 +675,8 @@ func TestPQLDeepEquality(t *testing.T) {
 						Name: "Row",
 						Args: map[string]interface{}{
 							"a": &Condition{
-								Op:    BETWEEN,
-								Value: []interface{}{int64(5), int64(8)},
+								Op:    BTWN_LT_LT,
+								Value: []interface{}{int64(4), int64(9)},
 							},
 						},
 					},

pql/token.go

@@ -21,14 +21,24 @@ const (
 	// Special tokens
 	ILLEGAL Token = iota
 
-	ASSIGN  // =
-	EQ      // ==
-	NEQ     // !=
-	LT      // <
-	LTE     // <=
-	GT      // >
-	GTE     // >=
-	BETWEEN // ><
+	ASSIGN // =
+	EQ     // ==
+	NEQ    // !=
+	LT     // <
+	LTE    // <=
+	GT     // >
+	GTE    // >=
+
+	BETWEEN // ><  (this is like a <= x <= b)
+
+	// not used in lexing/parsing, but so that the parser can signal
+	// to the executor how to treat the arguments. We used to just add
+	// 1 to the arguments if they were LT so the executor could assume
+	// it was always <=, <=, but then we needed to support
+	// floats/decimals and couldn't do that any more.
+	BTWN_LT_LTE // a < x <= b
+	BTWN_LTE_LT // a <= x < b
+	BTWN_LT_LT  // a < x < b
 )
 
 var tokens = [...]string{

prometheus/prometheus.go

@@ -27,7 +27,7 @@ import (
 
 const (
 	// namespace is prepended to each metric event name with "_"
-	namespace = "pilosa"
+	defaultNamespace = "pilosa"
 )
 
 // Ensure client implements interface.
@@ -46,11 +46,22 @@ type prometheusClient struct {
 	gaugeVecs   map[string]*prometheus.GaugeVec
 	observers   map[string]prometheus.Observer
 	summaryVecs map[string]*prometheus.SummaryVec
+	namespace   string
+}
+
+// ClientOption is a functional option type for prometheusClient
+type ClientOption func(c *prometheusClient)
+
+// OptClientPrefix is a functional option on prometheusClient used to set the namespace
+func OptClientNamespace(namespace string) ClientOption {
+	return func(c *prometheusClient) {
+		c.namespace = namespace
+	}
 }
 
 // NewPrometheusClient returns a new instance of StatsClient.
-func NewPrometheusClient() (*prometheusClient, error) {
-	return &prometheusClient{
+func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) {
+	client := &prometheusClient{
 		logger:      logger.NopLogger,
 		counters:    make(map[string]prometheus.Counter),
 		counterVecs: make(map[string]*prometheus.CounterVec),
@@ -58,7 +69,14 @@ func NewPrometheusClient() (*prometheusClient, error) {
 		gaugeVecs:   make(map[string]*prometheus.GaugeVec),
 		observers:   make(map[string]prometheus.Observer),
 		summaryVecs: make(map[string]*prometheus.SummaryVec),
-	}, nil
+		namespace:   defaultNamespace,
+	}
+
+	for _, opt := range opts {
+		opt(client)
+	}
+
+	return client, nil
 }
 
 // Open no-op to satisfy interface
@@ -76,7 +94,7 @@ func (c *prometheusClient) Tags() []string {
 
 // labels returns an instance of prometheus.Labels with the value of the set tags.
 func (c *prometheusClient) labels() prometheus.Labels {
-	return tagsToLabels(c.tags)
+	return tagsToLabels(c.tags, c.logger)
 }
 
 // WithTags returns a new client with additional tags appended.
@@ -90,6 +108,7 @@ func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient {
 		gaugeVecs:   c.gaugeVecs,
 		observers:   c.observers,
 		summaryVecs: c.summaryVecs,
+		namespace:   c.namespace,
 	}
 }
 
@@ -103,7 +122,7 @@ func (c *prometheusClient) Count(name string, value int64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.CounterOpts{
-		Namespace: namespace,
+		Namespace: c.namespace,
 		Name:      name,
 	}
 	if len(labels) == 0 {
@@ -152,7 +171,7 @@ func (c *prometheusClient) Gauge(name string, value float64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.GaugeOpts{
-		Namespace: namespace,
+		Namespace: c.namespace,
 		Name:      name,
 	}
 	if len(labels) == 0 {
@@ -193,7 +212,7 @@ func (c *prometheusClient) Histogram(name string, value float64, rate float64) {
 	name = strings.Replace(name, ".", "_", -1)
 	labels := c.labels()
 	opts := prometheus.SummaryOpts{
-		Namespace:  namespace,
+		Namespace:  c.namespace,
 		Name:       name,
 		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 	}
@@ -233,8 +252,7 @@ func (c *prometheusClient) Set(name string, value string, rate float64) {
 
 // Timing tracks timing information for a metric.
 func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) {
-	durationMs := value / time.Second
-	c.Histogram(name, float64(durationMs), rate)
+	c.Histogram(name, value.Seconds(), rate)
 }
 
 // SetLogger sets the logger for client.
@@ -277,12 +295,13 @@ func unionStringSlice(a, b []string) []string {
 	return other
 }
 
-func tagsToLabels(tags []string) (labels prometheus.Labels) {
+func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) {
 	labels = make(prometheus.Labels)
 	for _, tag := range tags {
 		tagParts := strings.SplitAfterN(tag, ":", 2)
 		if len(tagParts) != 2 {
 			// only process tags in "key:value" form
+			logger.Printf("Error: invalid Prometheus label: %v\n", tag)
 			continue
 		}
 		labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1]

proto/interface.go

@@ -0,0 +1,314 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// StreamClient is an interface for a stream
+// which can return a RowResponse sent to a
+// stream via Send().
+type StreamClient interface {
+	Recv() (*RowResponse, error)
+}
+
+// StreamServer is an interface for a stream
+// which can accept a RowResponse to be later
+// returned by the stream via Recv().
+type StreamServer interface {
+	Send(*RowResponse) error
+}
+
+// ToTabler is an interface for any type that can
+// represent itself as a TableResponse.
+type ToTabler interface {
+	ToTable() (*TableResponse, error)
+}
+
+// ToRowser is an interface for any type that can
+// represent itself as one or more RowResponses.
+// ToRows takes a callback function which should be
+// called for each row in the response.
+type ToRowser interface {
+	ToRows(func(*RowResponse) error) error
+}
+
+// RowsToTable is a helper function which takes a ToRowser,
+// along with the number of rows, and returns a TableResponse.
+// Obviously passing the number of rows seems unnecessary,
+// and we could remove that requirement, but for now we
+// do it to allow for pre-allocation of the rows slice.
+func RowsToTable(tr ToRowser, n int) (*TableResponse, error) {
+	var headers []*ColumnInfo
+	rows := make([]*Row, n)
+
+	// This callback gets called for every "row" in r.
+	// Each row populates its position in the pre-allocated
+	// `rows`. The headers get set based on those received
+	// in the first row.
+	var idx int
+	cb := func(rr *RowResponse) error {
+		if idx == 0 {
+			headers = rr.GetHeaders()
+		}
+		rows[idx] = &Row{Columns: rr.GetColumns()}
+		idx++
+		return nil
+	}
+
+	if err := tr.ToRows(cb); err != nil {
+		return nil, errors.Wrap(err, "calling callback")
+	}
+
+	return &TableResponse{
+		Headers: headers,
+		Rows:    rows,
+	}, nil
+}
+
+// EOF acts as an io.EOF encoded into a RowResponse.
+var EOF *RowResponse = &RowResponse{
+	StatusError: &StatusError{
+		Code:    0,
+		Message: "EOF",
+	},
+}
+
+// Error is a helper function to create a RowResponse
+// based on an error message. If the error is a grpc
+// Status, then the status code is passed through.
+func Error(err error) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrap prepends a message to the existing status
+// error message.
+func ErrorWrap(err error, message string) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: message + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrapf prepends a message to the existing status
+// error message with the format specifier.
+func ErrorWrapf(err error, format string, args ...interface{}) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: fmt.Sprintf(format, args...) + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorCode is a helper function to create a RowResponse
+// based on a grpc status code and an error message.
+func ErrorCode(err error, c codes.Code) *RowResponse {
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(c),
+			Message: err.Error(),
+		},
+	}
+}
+
+// RowResponseSorter implements the sort interface for a
+// provided []RowResponse based on the column index, type,
+// and sort direction.
+type RowResponseSorter struct {
+	colIdx        []int
+	colDescending []bool
+	colType       []string
+
+	rrs []*RowResponse
+}
+
+// NewRowResponseSorter return a new RowResponseSorter. It
+// does input validation and returns an error if the inputs
+// aren't compatible.
+func NewRowResponseSorter(idxs []int, dirs []bool, typs []string, rrs []*RowResponse) (*RowResponseSorter, error) {
+	// Ensure the input slices are non-empty and equal size.
+	if len(idxs) == 0 {
+		return nil, errors.New("index list cannot be empty")
+	}
+	if len(dirs) != len(idxs) || len(typs) != len(idxs) {
+		return nil, errors.New("index, direction, and type lists must be the same size")
+	}
+
+	// Ensure the provided data types are supported by the sorter.
+	for i := range typs {
+		switch typs[i] {
+		case "[]uint64", "[]string", "bool", "float64", "int64", "string", "uint64":
+			// pass
+		default:
+			return nil, fmt.Errorf("unsupported data type: %s", typs[i])
+		}
+	}
+
+	// Ensure max(colIdx) is within size of rr.Columns.
+	if len(rrs) > 0 {
+		var maxColIdx int
+		for i := range idxs {
+			if idxs[i] > maxColIdx {
+				maxColIdx = idxs[i]
+			}
+		}
+		if maxColIdx >= len(rrs[0].Columns) {
+			return nil, fmt.Errorf("column index is out of range: %d", maxColIdx)
+		}
+	}
+
+	return &RowResponseSorter{
+		colIdx:        idxs,
+		colDescending: dirs,
+		colType:       typs,
+		rrs:           rrs,
+	}, nil
+
+}
+
+func (r RowResponseSorter) Len() int      { return len(r.rrs) }
+func (r RowResponseSorter) Swap(i, j int) { r.rrs[i], r.rrs[j] = r.rrs[j], r.rrs[i] }
+func (r RowResponseSorter) Less(i, j int) bool {
+	ri := r.rrs[i]
+	rj := r.rrs[j]
+
+	for i, idx := range r.colIdx {
+		coli := ri.Columns[idx]
+		colj := rj.Columns[idx]
+		var comp int
+		switch r.colType[i] {
+		case "[]uint64":
+			ai := coli.GetUint64ArrayVal().Vals
+			aj := colj.GetUint64ArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					piv := ai[ii]
+					pjv := aj[ii]
+					if piv == pjv {
+						continue
+					} else if piv < pjv {
+						return -1
+					} else {
+						return 1
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "[]string":
+			ai := coli.GetStringArrayVal().Vals
+			aj := colj.GetStringArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					sComp := strings.Compare(ai[ii], aj[ii])
+					if sComp == 0 {
+						continue
+					} else {
+						return sComp
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "bool":
+			bi := coli.GetBoolVal()
+			bj := colj.GetBoolVal()
+			if bi == bj {
+				comp = 0
+			} else if !bi && bj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "float64":
+			fi := coli.GetFloat64Val()
+			fj := colj.GetFloat64Val()
+			if fi == fj {
+				comp = 0
+			} else if fi < fj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "int64":
+			ni := coli.GetInt64Val()
+			nj := colj.GetInt64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "string":
+			comp = strings.Compare(coli.GetStringVal(), colj.GetStringVal())
+		case "uint64":
+			ni := coli.GetUint64Val()
+			nj := colj.GetUint64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		}
+
+		isDescending := r.colDescending[i]
+
+		switch comp {
+		case 0:
+			continue
+		case -1:
+			if isDescending {
+				return false
+			}
+			return true
+		case 1:
+			if isDescending {
+				return true
+			}
+			return false
+		}
+	}
+	return false
+}

proto/pilosa.proto

@@ -0,0 +1,81 @@
+syntax = "proto3";
+package pilosa;
+
+message QueryPQLRequest {
+  string index = 1;
+  string pql = 2;
+}
+
+message StatusError{
+  uint32 Code = 1;
+  string Message = 2;
+}
+
+message RowResponse{
+  repeated ColumnInfo headers = 1;
+  repeated ColumnResponse columns = 2;
+  StatusError StatusError = 3;
+}
+
+message Row {
+  repeated ColumnResponse columns = 1;
+}
+
+message TableResponse{
+  repeated ColumnInfo headers = 1;
+  repeated Row rows = 2;
+  StatusError StatusError = 3;
+}
+
+message ColumnInfo {
+  string name = 1;
+  string datatype = 2;
+}
+
+message ColumnResponse{
+  oneof columnVal {
+    string stringVal = 1;
+    uint64 uint64Val = 2;
+    int64 int64Val = 3;
+    bool boolVal = 4;
+    bytes blobVal = 5;
+    Uint64Array uint64ArrayVal = 6;
+    StringArray stringArrayVal = 7;
+    double float64Val = 8;
+    Decimal decimalVal = 9;
+  }
+}
+
+message Decimal {
+	int64 value = 1;
+	int64 scale = 2;
+}
+
+message InspectRequest {
+  string index = 1;
+  IdsOrKeys columns = 2;
+  repeated string filterFields = 3;
+  uint64 limit = 4;
+  uint64 offset = 5;
+}
+
+message Uint64Array {
+  repeated uint64 vals = 1;
+}
+
+message StringArray {
+  repeated string vals = 1;
+}
+
+message IdsOrKeys {
+  oneof type {
+    Uint64Array ids = 1;
+    StringArray keys = 2;
+  }
+}
+
+service Pilosa {
+  rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};
+  rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {};
+  rpc Inspect(InspectRequest) returns (stream RowResponse) {};
+}

roaring/container_stash.go

@@ -267,9 +267,6 @@ func (c *Container) Freeze() *Container {
 	if c.flags&flagFrozen != 0 {
 		return c
 	}
-	// unmapOrClone should unmap-in-place because the existing
-	// container isn't frozen (or we'd already have returned it).
-	c = c.unmapOrClone()
 	c.flags |= flagFrozen
 	return c
 }
@@ -419,6 +416,48 @@ func (c *Container) bitmap() []uint64 {
 	return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)}))
 }
 
+// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
+// is provided. The target should be zeroed, or this becomes an implicit
+// union.
+func (c *Container) AsBitmap(target []uint64) (out []uint64) {
+	if c.typeID == containerBitmap {
+		return c.bitmap()
+	}
+	// Reminder: len(nil) == 0.
+	if len(target) < 1024 {
+		out = make([]uint64, 1024)
+	} else {
+		out = target
+		for i := range out {
+			out[i] = 0
+		}
+	}
+	if c.typeID == containerArray {
+		a := c.array()
+		for _, v := range a {
+			out[v/64] |= 1 << (v % 64)
+		}
+		return out
+	}
+	if c.typeID == containerRun {
+		runs := c.runs()
+		for _, r := range runs {
+			splatRun(out, r)
+		}
+		return out
+	}
+	// in theory this shouldn't happen?
+	return out
+}
+
+func splatRun(into []uint64, from interval16) {
+	// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
+	//note v must be int or will overflow
+	for v := int(from.start); v <= int(from.last); v++ {
+		into[v/64] |= (uint64(1) << uint(v%64))
+	}
+}
+
 // setBitmap stores a set of uint64s as data.
 func (c *Container) setBitmap(bitmap []uint64) {
 	if c == nil || c.frozen() {

roaring/containers_btree.go

@@ -62,12 +62,6 @@ func (btc *bTreeContainers) Put(key uint64, c *Container) {
 	// Get can result in the tree containing a different container
 	// than we'll get on next lookup.
 	btc.lastKey, btc.lastContainer = key, c
-	// If a mapped container is added to the tree, reset the
-	// lastContainer cache so that the cache is not pointing
-	// at a read-only mmap.
-	if c.Mapped() {
-		btc.lastKey = ^uint64(0)
-	}
 	btc.tree.Set(key, c)
 }
 
@@ -119,7 +113,6 @@ func (btc *bTreeContainers) GetOrCreate(key uint64) *Container {
 		btc.lastContainer = cont
 		return cont
 	}
-
 	btc.lastContainer = v
 	return btc.lastContainer
 }
@@ -229,6 +222,9 @@ func (btc *bTreeContainers) UpdateEvery(fn func(uint64, *Container, bool) (*Cont
 	// currently not handling the error from this, but in practice it has
 	// to be io.EOF.
 	_ = e.Every(fn)
+	// invalidate cache.
+	btc.lastKey = ^uint64(0)
+	btc.lastContainer = nil
 }
 
 type btcIterator struct {

roaring/containers_slice.go

@@ -43,7 +43,8 @@ func (sc *sliceContainers) Put(key uint64, c *Container) {
 	} else {
 		sc.containers[i] = c
 	}
-
+	sc.lastKey = key
+	sc.lastContainer = c
 }
 
 func (sc *sliceContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) {
@@ -159,7 +160,7 @@ func (sc *sliceContainers) Reset() {
 	sc.keys = sc.keys[:0]
 	sc.containers = sc.containers[:0]
 	sc.lastContainer = nil
-	sc.lastKey = 0
+	sc.lastKey = ^uint64(0)
 }
 
 func (sc *sliceContainers) ResetN(n int) {
@@ -171,7 +172,7 @@ func (sc *sliceContainers) ResetN(n int) {
 		sc.containers = sc.containers[:0]
 	}
 	sc.lastContainer = nil
-	sc.lastKey = 0
+	sc.lastKey = ^uint64(0)
 }
 
 func (sc *sliceContainers) seek(key uint64) (int, bool) {
@@ -212,7 +213,7 @@ func (sc *sliceContainers) Update(key uint64, fn func(*Container, bool) (*Contai
 		// don't expand the slice just to add a nil container, we
 		// could return that anyway
 		if write && nc != nil {
-			sc.insertAt(key, nc, -i-1)
+			sc.insertAt(key, nc, i)
 		}
 	}
 }
@@ -227,6 +228,9 @@ func (sc *sliceContainers) UpdateEvery(fn func(uint64, *Container, bool) (*Conta
 			sc.containers[i] = nc
 		}
 	}
+	// invalidate cache.
+	sc.lastKey = ^uint64(0)
+	sc.lastContainer = nil
 }
 
 type sliceIterator struct {

roaring/generation_debug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package roaring
+
+const generationDebug = true

roaring/generation_nodebug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package roaring
+
+const generationDebug = false

roaring/roaring_internal_test.go

@@ -147,6 +147,16 @@ func TestRunCountRange(t *testing.T) {
 	c.add(10)
 	c.add(11)
 
+	cnt = c.runCountRange(4, 8)
+	if cnt != 3 {
+		t.Fatalf("should get 3 from range overlaps front of interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(5, 8)
+	if cnt != 3 {
+		t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
+	}
+
 	cnt = c.runCountRange(6, 8)
 	if cnt != 2 {
 		t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
@@ -162,6 +172,31 @@ func TestRunCountRange(t *testing.T) {
 		t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt)
 	}
 
+	cnt = c.runCountRange(8, 10)
+	if cnt != 2 {
+		t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(8, 11)
+	if cnt != 3 {
+		t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(8, 12)
+	if cnt != 4 {
+		t.Fatalf("should get 4 from range overlaps back of interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(5, 12)
+	if cnt != 7 {
+		t.Fatalf("should get 7 from interval within range, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(5, 11)
+	if cnt != 6 {
+		t.Fatalf("should get 6 from interval equal to range, but got: %v", cnt)
+	}
+
 	c.add(17)
 	c.add(19)
 	c.add(18)
@@ -2710,6 +2745,15 @@ func unionInPlaceWrapper(a, b *Container) *Container {
 	return out.Containers.Get(0)
 }
 
+func differenceInPlaceWrapper(a, b *Container) *Container {
+	out := NewBitmap()
+	out.Containers.Put(0, a.Clone())
+	B := NewBitmap()
+	B.Containers.Put(0, b)
+	out.DifferenceInPlace(B)
+	return out.Containers.Get(0)
+}
+
 func TestContainerCombinations(t *testing.T) {
 
 	cts := setupContainerTests()
@@ -3283,6 +3327,117 @@ func TestContainerCombinations(t *testing.T) {
 		{flip, "outerBitsSet", "", "innerBitsSet"},
 		{flip, "oddBitsSet", "", "evenBitsSet"},
 		{flip, "evenBitsSet", "", "oddBitsSet"},
+
+		// differenceInPlace
+		{differenceInPlaceWrapper, "empty", "empty", "empty"},
+		{differenceInPlaceWrapper, "empty", "full", "empty"},
+		{differenceInPlaceWrapper, "empty", "firstBitSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "lastBitSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "empty", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "empty", "innerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "evenBitsSet", "empty"},
+		//
+		{differenceInPlaceWrapper, "full", "empty", "full"},
+		{differenceInPlaceWrapper, "full", "full", "empty"},
+		{differenceInPlaceWrapper, "full", "firstBitSet", "firstBitUnset"},
+		{differenceInPlaceWrapper, "full", "lastBitSet", "lastBitUnset"},
+		{differenceInPlaceWrapper, "full", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "full", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "full", "innerBitsSet", "outerBitsSet"},
+		{differenceInPlaceWrapper, "full", "outerBitsSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "full", "oddBitsSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "full", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "firstBitSet", "empty", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "full", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "firstBitSet", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "lastBitSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "innerBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "oddBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "evenBitsSet", "empty"},
+		//
+		{differenceInPlaceWrapper, "lastBitSet", "empty", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "full", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "firstBitSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "lastBitSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "innerBitsSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "evenBitsSet", "lastBitSet"},
+		//
+		{differenceInPlaceWrapper, "firstBitUnset", "empty", "firstBitUnset"},
+		{differenceInPlaceWrapper, "firstBitUnset", "full", "empty"},
+		{differenceInPlaceWrapper, "firstBitUnset", "firstBitSet", "firstBitUnset"},
+		{differenceInPlaceWrapper, "firstBitUnset", "lastBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "firstBitUnset", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "innerBitsSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "outerBitsSet", "innerBitsSet"},
+		//{differenceInPlaceWrapper, "firstBitUnset", "oddBitsSet", ""},
+		{differenceInPlaceWrapper, "firstBitUnset", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "lastBitUnset", "empty", "lastBitUnset"},
+		{differenceInPlaceWrapper, "lastBitUnset", "full", "empty"},
+		{differenceInPlaceWrapper, "lastBitUnset", "firstBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "lastBitSet", "lastBitUnset"},
+		{differenceInPlaceWrapper, "lastBitUnset", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "lastBitUnset", "innerBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "outerBitsSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "oddBitsSet", "evenBitsSet"},
+		//{differenceInPlaceWrapper, "lastBitUnset", "evenBitsSet", ""},
+		//
+		{differenceInPlaceWrapper, "innerBitsSet", "empty", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "firstBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "lastBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "innerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "outerBitsSet", "innerBitsSet"},
+		//{differenceInPlaceWrapper, "innerBitsSet", "oddBitsSet", ""},
+		//{differenceInPlaceWrapper, "innerBitsSet", "evenBitsSet", ""},
+		//
+		{differenceInPlaceWrapper, "outerBitsSet", "empty", "outerBitsSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "outerBitsSet", "firstBitSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "lastBitSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "innerBitsSet", "outerBitsSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "outerBitsSet", "oddBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "evenBitsSet", "lastBitSet"},
+		//
+		{differenceInPlaceWrapper, "oddBitsSet", "empty", "oddBitsSet"},
+		{differenceInPlaceWrapper, "oddBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "firstBitSet", "oddBitsSet"},
+		//{differenceInPlaceWrapper, "oddBitsSet", "lastBitSet", ""},
+		{differenceInPlaceWrapper, "oddBitsSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "oddBitsSet", "innerBitsSet", "lastBitSet"},
+		//{differenceInPlaceWrapper, "oddBitsSet", "outerBitsSet", ""},
+		{differenceInPlaceWrapper, "oddBitsSet", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "evenBitsSet", "empty", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "full", "empty"},
+		//{differenceInPlaceWrapper, "evenBitsSet", "firstBitSet", ""},
+		{differenceInPlaceWrapper, "evenBitsSet", "lastBitSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "evenBitsSet", "innerBitsSet", "firstBitSet"},
+		//{differenceInPlaceWrapper, "evenBitsSet", "outerBitsSet", ""},
+		{differenceInPlaceWrapper, "evenBitsSet", "oddBitsSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "evenBitsSet", "empty"},
 	}
 	for _, testOp := range testOps {
 		for _, x := range containerTypes {
@@ -3896,3 +4051,12 @@ func TestBitmapAny(t *testing.T) {
 		t.Error("shouldn't be any left")
 	}
 }
+
+func TestDifferenceInPlace_N(t *testing.T) {
+	a := doContainer(containerRun, runFull())
+	b := doContainer(containerBitmap, bitmapFull())
+	r := differenceInPlaceWrapper(a, b)
+	if r.N() != 0 {
+		t.Error("expected difference of containers to have n=0")
+	}
+}

roaring/roaring_test.go

@@ -457,6 +457,12 @@ func TestBitmap_RunCountRange(t *testing.T) {
 	if n := bm2.CountRange(3, 2); n != 0 {
 		t.Fatalf("unexpected n: %d", n)
 	}
+
+	bm3 := roaring.NewFileBitmap(1, 2, 3, 4)
+	bm3.Optimize() // convert to runs
+	if n := bm3.CountRange(1, 3); n != 2 {
+		t.Fatalf("unexpected n: %d", n)
+	}
 }
 
 func TestBitmap_Intersection(t *testing.T) {
@@ -1739,3 +1745,57 @@ func BenchmarkUnionBulk(b *testing.B) {
 			UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
 	}
 }
+
+func TestBitmap_DifferenceInPlace(t *testing.T) {
+	// array
+	arraybm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 1024; i += 4 {
+		_, _ = arraybm.Add((1 << 16) + i)
+	}
+	arraybm.Optimize()
+
+	// bitmap
+	bitmapbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 16384; i += 2 {
+		_, _ = bitmapbm.Add((2 << 16) + i)
+	}
+	bitmapbm.Optimize()
+
+	// small run
+	smallrunbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 1024; i++ {
+		_, _ = smallrunbm.Add((3 << 16) + i)
+	}
+	smallrunbm.Optimize()
+
+	// large run
+	largerunbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 65535; i++ {
+		_, _ = largerunbm.Add((4 << 16) + i)
+	}
+	largerunbm.Optimize()
+
+	// test the difference in place
+	bm := testBM()
+	bm.DifferenceInPlace(arraybm, bitmapbm, smallrunbm, largerunbm)
+	if bm.Count() != 0 {
+		t.Fatalf("expected bitmap count to be 0, but got: %d", bm.Count())
+	}
+
+	bm = testBM()
+	bm.DifferenceInPlace(bitmapbm, smallrunbm, largerunbm)
+	if bm.Count() != 256 {
+		t.Fatalf("expected bitmap count to be 256, but got: %d", bm.Count())
+	}
+}
+
+func BenchmarkDifferencInPlace(b *testing.B) {
+	data := getBenchData(b)
+	for n := 0; n < b.N; n++ {
+		bm := roaring.NewBitmap()
+		bm.
+			UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
+		bm.
+			DifferenceInPlace(data.a1, data.r2, data.b, data.r1)
+	}
+}

roaring/source.go

@@ -0,0 +1,98 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roaring
+
+import (
+	"strings"
+)
+
+// A Source represents the source a given bitmap gets its data from,
+// such as a memory-mapped file. When combining bitmaps, we might
+// track them together in a single combined-source of some sort.
+type Source interface {
+	ID() string
+	Dead() bool
+}
+
+// MergeSources combines sources. If you have two bitmaps, and you're
+// combining them, then the combination's source is a combination of
+// those two sources.
+func MergeSources(sources ...Source) Source {
+	sourceCount := 0
+	totalCount := 0
+	var lastSource Source
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		lastSource = s
+		if s, ok := s.(combinedSource); ok {
+			sourceCount++
+			totalCount += len(s)
+		} else {
+			sourceCount++
+			totalCount++
+		}
+	}
+	// if there's no sources (this includes all sources being
+	// empty combinedSources), we don't have a source.
+	if totalCount == 0 {
+		return nil
+	}
+	// if there's exactly one source, combined or otherwise, that's
+	// fine, we'll just return it.
+	if sourceCount == 1 {
+		return lastSource
+	}
+	// make a new combinedSource, flattening any combinedSources
+	// already present.
+	newSources := make([]Source, 0, totalCount)
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		if s, ok := s.(combinedSource); ok {
+			newSources = append(newSources, s...)
+		} else {
+			newSources = append(newSources, s)
+		}
+	}
+	return combinedSource(newSources)
+}
+
+// SetSource tells the bitmap what source to associate with new things it
+// creates. This is possibly logically incorrect.
+func (b *Bitmap) SetSource(s Source) {
+	b.Source = s
+}
+
+type combinedSource []Source
+
+func (c combinedSource) ID() string {
+	ids := make([]string, len(c))
+	for i := range c {
+		ids[i] = c[i].ID()
+	}
+	return strings.Join(ids, ",")
+}
+
+func (c combinedSource) Dead() bool {
+	for i := range c {
+		if c[i].Dead() {
+			return true
+		}
+	}
+	return false
+}

roaring/unmarshal_binary.go

@@ -30,7 +30,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return nil
 	}
 	statsHit("Bitmap/UnmarshalBinary")
-	b.opN = 0 // reset opN since we're reading new data.
+	// reset ops/opN since we're reading new data.
+	b.ops = 0
+	b.opN = 0
 	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
 	if fileMagic == MagicNumber { // if pilosa roaring
 		return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
@@ -86,7 +88,12 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error {
 
 		// Map byte slice directly to the container data.
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerArray:
 			c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
@@ -106,7 +113,12 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) error {
 	citer, _ := b.Containers.Iterator(0)
 	for i := 0; i < int(keyN); i++ {
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerRun:
 			runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize])
@@ -174,12 +186,17 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 
 		// Map byte slice directly to the container data.
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
 
 		// this shouldn't happen, since we don't normally store nils.
 		if c == nil {
 			continue
 		}
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerRun:
 			runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
@@ -205,15 +222,15 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 		// Unmarshal the op and apply it.
 		var opr op
 		if err := opr.UnmarshalBinary(buf); err != nil {
-			// FIXME(benbjohnson): return error with position so file can be trimmed.
-			return err
+			return newFileShouldBeTruncatedError(err, int64(opsOffset))
 		}
 		opr.apply(b)
 		// Increase the op count.
 		b.ops++
 		b.opN += opr.count()
+		opsOffset += opr.size()
 		// Move the buffer forward.
-		buf = buf[opr.size():]
+		buf = data[opsOffset:]
 	}
 
 	return nil

row.go

@@ -18,6 +18,8 @@ import (
 	"encoding/json"
 	"sort"
 
+	"github.com/molecula/ext"
+	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -43,6 +45,102 @@ func NewRow(columns ...uint64) *Row {
 	return r
 }
 
+// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
+// transposes; data that was in any shard for Row 0 is now considered shard 0,
+// etcetera.
+func NewRowFromBitmap(b *roaring.Bitmap) *Row {
+	r := &Row{}
+	if b == nil {
+		return r
+	}
+	rowNum := uint64(0)
+	for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) {
+		rowNum = col / ShardWidth
+		seg := rowSegment{
+			shard:    rowNum,
+			data:     b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth),
+			writable: true,
+		}
+		seg.n = seg.data.Count()
+		r.segments = append(r.segments, seg)
+		rowNum++
+	}
+	return r
+}
+
+// NewRowFromRoaring parses a roaring data file as a row, dividing it into
+// bitmaps and rowSegments based on shard width.
+func NewRowFromRoaring(data []byte) *Row {
+	bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth)
+	r := &Row{segments: make([]rowSegment, len(bitmaps))}
+	for i := range bitmaps {
+		segment := rowSegment{
+			shard:    shards[i],
+			data:     bitmaps[i],
+			writable: false,
+			n:        bitmaps[i].Count(),
+		}
+		r.segments[i] = segment
+	}
+	return r
+}
+
+// ToTable implements the ToTabler interface.
+func (r *Row) ToTable() (*pb.TableResponse, error) {
+	var n int
+	if len(r.Keys) > 0 {
+		n = len(r.Keys)
+	} else {
+		n = len(r.Columns())
+	}
+	return pb.RowsToTable(r, n)
+}
+
+// ToRows implements the ToRowser interface.
+func (r *Row) ToRows(callback func(*pb.RowResponse) error) error {
+	if len(r.Keys) > 0 {
+		// Column keys
+		ci := []*pb.ColumnInfo{
+			{Name: "_id", Datatype: "string"},
+		}
+		for _, x := range r.Keys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+			ci = nil //only send on the first
+		}
+	} else {
+		// Column IDs
+		ci := []*pb.ColumnInfo{
+			{Name: "_id", Datatype: "uint64"},
+		}
+		for _, x := range r.Columns() {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+			ci = nil //only send on the first
+		}
+	}
+	return nil
+}
+
+// Roaring returns the row treated as a unified roaring bitmap.
+func (r *Row) Roaring() []byte {
+	bitmaps := make([]*roaring.Bitmap, len(r.segments))
+	for i := range r.segments {
+		bitmaps[i] = r.segments[i].data
+	}
+	return roaring.BitmapsToRoaring(bitmaps)
+}
+
 // IsEmpty returns true if the row doesn't contain any set bits.
 func (r *Row) IsEmpty() bool {
 	if len(r.segments) == 0 {
@@ -194,24 +292,99 @@ func (r *Row) Union(others ...*Row) *Row {
 	return &Row{segments: output}
 }
 
-// Difference returns the diff of r and other.
-func (r *Row) Difference(other *Row) *Row {
+// GenericBinaryOp returns the output of a generic op on r and other.
+func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
 	var segments []rowSegment
-
 	itr := newMergeSegmentIterator(r.segments, other.segments)
 	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s0 == nil {
-			continue
-		} else if s1 == nil {
+		if s1 == nil {
 			segments = append(segments, *s0)
 			continue
+		} else if s0 == nil {
+			segments = append(segments, *s1)
+			continue
 		}
-		segments = append(segments, *s0.Difference(s1))
+		segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
 	}
 
 	return &Row{segments: segments}
 }
 
+// GenericNaryOp returns the output of an nary op on r and others.
+func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
+	segments := make([][]rowSegment, 0, len(others)+1)
+	if len(r.segments) > 0 {
+		segments = append(segments, r.segments)
+	}
+	nextSegs := make([][]rowSegment, 0, len(others)+1)
+	toProcess := make([]*rowSegment, 0, len(others)+1)
+	var output []rowSegment
+	for _, other := range others {
+		if len(other.segments) > 0 {
+			segments = append(segments, other.segments)
+		}
+	}
+	for len(segments) > 0 {
+		shard := segments[0][0].shard
+		for _, segs := range segments {
+			if segs[0].shard < shard {
+				shard = segs[0].shard
+			}
+		}
+		nextSegs = nextSegs[:0]
+		toProcess := toProcess[:0]
+		for _, segs := range segments {
+			if segs[0].shard == shard {
+				toProcess = append(toProcess, &segs[0])
+				segs = segs[1:]
+			}
+			if len(segs) > 0 {
+				nextSegs = append(nextSegs, segs)
+			}
+		}
+		// at this point, "toProcess" is a list of all the segments
+		// sharing the lowest ID, and nextSegs is a list of all the others.
+		// Swap the segment lists (so we don't have to reallocate it)
+		segments, nextSegs = nextSegs, segments
+		output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
+	}
+	return &Row{segments: output}
+}
+
+// Difference returns the diff of r and other.
+func (r *Row) Difference(others ...*Row) *Row {
+	var output []rowSegment
+	o := make(map[uint64][]*rowSegment)
+
+	for x := range others {
+		for y := range others[x].segments {
+			segment := others[x].segments[y]
+			o[segment.shard] = append(o[segment.shard], &segment)
+		}
+	}
+	for _, segment := range r.segments {
+
+		dest, ok := o[segment.shard]
+		if ok {
+			output = append(output, *segment.Difference(dest...))
+		} else {
+			output = append(output, segment)
+		}
+	}
+	return &Row{segments: output}
+}
+
+// GenericUnary returns the results of a generic op on r.
+func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
+	work := r
+	var segments []rowSegment
+	for _, segment := range work.segments {
+		opped := segment.GenericUnaryOp(op, args)
+		segments = append(segments, *opped)
+	}
+	return &Row{segments: segments}
+}
+
 // Shift returns the bitwise shift of r by n bits.
 // Currently only positive shift values are supported.
 func (r *Row) Shift(n int64) (*Row, error) {
@@ -299,6 +472,15 @@ func (r *Row) Count() uint64 {
 	return n
 }
 
+// GenericCount applies an op to lots of things.
+func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
+	var n int64
+	for i := range r.segments {
+		n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
+	}
+	return uint64(n)
+}
+
 // MarshalJSON returns a JSON-encoded byte slice of r.
 func (r *Row) MarshalJSON() ([]byte, error) {
 	var o struct {
@@ -326,6 +508,21 @@ func (r *Row) Columns() []uint64 {
 	return a
 }
 
+// Includes returns true if the row contains the given column.
+func (r *Row) Includes(col uint64) bool {
+	// TODO: improve the efficiency of this method by
+	// performing the column filter at the bitmap level
+	// rather than iterating through the results here.
+	for i := range r.segments {
+		for _, c := range r.segments[i].Columns() {
+			if c == col {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // rowSegment holds a subset of a row.
 // This could point to a mmapped roaring bitmap or an in-memory bitmap. The
 // width of the segment will always match the shard width.
@@ -344,9 +541,20 @@ type rowSegment struct {
 }
 
 func (s *rowSegment) Freeze() {
-	s.data.Freeze()
+	s.data = s.data.Freeze()
 }
 
+/*
+// Raw returns the row segment as a byte slice.
+// It may be used by the gRPC server to deliver results
+// as a roaring bitmap instead of a stream of RowResults.
+func (s *rowSegment) Raw() (uint64, []byte) {
+	var buf bytes.Buffer
+	s.data.WriteTo(&buf)
+	return s.shard, buf.Bytes()
+}
+*/
+
 // Merge adds chunks from other to s.
 // Chunks in s are overwritten if they exist in other.
 func (s *rowSegment) Merge(other *rowSegment) {
@@ -366,7 +574,7 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
 // Intersect returns the itersection of s and other.
 func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
 	data := s.data.Intersect(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -393,9 +601,40 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
 	}
 }
 
+// GenericOp performs a generic op on s and other
+func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
+	data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
+// GenericOp performs a generic op on s and others
+func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
+	bitmaps := make([]ext.Bitmap, len(others)+1)
+	bitmaps[0] = WrapBitmap(s.data)
+	for i, seg := range others {
+		bitmaps[i+1] = WrapBitmap(seg.data)
+	}
+	data := op(bitmaps, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // Difference returns the diff of s and other.
-func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
-	data := s.data.Difference(other.data)
+func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
+	datas := make([]*roaring.Bitmap, len(others))
+	for i, other := range others {
+		datas[i] = other.data
+	}
+	data := s.data.Difference(datas...)
 	data.Freeze()
 
 	return &rowSegment{
@@ -409,7 +648,7 @@ func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
 // Xor returns the xor of s and other.
 func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
 	data := s.data.Xor(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -426,7 +665,7 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "shifting roaring data")
 	}
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -436,6 +675,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	}, nil
 }
 
+// GenericUnary returns s subject to op.
+func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
+	//TODO deal with overflow
+	data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
+
+	return &rowSegment{
+		data:  data,
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // SetBit sets the i-th column of the row.
 func (s *rowSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()