Merge pull request #157 from travisturner/int-fragment-sync-quick-fix

temporary fix for int field replica sync bug

.circleci/config.yml

@@ -8,10 +8,13 @@ defaults: &defaults
 fast-checkout: &fast-checkout
   attach_workspace:
     at: .
+add-github-auth: &add-github-auth
+  run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com"
 jobs:
   setup:
     <<: *defaults
     steps:
+      - *add-github-auth
       - checkout
       - restore_cache:
           keys:
@@ -28,11 +31,13 @@ jobs:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: make check-license-headers
   linter:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
       - run: sudo cp bin/golangci-lint /usr/local/bin/
       - run: make golangci-lint
@@ -40,26 +45,39 @@ jobs:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: make build GOOS=linux GOARCH=arm GOARM=5
       - run: make build GOOS=linux GOARCH=arm GOARM=6
       - run: make build GOOS=linux GOARCH=arm GOARM=7
       - run: make build GOOS=linux GOARCH=arm64
+  test-golang-1.14-rc:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.14-rc
+    steps:
+      - *fast-checkout
+      - *add-github-auth
+      - run: sudo apt-get install lsof
+      - run: make test
   test-golang-1.13: &base-test
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run: make test
   test-golang-1.13-shard22:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run: make test SHARD_WIDTH=22
   test-golang-1.13-race:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run:
           command: make test TESTFLAGS="-race -v -timeout=30m"
@@ -73,6 +91,7 @@ jobs:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run: make test ENTERPRISE=1
   test-golang-1.12:
@@ -81,6 +100,7 @@ jobs:
       - image: circleci/golang:1.12
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run: make test
   test-golang-1.11:
@@ -89,18 +109,21 @@ jobs:
       - image: circleci/golang:1.11
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: sudo apt-get install lsof
       - run: make test
   cluster-tests:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - setup_remote_docker
       - run: make clustertests-build
   prerelease:
     <<: *base-test
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: make prerelease
       - store_artifacts:
           path: build
@@ -111,6 +134,7 @@ jobs:
     <<: *defaults
     steps:
       - *fast-checkout
+      - *add-github-auth
       - run: make release
       - store_artifacts:
           path: build
@@ -123,6 +147,7 @@ jobs:
     steps:
       - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
       - *fast-checkout
+      - *add-github-auth
       - run: sudo pip install awscli
       - run: make prerelease-upload
   dockerhub-upload:
@@ -130,11 +155,23 @@ jobs:
     steps:
       - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
       - *fast-checkout
+      - *add-github-auth
       - setup_remote_docker
       - run: make docker
       - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
       - run: docker push pilosa/pilosa:master
+  docker-enterprise-upload:
+    <<: *defaults
+    steps:
+      #- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - *fast-checkout
+      - *add-github-auth
+      - setup_remote_docker
+      - run: make docker-enterprise
+      - run: docker tag pilosa:$(git describe --tags) molecula.azurecr.io/pilosa-enterprise:unstable
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: docker push molecula.azurecr.io/pilosa-enterprise:unstable
 workflows:
   version: 2
   test:
@@ -149,6 +186,9 @@ workflows:
       - test-build-arm:
           requires:
             - setup
+      - test-golang-1.14-rc:
+          requires:
+            - setup
       - test-golang-1.13-enterprise:
           requires:
             - setup
@@ -185,11 +225,4 @@ workflows:
               only: /^v.*/
             branches:
               ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+

.golangci.yml

@@ -0,0 +1,3 @@
+run:
+  skip-files:
+    - pql/pql.peg.go

Dockerfile

@@ -1,8 +1,11 @@
 FROM golang:1.13.0 as builder
 
+ARG BUILD_FLAGS
+ARG MAKE_FLAGS
+
 COPY . pilosa
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS}
 
 FROM alpine:3.9.4
 

Dockerfile-clustertests

@@ -1,17 +1,14 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.13
 
 LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/pilosa/pilosa/
 
 RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
+    && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

Makefile

@@ -16,8 +16,16 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
 BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
+define LICENSE_HASH_CODE
+    head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
+endef
+LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
+
+PLUGINS=distinct
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export PLUGINS
 
 # Run tests and compile Pilosa
 default: test build
@@ -32,7 +40,7 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
 
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@@ -81,14 +89,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
 # running. This will catch changes to internal/clustertests/*.go, but if you
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
-clustertests:
+clustertests: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) build client1
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
-clustertests-build:
+clustertests-build: vendor
 	docker-compose -f $(DOCKER_COMPOSE) down
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
@@ -115,14 +123,23 @@ generate-stringer:
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+# dunno if protoc-gen-gofast is actually needed here
+generate-proto-grpc: require-protoc require-protoc-gen-gofast
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+
 # `go generate` all needed packages
 generate: generate-protoc generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
-docker:
-	docker build -t "pilosa:$(VERSION)" .
+docker: vendor
+	docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
 	@echo Created docker image: pilosa:$(VERSION)
 
+# Create Docker image from Dockerfile (enterprise)
+docker-enterprise: vendor
+	docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" .
+	@echo Created docker image: pilosa-enterprise:$(VERSION)
+
 # Compile Pilosa inside Docker container
 docker-build:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
@@ -133,7 +150,7 @@ docker-test:
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --skip-files '.*\.peg\.go'
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -161,9 +178,8 @@ gometalinter: require-gometalinter vendor
 # Verify that all Go files have license header
 check-license-headers: SHELL:=/bin/bash
 check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
+	@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
+	    do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
 
 ######################
 # Build dependencies #

api.go

@@ -23,7 +23,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -146,9 +148,11 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 	}
 	execOpts := &execOptions{
 		Remote:          req.Remote,
+		Profile:         req.Profile,
 		ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
 		ExcludeColumns:  req.ExcludeColumns,  // NOTE: Kept for Pilosa 1.x compat.
 		ColumnAttrs:     req.ColumnAttrs,     // NOTE: Kept for Pilosa 1.x compat.
+		EmbeddedData:    req.EmbeddedData,    // precomputed values that needed to be passed with the request
 	}
 	resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
 	if err != nil {
@@ -383,7 +387,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 
 	// only set and time fields are supported
 	if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime {
-		return NewBadRequestError(errors.New("roaring import is only supported for set and time fields"))
+		return NewBadRequestError(errors.Errorf("roaring import is only supported for set and time fields, not '%s' fields.", field.Type()))
 	}
 
 	errCh := make(chan error, len(nodes))
@@ -540,8 +544,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		var colStr string
 		var err error
 
-		if field.keys() {
-			if rowStr, err = field.translateStore.TranslateID(rowID); err != nil {
+		if field.Keys() {
+			// TODO: handle case: field.ForeignIndex
+			if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil {
 				return errors.Wrap(err, "translating row")
 			}
 		} else {
@@ -549,7 +554,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
 		}
 
 		if index.Keys() {
-			if colStr, err = index.translateStore.TranslateID(columnID); err != nil {
+			if store := index.TranslateStore(api.cluster.idPartition(indexName, columnID)); store == nil {
+				return errors.Wrap(err, "partition does not exist")
+			} else if colStr, err = store.TranslateID(columnID); err != nil {
 				return errors.Wrap(err, "translating column")
 			}
 		} else {
@@ -889,10 +896,17 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
 	return attrs, nil
 }
 
-// ImportOptions holds the options for the API.Import method.
+// ImportOptions holds the options for the API.Import
+// method.
+//
+// TODO(2.0) we have entirely missed the point of functional options
+// by exporting this structure. If it needs to be exported for some
+// reason, we should consider not using functional options here which
+// just adds complexity.
 type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
+	Presorted      bool
 }
 
 // ImportOption is a functional option type for API.Import.
@@ -916,6 +930,13 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption {
 	}
 }
 
+func OptImportOptionsPresorted(b bool) ImportOption {
+	return func(o *ImportOptions) error {
+		o.Presorted = b
+		return nil
+	}
+}
+
 // Import bulk imports data into a particular index,field,shard.
 func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
@@ -941,12 +962,12 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
 	// check to see if keys need translation.
 	if !options.IgnoreKeyCheck {
 		// Translate row keys.
-		if field.keys() {
+		if field.Keys() {
 			if len(req.RowIDs) != 0 {
 				return errors.New("row ids cannot be used because field uses string keys")
 			}
-			if req.RowIDs, err = field.translateStore.TranslateKeys(req.RowKeys); err != nil {
-				return errors.Wrap(err, "translating rows")
+			if req.RowIDs, err = api.cluster.translateFieldKeys(ctx, field, req.RowKeys...); err != nil {
+				return errors.Wrapf(err, "translating field keys")
 			}
 		}
 
@@ -955,14 +976,14 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
 		}
 
 		// For translated data, map the columnIDs to shards. If
 		// this node does not own the shard, forward to the node that does.
-		if index.Keys() || field.keys() {
+		if index.Keys() || field.Keys() {
 			m := make(map[uint64][]Bit)
 
 			for i, colID := range req.ColumnIDs {
@@ -1036,6 +1057,10 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 		return errors.Wrap(err, "validating api method")
 	}
 
+	if err := req.Validate(); err != nil {
+		return errors.Wrap(err, "validating import value request")
+	}
+
 	// Set up import options.
 	options, err := setUpImportOptions(opts...)
 	if err != nil {
@@ -1056,60 +1081,138 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
 			if len(req.ColumnIDs) != 0 {
 				return errors.New("column ids cannot be used because index uses string keys")
 			}
-			if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil {
+			if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil {
 				return errors.Wrap(err, "translating columns")
 			}
+			req.Shard = math.MaxUint64
+		}
 
-			// For translated data, map the columnIDs to shards. If
-			// this node does not own the shard, forward to the node that does.
-			m := make(map[uint64][]FieldValue)
-
-			for i, colID := range req.ColumnIDs {
-				shard := colID / ShardWidth
-				if _, ok := m[shard]; !ok {
-					m[shard] = make([]FieldValue, 0)
-				}
-				m[shard] = append(m[shard], FieldValue{
-					Value:    req.Values[i],
-					ColumnID: colID,
-				})
+		// Translate values when the field uses keys (for example, when
+		// the field has a ForeignIndex with keys).
+		if field.Keys() {
+			// Perform translation.
+			uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues)
+			if err != nil {
+				return err
 			}
 
-			// Signal to the receiving nodes to ignore checking for key translation.
-			opts = append(opts, OptImportOptionsIgnoreKeyCheck(true))
-
-			var eg errgroup.Group
-			for shard, vals := range m {
-				// TODO: if local node owns this shard we don't need to go through the client
-				shard := shard
-				vals := vals
-				eg.Go(func() error {
-					return api.server.defaultClient.ImportValue(ctx, req.Index, req.Field, shard, vals, opts...)
-				})
+			// Because the BSI field supports negative values, we have to
+			// convert the uint64 keys to a slice of int64.
+			ints := make([]int64, len(uints))
+			for i := range uints {
+				ints[i] = int64(uints[i])
 			}
-			return eg.Wait()
+			req.Values = ints
 		}
 	}
 
-	// Validate shard ownership.
-	if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
+	if !options.Presorted {
+		sort.Sort(req)
+	}
+
+	// if we're importing into a specific shard
+	if req.Shard != math.MaxUint64 {
+		// Check that column IDs match the stated shard.
+		if s1, s2 := req.ColumnIDs[0]/ShardWidth, req.ColumnIDs[len(req.ColumnIDs)-1]/ShardWidth; s1 != s2 && s2 != req.Shard {
+			return errors.Errorf("shard %d specified, but import spans shards %d to %d", req.Shard, s1, s2)
+		}
+		// Validate shard ownership. TODO - we should forward to the
+		// correct node rather than barfing here.
+		if err := api.validateShardOwnership(req.Index, req.Shard); err != nil {
+			return errors.Wrap(err, "validating shard ownership")
+		}
+		// Import columnIDs into existence field.
+		if !options.Clear {
+			if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
+				api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+				return errors.Wrap(err, "importing existence columns")
+			}
+		}
+
+		// Import into fragment.
+		if len(req.Values) > 0 {
+			err = field.importValue(req.ColumnIDs, req.Values, options)
+			if err != nil {
+				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+			}
+		} else if len(req.FloatValues) > 0 {
+			err = field.importFloatValue(req.ColumnIDs, req.FloatValues, options)
+			if err != nil {
+				api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+			}
+		}
+
+		return errors.Wrap(err, "importing value")
+	}
+
+	options.IgnoreKeyCheck = true
+	start := 0
+	shard := req.ColumnIDs[0] / ShardWidth
+	var eg errgroup.Group // TODO make this a pooled errgroup
+	for i, colID := range req.ColumnIDs {
+		if colID/ShardWidth != shard {
+			subreq := &ImportValueRequest{
+				Index:     req.Index,
+				Field:     req.Field,
+				Shard:     shard,
+				ColumnIDs: req.ColumnIDs[start:i],
+			}
+			if req.Values != nil {
+				subreq.Values = req.Values[start:i]
+			} else if req.FloatValues != nil {
+				subreq.FloatValues = req.FloatValues[start:i]
+			}
+
+			eg.Go(func() error {
+				return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+			})
+			start = i
+			shard = colID / ShardWidth
+		}
+	}
+	subreq := &ImportValueRequest{
+		Index:     req.Index,
+		Field:     req.Field,
+		Shard:     shard,
+		ColumnIDs: req.ColumnIDs[start:],
+	}
+	if req.Values != nil {
+		subreq.Values = req.Values[start:]
+	} else if req.FloatValues != nil {
+		subreq.FloatValues = req.FloatValues[start:]
+	}
+	eg.Go(func() error {
+		// TODO we should elevate the logic for figuring out which
+		// node(s) to send to into API instead of having those details
+		// in the client implementation.
+		return api.server.defaultClient.ImportValue2(ctx, subreq, options)
+	})
+	return eg.Wait()
+
+}
+
+func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error {
+	span, _ := tracing.StartSpanFromContext(ctx, "API.ImportColumnAttrs")
+	defer span.Finish()
+
+	index, err := api.Index(ctx, req.Index)
+	if err != nil {
+		return errors.Wrap(err, "getting index")
+	}
+
+	if err := api.validateShardOwnership(req.Index, uint64(req.Shard)); err != nil {
 		return errors.Wrap(err, "validating shard ownership")
 	}
 
-	// Import columnIDs into existence field.
-	if !options.Clear {
-		if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
-			api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
-			return errors.Wrap(err, "importing existence columns")
-		}
+	bulkAttrs := make(map[uint64]map[string]interface{})
+	for n := 0; n < len(req.ColumnIDs); n++ {
+		bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}
 	}
-
-	// Import into fragment.
-	err = field.importValue(req.ColumnIDs, req.Values, options)
-	if err != nil {
-		api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+	if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil {
+		api.server.logger.Printf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing column attrs")
 	}
-	return errors.Wrap(err, "importing")
+	return nil
 }
 
 func importExistenceColumns(index *Index, columnIDs []uint64) error {
@@ -1286,14 +1389,75 @@ func (api *API) Info() serverInfo {
 }
 
 // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
-func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) {
+func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")
 	defer span.Finish()
-	return api.holder.TranslateEntryReader(ctx, offsets)
+
+	// Ensure all readers are cleaned up if any error.
+	var a []TranslateEntryReader
+	defer func() {
+		if err != nil {
+			for i := range a {
+				a[i].Close()
+			}
+		}
+	}()
+
+	// Fetch all index partition readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for partitionID, offset := range indexMap.Partitions {
+			store := index.TranslateStore(partitionID)
+			if store == nil {
+				return nil, ErrTranslateStoreNotFound
+			}
+
+			r, err := store.EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "index partition translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	// Fetch all field readers.
+	for indexName, indexMap := range offsets {
+		index := api.holder.Index(indexName)
+		if index == nil {
+			return nil, ErrIndexNotFound
+		}
+
+		for fieldName, offset := range indexMap.Fields {
+			field := index.Field(fieldName)
+			if field == nil {
+				return nil, ErrFieldNotFound
+			}
+
+			r, err := field.TranslateStore().EntryReader(ctx, uint64(offset))
+			if err != nil {
+				return nil, errors.Wrap(err, "field translate reader")
+			}
+			a = append(a, r)
+		}
+	}
+
+	return NewMultiTranslateEntryReader(ctx, a), nil
+}
+
+func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	return api.cluster.translateIndexKey(ctx, indexName, key)
+}
+
+func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	return api.cluster.translateIndexIDs(ctx, indexName, ids)
 }
 
 // TranslateKeys handles a TranslateKeyRequest.
-func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
+func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) {
 	var req TranslateKeysRequest
 	if buf, err := ioutil.ReadAll(r); err != nil {
 		return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error"))
@@ -1302,13 +1466,22 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	}
 
 	// Lookup store for either index or field and translate keys.
-	store, err := api.holder.TranslateStore(req.Index, req.Field)
-	if err != nil {
-		return nil, err
-	}
-	ids, err := store.TranslateKeys(req.Keys)
-	if err != nil {
-		return nil, err
+	var ids []uint64
+	if req.Field == "" {
+		if ids, err = api.cluster.translateIndexKeys(ctx, req.Index, req.Keys); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			ids, err = api.cluster.translateIndexKeys(ctx, fi, req.Keys)
+			if err != nil {
+				return nil, err
+			}
+		} else if ids, err = api.cluster.translateFieldKeys(ctx, field, req.Keys...); err != nil {
+			return nil, errors.Wrapf(err, "translating field keys")
+		}
 	}
 
 	// Encode response.
@@ -1319,6 +1492,42 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) {
 	return buf, nil
 }
 
+// TranslateIDs handles a TranslateIDRequest.
+func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) {
+	var req TranslateIDsRequest
+	if buf, err := ioutil.ReadAll(r); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error"))
+	} else if err := api.Serializer.Unmarshal(buf, &req); err != nil {
+		return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error"))
+	}
+
+	// Lookup store for either index or field and translate ids.
+	var keys []string
+	if req.Field == "" {
+		if keys, err = api.cluster.translateIndexIDs(ctx, req.Index, req.IDs); err != nil {
+			return nil, err
+		}
+	} else {
+		if field := api.holder.Field(req.Index, req.Field); field == nil {
+			return nil, ErrFieldNotFound
+		} else if fi := field.ForeignIndex(); fi != "" {
+			keys, err = api.cluster.translateIndexIDs(ctx, fi, req.IDs)
+			if err != nil {
+				return nil, err
+			}
+		} else if keys, err = field.TranslateStore().TranslateIDs(req.IDs); err != nil {
+			return nil, err
+		}
+	}
+
+	// Encode response.
+	buf, err := api.Serializer.Marshal(&TranslateIDsResponse{Keys: keys})
+	if err != nil {
+		return nil, errors.Wrap(err, "translate ids response encoding error")
+	}
+	return buf, nil
+}
+
 // PrimaryReplicaNodeURL returns the URL of the cluster's primary replica.
 func (api *API) PrimaryReplicaNodeURL() url.URL {
 	node := api.cluster.PrimaryReplicaNode()

api/client/grpc.go

@@ -0,0 +1,181 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"sync"
+
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTarget string
+	tlsConfig  *tls.Config
+
+	mu   sync.RWMutex
+	conn *grpc.ClientConn
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTarget string, tlsConfig *tls.Config) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTarget: dialTarget,
+		tlsConfig:  tlsConfig,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTarget, opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		// TODO: log this error with logger
+		log.Printf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -30,6 +31,137 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+// attrFun defines a mapping from columnID -> attr value
+func attrFun(id uint64) string {
+	//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func TestAPI_ImportColumnAttrs(t *testing.T) {
+	/*
+	   columns seconds
+	   100      1.150
+	   1000     1.568
+	   10000    5.156
+	   100000  38.179
+	*/
+	c := test.MustRunCluster(t, 2,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node0"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID("node1"),
+				pilosa.OptServerClusterHasher(&offsetModHasher{}),
+			)},
+	)
+	defer c.Close()
+
+	m0 := c[0]
+	m1 := c[1]
+	t.Run("ImportColumnAttrs", func(t *testing.T) {
+		ctx := context.Background()
+		index := "i"
+		field := "f"
+		attrKey := "k"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some attrs for two shards
+		numAttrs := 100
+		columnIDs0 := make([]uint64, 0, numAttrs)
+		attrVals0 := make([]string, 0, numAttrs)
+		columnIDs1 := make([]uint64, 0, numAttrs)
+		attrVals1 := make([]string, 0, numAttrs)
+		for n := 0; n < 1000000; n += 1000000 / numAttrs {
+			columnIDs0 = append(columnIDs0, uint64(n))
+			val0 := attrFun(uint64(n))
+			attrVals0 = append(attrVals0, val0)
+			setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field)
+			if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil {
+				t.Fatal(err)
+			}
+
+			columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
+			val1 := attrFun(uint64(n + ShardWidth))
+			attrVals1 = append(attrVals1, val1)
+			setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field)
+			if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// send shard0 to node1
+		req := &pilosa.ImportColumnAttrsRequest{
+			AttrKey:   attrKey,
+			ColumnIDs: columnIDs0,
+			AttrVals:  attrVals0,
+			Shard:     0,
+			Index:     index,
+		}
+
+		if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// send shard1 to node0
+		req = &pilosa.ImportColumnAttrsRequest{
+			AttrKey:   attrKey,
+			ColumnIDs: columnIDs1,
+			AttrVals:  attrVals1,
+			Shard:     1,
+			Index:     index,
+		}
+
+		if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query node0.
+		pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+		// Query node1.
+		pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
+		res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.ColumnAttrSets) != 100 {
+			t.Fatal("incorrect number of column attrs set")
+		}
+
+		for _, v := range res.ColumnAttrSets {
+			attrVal := attrFun(v.ID)
+			if attrVal != v.Attrs[attrKey] {
+				t.Fatal(err)
+			}
+		}
+
+	})
+}
+
 func TestAPI_Import(t *testing.T) {
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
@@ -71,14 +203,15 @@ func TestAPI_Import(t *testing.T) {
 
 		// Generate some keyed records.
 		rowIDs := []uint64{}
-		colKeys := []string{}
 		timestamps := []int64{}
 		for i := 1; i <= 10; i++ {
 			rowIDs = append(rowIDs, rowID)
 			timestamps = append(timestamps, timestamp)
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Keys are sharded so ordering is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportRequest{
@@ -129,58 +262,6 @@ func TestAPI_Import(t *testing.T) {
 		}
 
 	})
-
-	t.Run("RowKeyColumnID", func(t *testing.T) {
-		ctx := context.Background()
-		index := "rkci"
-		field := "f"
-
-		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-		if err != nil {
-			t.Fatalf("creating index: %v", err)
-		}
-		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys())
-		if err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-
-		rowKey := "rowkey"
-
-		// Generate some keyed records.
-		rowKeys := []string{rowKey, rowKey, rowKey}
-		colIDs := []uint64{1, 2, pilosa.ShardWidth + 1}
-		timestamps := []int64{0, 0, 0}
-
-		// Import data with keys to the coordinator (node0) and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
-		req := &pilosa.ImportRequest{
-			Index:      index,
-			Field:      field,
-			Shard:      0,
-			RowKeys:    rowKeys,
-			ColumnIDs:  colIDs,
-			Timestamps: timestamps,
-		}
-		if err := m0.API.Import(ctx, req); err != nil {
-			t.Fatal(err)
-		}
-
-		pql := fmt.Sprintf("Row(%s=%s)", field, rowKey)
-
-		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-
-		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
-		}
-	})
 }
 
 func TestAPI_ImportValue(t *testing.T) {
@@ -219,12 +300,13 @@ func TestAPI_ImportValue(t *testing.T) {
 
 		// Generate some keyed records.
 		values := []int64{}
-		colKeys := []string{}
 		for i := 1; i <= 10; i++ {
 			values = append(values, int64(i))
-			colKeys = append(colKeys, fmt.Sprintf("col%d", i))
 		}
 
+		// Column keys are sharded so their order is not guaranteed.
+		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+
 		// Import data with keys to the coordinator (node0) and verify that it gets
 		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
 		req := &pilosa.ImportValueRequest{
@@ -258,6 +340,202 @@ func TestAPI_ImportValue(t *testing.T) {
 			t.Fatal(err)
 		}
 	})
+
+	t.Run("ValDecimalField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdec"
+		field := "fdec"
+
+		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		fld, err := m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)+0.1)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+		if err := m1.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf("Row(%s>6)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: %+v", ids)
+		}
+
+		sum, count, err := fld.FloatSum(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatsum: %v", err)
+		} else if sum != 0.1+1.1+2.1+3.1+4.1+5.1+6.1+7.1+8.1+9.1 {
+			t.Fatalf("unexpected sum: %f", sum)
+		} else if count != 10 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		min, count, err := fld.FloatMin(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatmin: %v", err)
+		} else if min != 0.1 {
+			t.Fatalf("unexpected min: %f", min)
+		} else if count != 1 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		max, count, err := fld.FloatMax(nil, field)
+		if err != nil {
+			t.Fatalf("getting floatmax: %v", err)
+		} else if max != 9.1 {
+			t.Fatalf("unexpected max: %f", max)
+		} else if count != 1 {
+			t.Fatalf("unexpected count: %d", count)
+		}
+
+		val, exists, err := fld.FloatValue(1)
+		if err != nil {
+			t.Fatalf("unepxected err getting floatvalue")
+		} else if !exists {
+			t.Fatalf("column 1 should exist")
+		} else if val != 1.1 {
+			t.Fatalf("unexpected floatvalue %f", val)
+		}
+
+		changed, err := fld.SetFloatValue(11, 11.1)
+		if err != nil {
+			t.Fatalf("setting float value: %v", err)
+		} else if !changed {
+			t.Fatalf("expected change")
+		}
+
+		val, exists, err = fld.FloatValue(11)
+		if err != nil {
+			t.Fatalf("getting float val: %v", err)
+		} else if !exists {
+			t.Fatalf("should exist")
+		} else if val != 11.1 {
+			t.Fatalf("unexpected val: %f", 11.1)
+		}
+	})
+
+	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valdecneg"
+		field := "fdecneg"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []float64{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			values = append(values, float64(i)*100+10)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:       index,
+			Field:       field,
+			ColumnIDs:   colIDs,
+			FloatValues: values,
+		}
+		if err := m1.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf("Row(%s>600)", field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) {
+			t.Fatalf("unexpected column keys: %+v", ids)
+		}
+
+	})
+
+	t.Run("ValStringField", func(t *testing.T) {
+		ctx := context.Background()
+		index := "valstr"
+		field := "fstr"
+
+		fgnIndex := "fgnvalstr"
+
+		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+
+		_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
+		if err != nil {
+			t.Fatalf("creating foreign index: %v", err)
+		}
+		_, err = m0.API.CreateField(ctx, index, field,
+			pilosa.OptFieldTypeInt(0, math.MaxInt64),
+			pilosa.OptFieldForeignIndex(fgnIndex),
+		)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		// Generate some keyed records.
+		values := []string{}
+		colIDs := []uint64{}
+		for i := 0; i < 10; i++ {
+			value := fmt.Sprintf("strval-%d", (i)*100+10)
+			values = append(values, value)
+			colIDs = append(colIDs, uint64(i))
+		}
+
+		// Import data with keys to the coordinator (node0) and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportValueRequest{
+			Index:        index,
+			Field:        field,
+			ColumnIDs:    colIDs,
+			StringValues: values,
+		}
+		if err := m0.API.ImportValue(ctx, req); err != nil {
+			t.Fatal(err)
+		}
+
+		pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
+
+		// Query node0.
+		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+			t.Fatal(err)
+		} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
+			t.Fatalf("unexpected columns: %+v", ids)
+		}
+	})
 }
 
 // offsetModHasher represents a simple, mod-based hashing offset by 1.

boltdb/translate.go

@@ -11,9 +11,11 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package boltdb
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"path/filepath"
@@ -32,8 +34,8 @@ var (
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
+	s := NewTranslateStore(index, field, partitionID, partitionN)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,12 +47,22 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
-	index string
-	field string
+	index       string
+	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
@@ -63,10 +75,12 @@ type TranslateStore struct {
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
 	return &TranslateStore{
 		index:       index,
 		field:       field,
+		partitionID: partitionID,
+		partitionN:  partitionN,
 		closing:     make(chan struct{}),
 		writeNotify: make(chan struct{}),
 	}
@@ -108,6 +122,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,12 +154,12 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
+// TranslateKey converts a string key to an integer ID.
 // If key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	// Find id by key under read lock.
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
+		id, _ = findIDByKey(tx.Bucket([]byte("keys")), key)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -156,13 +175,16 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	var written bool
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
 		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
+
+		var boltKey []byte
+		if id, boltKey = findIDByKey(bkt, key); id != 0 {
 			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
+		}
+
+		id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+		if err := bkt.Put(boltKey, u64tob(id)); err != nil {
 			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil {
 			return err
 		}
 		written = true
@@ -178,8 +200,8 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
 	return id, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
+// TranslateKeys converts a slice of string keys to a slice of integer IDs.
+// If a key does not have an associated id then one is created.
 func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if len(keys) == 0 {
 		return nil, nil
@@ -193,7 +215,7 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket([]byte("keys"))
 		for i, key := range keys {
-			if id := findIDByKey(bkt, key); id != 0 {
+			if id, _ := findIDByKey(bkt, key); id != 0 {
 				ids[i] = id
 				found++
 			}
@@ -218,13 +240,15 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
 				continue
 			}
 
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
+			var boltKey []byte
+			if ids[i], boltKey = findIDByKey(bkt, key); ids[i] != 0 {
 				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
+			}
+
+			ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+			if err := bkt.Put(boltKey, u64tob(ids[i])); err != nil {
 				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
+			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), boltKey); err != nil {
 				return err
 			}
 			written = true
@@ -286,7 +310,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +335,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
-			max = btou64(key)
-		}
+		max = maxID(tx)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +343,14 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -387,13 +417,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
-	if value := bkt.Get([]byte(key)); value != nil {
-		return btou64(value)
+// emptyKey is a sentinel byte slice which stands for "" as a key.
+var emptyKey = []byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }

boltdb/translate_test.go

@@ -28,24 +28,24 @@ func TestTranslateStore_TranslateKey(t *testing.T) {
 	s := MustOpenNewTranslateStore()
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
+	// Ensure initial key translates to first ID for shard
 	if id, err := s.TranslateKey("foo"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, uint64(247463937); got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure next key autoincrements.
 	if id, err := s.TranslateKey("bar"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
+	} else if got, want := id, uint64(247463938); got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating existing key returns original ID.
 	if id, err := s.TranslateKey("foo"); err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
+	} else if got, want := id, uint64(247463937); got != want {
 		t.Fatalf("TranslateKey()=%d, want %d", got, want)
 	}
 }
@@ -57,29 +57,29 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
 	// Ensure initial keys translate to incrementing IDs.
 	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], uint64(247463937); got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], uint64(247463938); got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslation returns original IDs.
 	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], uint64(247463937); got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
+	} else if got, want := ids[1], uint64(247463938); got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
 	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
+	} else if got, want := ids[0], uint64(247463937); got != want {
 		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
+	} else if got, want := ids[1], uint64(247463939); got != want {
 		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
+	} else if got, want := ids[2], uint64(247463938); got != want {
 		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
 	}
 }
@@ -89,24 +89,38 @@ func TestTranslateStore_TranslateID(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
+	id1, err := s.TranslateKey("foo")
+	if err != nil {
 		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	}
+	id2, err := s.TranslateKey("bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	id3, err := s.TranslateKey("")
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
+	if key, err := s.TranslateID(id1); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "foo"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
 
-	if key, err := s.TranslateID(2); err != nil {
+	if key, err := s.TranslateID(id2); err != nil {
 		t.Fatal(err)
 	} else if got, want := key, "bar"; got != want {
 		t.Fatalf("TranslateID()=%s, want %s", got, want)
 	}
+
+	if key, err := s.TranslateID(id3); err != nil {
+		t.Fatal(err)
+	} else if got, want := key, ""; got != want {
+		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	}
+
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
@@ -114,12 +128,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.TranslateKeys([]string{"foo", "bar"})
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -151,7 +166,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, uint64(247463937); got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,7 +175,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, uint64(247463938); got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -174,7 +189,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, uint64(247463939); got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -211,7 +226,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, uint64(247463937); got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -302,7 +317,7 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
 	s.Path = f.Name()
 	return s
 }

cache.go

@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -172,6 +173,7 @@ func (c *rankCache) Add(id uint64, n uint64) {
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -185,6 +187,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
@@ -316,10 +319,22 @@ func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +410,18 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 

cache_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/pilosa/pilosa/v2"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
 }

client.go

@@ -44,6 +44,8 @@ type FieldValue struct {
 // While I understand that putting the entire Client behind an interface might require this many methods,
 // I don't want to let it go unquestioned.
 type InternalClient interface {
+	InternalQueryClient
+
 	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
 	Schema(ctx context.Context) ([]*IndexInfo, error)
 	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
@@ -51,7 +53,6 @@ type InternalClient interface {
 	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
 	Nodes(ctx context.Context) ([]*Node, error)
 	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
 	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
 	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
 	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@@ -59,6 +60,7 @@ type InternalClient interface {
 	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
 	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
 	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
+	ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error
 	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
 	CreateField(ctx context.Context, index, field string) error
 	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
@@ -69,6 +71,7 @@ type InternalClient interface {
 	SendMessage(ctx context.Context, uri *URI, msg []byte) error
 	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
 	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
+	ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
 }
 
 //===============
@@ -76,6 +79,8 @@ type InternalClient interface {
 // InternalQueryClient is the internal interface for querying a node.
 type InternalQueryClient interface {
 	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
+	TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error)
+	TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
 }
 
 type nopInternalQueryClient struct{}
@@ -84,6 +89,14 @@ func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index
 	return nil, nil
 }
 
+func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+
+func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
+
 func newNopInternalQueryClient() *nopInternalQueryClient {
 	return &nopInternalQueryClient{}
 }
@@ -123,15 +136,30 @@ func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest
 func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
 	return nil, nil
 }
+func (n nopInternalClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) {
+	return nil, nil
+}
+func (n nopInternalClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
+	return nil, nil
+}
 func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
 func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
 	return nil
 }
+func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error {
+	return nil
+}
+
 func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
 	return nil
 }
+
+func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
+	return nil
+}
+
 func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
 	return nil
 }

cluster.go

@@ -40,8 +40,8 @@ import (
 )
 
 const (
-	// defaultPartitionN is the default number of partitions in a cluster.
-	defaultPartitionN = 256
+	// DefaultPartitionN is the default number of partitions in a cluster.
+	DefaultPartitionN = 256
 
 	// ClusterState represents the state returned in the /status endpoint.
 	ClusterStateStarting = "STARTING"
@@ -224,6 +224,8 @@ type cluster struct { // nolint: maligned
 
 	abortAntiEntropyCh chan struct{}
 
+	translationSyncer translationSyncer
+
 	mu         sync.RWMutex
 	jobs       map[int64]*resizeJob
 	currentJob *resizeJob
@@ -241,7 +243,7 @@ type cluster struct { // nolint: maligned
 func newCluster() *cluster {
 	return &cluster{
 		Hasher:     &jmphasher{},
-		partitionN: defaultPartitionN,
+		partitionN: DefaultPartitionN,
 		ReplicaN:   1,
 
 		joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
@@ -249,6 +251,8 @@ func newCluster() *cluster {
 		closing:             make(chan struct{}),
 		joining:             make(chan struct{}),
 
+		translationSyncer: NopTranslationSyncer,
+
 		InternalClient: newNopInternalClient(),
 
 		logger: logger.NopLogger,
@@ -390,14 +394,6 @@ func (c *cluster) addNode(node *Node) error {
 		return nil
 	}
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// add to topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -417,14 +413,6 @@ func (c *cluster) removeNode(nodeID string) error {
 	// remove from cluster
 	c.removeNodeBasicSorted(nodeID)
 
-	// If the cluster membership has changed, reset the primary for
-	// translate store replication.
-	if c.holder != nil {
-		if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil {
-			return err
-		}
-	}
-
 	// remove from topology
 	if c.Topology == nil {
 		return fmt.Errorf("Cluster.Topology is nil")
@@ -477,7 +465,7 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	switch state {
 	case ClusterStateNormal, ClusterStateDegraded:
-		// If state is RESIZING -> NORMAL then run cleanup.
+		// If state is RESIZING -> [NORMAL, DEGRADED] then run cleanup.
 		if c.state == ClusterStateResizing {
 			doCleanup = true
 		}
@@ -485,7 +473,17 @@ func (c *cluster) unprotectedSetState(state string) {
 
 	c.state = state
 
-	if state == ClusterStateResizing {
+	switch state {
+	case ClusterStateNormal:
+		// Because the cluster state is changing to NORMAL,
+		// we [potentially] need to reset the translation sync.
+		// If, for example, the cluster has changed size and is
+		// now settling to NORMAL, the partition ownership may
+		// have changed, and this will force that to be recalculated.
+		if err := c.translationSyncer.Reset(); err != nil {
+			c.logger.Printf("error resetting translation syncer: %s", err)
+		}
+	case ClusterStateResizing:
 		c.abortAntiEntropy()
 	}
 
@@ -867,8 +865,12 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour
 	return m, nil
 }
 
-// partition returns the partition that a shard belongs to.
-func (c *cluster) partition(index string, shard uint64) int {
+// shardPartition returns the partition that a shard belongs to.
+func (c *cluster) shardPartition(index string, shard uint64) int {
+	return shardPartition(index, shard, c.partitionN)
+}
+
+func shardPartition(index string, shard uint64, partitionN int) int {
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], shard)
 
@@ -876,7 +878,25 @@ func (c *cluster) partition(index string, shard uint64) int {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(index))
 	_, _ = h.Write(buf[:])
-	return int(h.Sum64() % uint64(c.partitionN))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// keyPartition returns the partition that a key belongs to.
+func (c *cluster) keyPartition(index, key string) int {
+	return keyPartition(index, key, c.partitionN)
+}
+
+func keyPartition(index, key string, partitionN int) int {
+	// Hash the bytes and mod by partition count.
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(index))
+	_, _ = h.Write([]byte(key))
+	return int(h.Sum64() % uint64(partitionN))
+}
+
+// idPartition returns the partition that an id belongs to.
+func (c *cluster) idPartition(index string, id uint64) int {
+	return shardPartition(index, id/ShardWidth, c.partitionN)
 }
 
 // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use.
@@ -888,7 +908,19 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
 
 // shardNodes returns a list of nodes that own a fragment. unprotected
 func (c *cluster) shardNodes(index string, shard uint64) []*Node {
-	return c.partitionNodes(c.partition(index, shard))
+	return c.partitionNodes(c.shardPartition(index, shard))
+}
+
+// KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use.
+func (c *cluster) KeyNodes(index, key string) []*Node {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.keyNodes(index, key)
+}
+
+// keyNodes returns a list of nodes that own a key. unprotected
+func (c *cluster) keyNodes(index, key string) []*Node {
+	return c.partitionNodes(c.keyPartition(index, key))
 }
 
 // ownsShard returns true if a host owns a fragment.
@@ -900,7 +932,6 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
 
 // partitionNodes returns a list of nodes that own a partition. unprotected.
 func (c *cluster) partitionNodes(partitionID int) []*Node {
-
 	// Default replica count to between one and the number of nodes.
 	// The replica count can be zero if there are no nodes.
 	replicaN := c.ReplicaN
@@ -922,11 +953,18 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
 	return nodes
 }
 
+// ownsPartition returns true if a host owns a partition.
+func (c *cluster) ownsPartition(nodeID string, partition int) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return Nodes(c.partitionNodes(partition)).ContainsID(nodeID)
+}
+
 // containsShards is like OwnsShards, but it includes replicas.
 func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 {
 	var shards []uint64
 	availableShards.ForEach(func(i uint64) {
-		p := c.partition(index, i)
+		p := c.shardPartition(index, i)
 		// Determine the nodes for partition.
 		nodes := c.partitionNodes(p)
 		for _, n := range nodes {
@@ -2047,6 +2085,180 @@ func (c *cluster) setStatic(hosts []string) error {
 	return nil
 }
 
+// translateFieldKey gets a single key from translateFieldKeys.
+func (c *cluster) translateFieldKey(ctx context.Context, field *Field, key string) (uint64, error) {
+	ids, err := c.translateFieldKeys(ctx, field, key)
+	if err != nil {
+		return 0, err
+	} else if len(ids) == 0 {
+		return 0, errors.New("translating key on coordinator returned empty set")
+	}
+	return ids[0], nil
+}
+
+// translateFieldKeys is basically a wrapper around
+// field.TranslateStore().TranslateKey(key), but in
+// the case where the local node's translate store
+// is read-only (i.e. it's not the primary translate
+// store), then this method will forward the translation
+// request to the coordinator.
+func (c *cluster) translateFieldKeys(ctx context.Context, field *Field, keys ...string) ([]uint64, error) {
+	ids, err := field.TranslateStore().TranslateKeys(keys)
+	// If we get a "read only" error, then forward the request
+	// to the coordinator.
+	if errors.Cause(err) == ErrTranslateStoreReadOnly {
+		coordinatorNode := c.coordinatorNode()
+		if ids, err := c.InternalClient.TranslateKeysNode(ctx, &coordinatorNode.URI, field.Index(), field.Name(), keys); err != nil {
+			return ids, errors.Wrap(err, "translating keys on coordinator")
+		} else {
+			return ids, nil
+		}
+	}
+	return ids, err
+}
+
+func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) {
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}})
+	if err != nil {
+		return 0, err
+	}
+	return keyMap[key], nil
+}
+
+func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys []string) ([]uint64, error) {
+	keySet := make(map[string]struct{})
+	for _, key := range keys {
+		keySet[key] = struct{}{}
+	}
+
+	keyMap, err := c.translateIndexKeySet(ctx, indexName, keySet)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make([]uint64, len(keys))
+	for i := range keys {
+		ids[i] = keyMap[keys[i]]
+	}
+	return ids, nil
+}
+
+func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}) (map[string]uint64, error) {
+	keyMap := make(map[string]uint64)
+
+	idx := c.holder.Index(indexName)
+	if idx == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split keys by partition.
+	keysByPartition := make(map[int][]string, c.partitionN)
+	for key := range keySet {
+		partitionID := c.keyPartition(indexName, key)
+		keysByPartition[partitionID] = append(keysByPartition[partitionID], key)
+	}
+
+	// Translate keys by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range keysByPartition {
+		partitionID := partitionID
+		keys := keysByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var ids []uint64
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, indexName, "", keys); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range keys {
+				keyMap[keys[i]] = ids[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return keyMap, nil
+}
+
+func (c *cluster) translateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) {
+	idSet := make(map[uint64]struct{})
+	for _, id := range ids {
+		idSet[id] = struct{}{}
+	}
+
+	idMap, err := c.translateIndexIDSet(ctx, indexName, idSet)
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, len(ids))
+	for i := range ids {
+		keys[i] = idMap[ids[i]]
+	}
+	return keys, nil
+}
+
+func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idSet map[uint64]struct{}) (map[uint64]string, error) {
+	idMap := make(map[uint64]string)
+
+	index := c.holder.Index(indexName)
+	if index == nil {
+		return nil, ErrIndexNotFound
+	}
+
+	// Split ids by partition.
+	idsByPartition := make(map[int][]uint64, c.partitionN)
+	for id := range idSet {
+		partitionID := c.idPartition(indexName, id)
+		idsByPartition[partitionID] = append(idsByPartition[partitionID], id)
+	}
+
+	// Translate ids by partition.
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := range idsByPartition {
+		partitionID := partitionID
+		ids := idsByPartition[partitionID]
+
+		g.Go(func() (err error) {
+			var keys []string
+			if c.ownsPartition(c.Node.ID, partitionID) {
+				if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil {
+					return err
+				}
+			} else {
+				nodes := c.partitionNodes(partitionID)
+				if keys, err = c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, indexName, "", ids); err != nil {
+					return err
+				}
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			for i := range ids {
+				idMap[ids[i]] = keys[i]
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return idMap, nil
+}
+
 // ClusterStatus describes the status of the cluster including its
 // state and node topology.
 type ClusterStatus struct {

cluster_internal_test.go

@@ -96,7 +96,7 @@ func newIndexWithTempPath(name string) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	index, err := NewIndex(path, name, DefaultPartitionN)
 	if err != nil {
 		panic(err)
 	}
@@ -158,6 +158,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node3)
 
 	idx := newIndexWithTempPath("i")
+	defer idx.Close()
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
@@ -351,7 +352,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := c.shardPartition(index, shard)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -948,6 +949,9 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
 
 }
 func TestCluster_confirmNodeDownTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	r := mux.NewRouter()
 	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
@@ -973,10 +977,12 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
 	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
 		t.Errorf("expected node to be down")
 	}
-
 }
 
 func TestCluster_confirmNodeDownDown(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
 	uri := URI{}
 	uri.Scheme = "http"
 	uri.Host = "DoesntMatter"
@@ -985,5 +991,4 @@ func TestCluster_confirmNodeDownDown(t *testing.T) {
 	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
 		t.Errorf("expected node to be down")
 	}
-
 }

cmd/import.go

@@ -53,7 +53,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
 	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
 	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")

cmd/server_test.go

@@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
 	tests := []commandTest{
 		// TEST 0
 		{
-			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"},
+			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
 			env: map[string]string{
 				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
 				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
@@ -53,6 +53,7 @@ func TestServerConfig(t *testing.T) {
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	max-writes-per-request = 3000
 
 	[cluster]
@@ -96,6 +97,7 @@ func TestServerConfig(t *testing.T) {
 			},
 			cfgFileContent: `
 	bind = "localhost:0"
+	bind-grpc = "localhost:0"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		disabled = true
@@ -122,6 +124,7 @@ func TestServerConfig(t *testing.T) {
 			env:  map[string]string{},
 			cfgFileContent: `
 	bind = "localhost:19444"
+	bind-grpc = "localhost:29444"
 	data-dir = "` + actualDataDir + `"
 	[cluster]
 		hosts = [

ctl/import.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -102,11 +103,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
 			if cmd.FieldOptions.TimeQuantum != "" {
-				cmd.FieldOptions.Type = "time"
+				cmd.FieldOptions.Type = pilosa.FieldTypeTime
 			} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
-				cmd.FieldOptions.Type = "int"
+				cmd.FieldOptions.Type = pilosa.FieldTypeInt
 			} else {
-				cmd.FieldOptions.Type = "set"
+				cmd.FieldOptions.Type = pilosa.FieldTypeSet
+				cmd.FieldOptions.CacheType = pilosa.CacheTypeRanked
+				cmd.FieldOptions.CacheSize = pilosa.DefaultCacheSize
 			}
 		}
 		err := cmd.ensureSchema(ctx)
@@ -163,8 +166,8 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
 // importPath parses a path into bits and imports it to the server.
 func (cmd *ImportCommand) importPath(ctx context.Context, fieldType string, useColumnKeys, useRowKeys bool, path string) error {
 	// If fieldType is `int`, treat the import data as values to be range-encoded.
-	if fieldType == pilosa.FieldTypeInt {
-		return cmd.bufferValues(ctx, useColumnKeys, path)
+	if fieldType == pilosa.FieldTypeInt || fieldType == pilosa.FieldTypeDecimal {
+		return cmd.bufferValues(ctx, useColumnKeys, fieldType == pilosa.FieldTypeDecimal, path)
 	}
 	return cmd.bufferBits(ctx, useColumnKeys, useRowKeys, path)
 }
@@ -285,9 +288,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowK
 	return nil
 }
 
-// bufferValues buffers slices of FieldValues to be imported as a batch.
-func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool, path string) error {
-	a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
+// bufferValues buffers slices of record identifiers and values to be imported as a batch.
+func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys, parseAsFloat bool, path string) error {
+	req := &pilosa.ImportValueRequest{
+		Index: cmd.Index,
+		Field: cmd.Field,
+		Shard: math.MaxUint64,
+	}
 
 	var r *csv.Reader
 
@@ -307,6 +314,7 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 
 	r.FieldsPerRecord = -1
 	rnum := 0
+
 	for {
 		rnum++
 
@@ -325,69 +333,44 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, useColumnKeys bool,
 			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
 		}
 
-		var val pilosa.FieldValue
-
 		// Parse column id.
 		if useColumnKeys {
-			val.ColumnKey = record[0]
+			req.ColumnKeys = append(req.ColumnKeys, record[0])
+		} else if columnID, err := strconv.ParseUint(record[0], 10, 64); err == nil {
+			req.ColumnIDs = append(req.ColumnIDs, columnID)
 		} else {
-			if val.ColumnID, err = strconv.ParseUint(record[0], 10, 64); err != nil {
-				return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
-			}
+			return fmt.Errorf("invalid column id on row %d: %q", rnum, record[0])
 		}
 
-		// Parse FieldValue.
-		value, err := strconv.ParseInt(record[1], 10, 64)
-		if err != nil {
-			return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
-		}
-		val.Value = value
-
-		a = append(a, val)
-
-		// If we've reached the buffer size then import FieldValues.
-		if len(a) == cmd.BufferSize {
-			if err := cmd.importValues(ctx, useColumnKeys, a); err != nil {
-				return err
+		// Parse value.
+		if parseAsFloat {
+			value, err := strconv.ParseFloat(record[1], 64)
+			if err != nil {
+				return errors.Wrapf(err, "parseing value '%s' as float", record[1])
 			}
-			a = a[:0]
+			req.FloatValues = append(req.FloatValues, value)
+		} else {
+			value, err := strconv.ParseInt(record[1], 10, 64)
+			if err != nil {
+				return errors.Wrapf(err, "invalid value on row %d: %q", rnum, record[1])
+			}
+			req.Values = append(req.Values, value)
+		}
+
+		// If we've reached the buffer size then import the batch.
+		if len(req.ColumnKeys) == cmd.BufferSize || len(req.ColumnIDs) == cmd.BufferSize {
+			if err := cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}); err != nil {
+				return errors.Wrap(err, "importing values")
+			}
+			req.ColumnIDs = req.ColumnIDs[:0]
+			req.ColumnKeys = req.ColumnKeys[:0]
+			req.Values = req.Values[:0]
+			req.FloatValues = req.FloatValues[:0]
 		}
 	}
 
 	// If there are still values in the buffer then flush them.
-	return cmd.importValues(ctx, useColumnKeys, a)
-}
-
-// importValues sends batches of FieldValues to the server.
-func (cmd *ImportCommand) importValues(ctx context.Context, useColumnKeys bool, vals []pilosa.FieldValue) error {
-	logger := log.New(cmd.Stderr, "", log.LstdFlags)
-
-	// If keys are used, all values are sent to the primary translate store (i.e. coordinator).
-	if useColumnKeys {
-		logger.Printf("importing keyed values: n=%d", len(vals))
-		if err := cmd.client.ImportValueK(ctx, cmd.Index, cmd.Field, vals); err != nil {
-			return errors.Wrap(err, "importing keys")
-		}
-		return nil
-	}
-
-	// Group vals by shard.
-	logger.Printf("grouping %d vals", len(vals))
-	valsByShard := http.FieldValues(vals).GroupByShard()
-
-	// Parse path into FieldValues.
-	for shard, vals := range valsByShard {
-		if cmd.Sort {
-			sort.Sort(http.FieldValues(vals))
-		}
-
-		logger.Printf("importing shard: %d, n=%d", shard, len(vals))
-		if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {
-			return errors.Wrap(err, "importing values")
-		}
-	}
-
-	return nil
+	return errors.Wrap(cmd.client.ImportValue2(ctx, req, &pilosa.ImportOptions{}), "importing values")
 }
 
 func (cmd *ImportCommand) TLSHost() string {

ctl/import_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
@@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	// Verify that the data is available on both nodes.
 	for _, host := range []string{host0, host1} {
-		qry := "Count(Row(f=foo0))"
-		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
-		if err != nil {
-			t.Fatalf("Querying data for validation: %s", err)
-		}
+		if err := test.RetryUntil(2*time.Second, func() error {
+			qry := "Count(Row(f=foo0))"
+			resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
+			if err != nil {
+				return fmt.Errorf("Querying data for validation: %s", err)
+			}
 
-		// Read body and unmarshal response.
-		exp := `{"results":[100]}` + "\n"
-		if body, err := ioutil.ReadAll(resp.Body); err != nil {
-			t.Fatalf("reading: %s", err)
-		} else if !reflect.DeepEqual(body, []byte(exp)) {
-			t.Fatalf("expected: %s, but got: %s", exp, body)
+			// Read body and unmarshal response.
+			exp := `{"results":[100]}` + "\n"
+			if body, err := ioutil.ReadAll(resp.Body); err != nil {
+				return fmt.Errorf("reading: %s", err)
+			} else if !reflect.DeepEqual(body, []byte(exp)) {
+				return fmt.Errorf("expected: %s, but got: %s", exp, body)
+			}
+			return nil
+		}); err != nil {
+			t.Fatal(err)
 		}
 	}
+
 }
 
 // Ensure that integer import with keys runs.

ctl/server.go

@@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
 	flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")

diagnostics.go

@@ -233,7 +233,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
 		numIndexes++
 		for _, field := range index.Fields() {
 			numFields++
-			if field.Type() == FieldTypeInt {
+			if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal {
 				bsiFieldCount++
 			}
 			if field.TimeQuantum() != "" {

docs/data-model.md

@@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
 #### Set
 
 Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
+Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
+```go
+[]byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+```
+(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
+In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
 
 ``` request
 curl localhost:10101/index/repository/field/info \

docs/getting-started.md

@@ -100,7 +100,7 @@ curl localhost:10101/index/repository -X POST
 ``` response
 {"success":true}
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ``` request
@@ -325,7 +325,7 @@ Next, let's create the `repository` index:
 	repository := schema.Index("repository")
 ```
 
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -615,7 +615,7 @@ Next, let's create the `repository` index:
 ```
 	Index repository = schema.index("repository");
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```
@@ -818,7 +818,7 @@ Next, let's create the `repository` index:
 ```
 repository = schema.index("repository")
 ```
-The index name must be 64 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
+The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
 
 Let's create the `stargazer` field which has user IDs of stargazers as its rows:
 ```

docs/query-language.md

@@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \
 
 #### Arguments and Types
 
-* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
+* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 230 characters or less in length.
 * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04).
 * `UINT` An unsigned integer (e.g. 42839).
 * `BOOL` A boolean value, `true` or `false`.
@@ -864,7 +864,7 @@ Rows(job)
 **Spec:**
 
 ```
-GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>)
+GroupBy(<ROWS_CALL>, [<ROWS_CALL>...], limit=<UINT>, filter=<ROW_CALL>, aggregate=<CALL>)
 ```
 
 **Description:**
@@ -874,14 +874,18 @@ taking one row each from the specified `Rows` calls. It returns only those
 combinations for which the count is greater than 0.
 
 The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
- Intersect, etc.) which will be intersected with each result prior to returning
- the count. This is analogous to a WHERE clause applied to a relational GROUP BY
- query.
+Intersect, etc.) which will be intersected with each result prior to returning
+the count. This is analagous to a WHERE clause applied to a relational GROUP BY
+query.
 
 The optional `limit` argument limits the number of results returned. The results
 are ordered, so as long as the data isn't changing, the same query will return
 the same result set.
 
+The optional `aggregate` argument takes a `Sum()` query which will be used to
+calculate the sum & count of each group. This is similar to using a `SUM()` in
+the SELECT clause of a relation GROUP BY query.
+
 Paging through results is supported by passing the `previous` argument to each
 of the `Rows` calls in the GroupBy. Take the last result from your previous
 `GroupBy` query, and pass each row ID in that result as the `previous` argument

encoding/proto/proto.go

@@ -225,6 +225,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeImportRoaringRequest(msg, mt)
 		return nil
+	case *pilosa.ImportColumnAttrsRequest:
+		msg := &internal.ImportColumnAttrsRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling ImportColumnAttrsRequest")
+		}
+		decodeImportColumnAttrsRequest(msg, mt)
+		return nil
 	case *pilosa.ImportResponse:
 		msg := &internal.ImportResponse{}
 		err := proto.Unmarshal(buf, msg)
@@ -265,6 +273,22 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
 		}
 		decodeTranslateKeysResponse(msg, mt)
 		return nil
+	case *pilosa.TranslateIDsRequest:
+		msg := &internal.TranslateIDsRequest{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling TranslateIDsRequest")
+		}
+		decodeTranslateIDsRequest(msg, mt)
+		return nil
+	case *pilosa.TranslateIDsResponse:
+		msg := &internal.TranslateIDsResponse{}
+		err := proto.Unmarshal(buf, msg)
+		if err != nil {
+			return errors.Wrap(err, "unmarshaling TranslateIDsResponse")
+		}
+		decodeTranslateIDsResponse(msg, mt)
+		return nil
 	default:
 		panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
 	}
@@ -318,6 +342,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeImportValueRequest(mt)
 	case *pilosa.ImportRoaringRequest:
 		return encodeImportRoaringRequest(mt)
+	case *pilosa.ImportColumnAttrsRequest:
+		return encodeImportColumnAttrsRequest(mt)
 	case *pilosa.ImportResponse:
 		return encodeImportResponse(mt)
 	case *pilosa.BlockDataRequest:
@@ -328,6 +354,10 @@ func encodeToProto(m pilosa.Message) proto.Message {
 		return encodeTranslateKeysRequest(mt)
 	case *pilosa.TranslateKeysResponse:
 		return encodeTranslateKeysResponse(mt)
+	case *pilosa.TranslateIDsRequest:
+		return encodeTranslateIDsRequest(mt)
+	case *pilosa.TranslateIDsResponse:
+		return encodeTranslateIDsResponse(mt)
 	}
 	return nil
 }
@@ -369,12 +399,14 @@ func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
 
 func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
 	return &internal.ImportValueRequest{
-		Index:      m.Index,
-		Field:      m.Field,
-		Shard:      m.Shard,
-		ColumnIDs:  m.ColumnIDs,
-		ColumnKeys: m.ColumnKeys,
-		Values:     m.Values,
+		Index:        m.Index,
+		Field:        m.Field,
+		Shard:        m.Shard,
+		ColumnIDs:    m.ColumnIDs,
+		ColumnKeys:   m.ColumnKeys,
+		Values:       m.Values,
+		FloatValues:  m.FloatValues,
+		StringValues: m.StringValues,
 	}
 }
 
@@ -394,15 +426,30 @@ func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.Import
 	}
 }
 
+func encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest {
+	return &internal.ImportColumnAttrsRequest{
+		Index:     m.Index,
+		Shard:     m.Shard,
+		AttrKey:   m.AttrKey,
+		AttrVals:  m.AttrVals,
+		ColumnIDs: m.ColumnIDs,
+	}
+}
+
 func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
-	return &internal.QueryRequest{
+	r := &internal.QueryRequest{
 		Query:           m.Query,
 		Shards:          m.Shards,
 		ColumnAttrs:     m.ColumnAttrs,
 		Remote:          m.Remote,
 		ExcludeRowAttrs: m.ExcludeRowAttrs,
 		ExcludeColumns:  m.ExcludeColumns,
+		EmbeddedData:    make([]*internal.Row, len(m.EmbeddedData)),
 	}
+	for i := range m.EmbeddedData {
+		r.EmbeddedData[i] = encodeRow(m.EmbeddedData[i])
+	}
+	return r
 }
 
 func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
@@ -415,12 +462,18 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		pb.Results[i] = &internal.QueryResult{}
 
 		switch result := m.Results[i].(type) {
+		case pilosa.SignedRow:
+			pb.Results[i].Type = queryResultTypeSignedRow
+			pb.Results[i].SignedRow = encodeSignedRow(result)
 		case *pilosa.Row:
 			pb.Results[i].Type = queryResultTypeRow
 			pb.Results[i].Row = encodeRow(result)
 		case []pilosa.Pair:
 			pb.Results[i].Type = queryResultTypePairs
 			pb.Results[i].Pairs = encodePairs(result)
+		case *pilosa.PairsField:
+			pb.Results[i].Type = queryResultTypePairsField
+			pb.Results[i].PairsField = encodePairsField(result)
 		case pilosa.ValCount:
 			pb.Results[i].Type = queryResultTypeValCount
 			pb.Results[i].ValCount = encodeValCount(result)
@@ -442,10 +495,13 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse {
 		case pilosa.Pair:
 			pb.Results[i].Type = queryResultTypePair
 			pb.Results[i].Pairs = []*internal.Pair{encodePair(result)}
+		case pilosa.PairField:
+			pb.Results[i].Type = queryResultTypePairField
+			pb.Results[i].Pairs = []*internal.Pair{encodePairField(result)}
 		case nil:
 			pb.Results[i].Type = queryResultTypeNil
 		default:
-			panic(fmt.Errorf("unknown type: %d", pb.Results[i].Type))
+			panic(fmt.Errorf("unknown type: %T", m.Results[i]))
 		}
 	}
 
@@ -532,15 +588,17 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions {
 		return nil
 	}
 	return &internal.FieldOptions{
-		Type:        o.Type,
-		CacheType:   o.CacheType,
-		CacheSize:   o.CacheSize,
-		Min:         o.Min,
-		Max:         o.Max,
-		Base:        o.Base,
-		BitDepth:    uint64(o.BitDepth),
-		TimeQuantum: string(o.TimeQuantum),
-		Keys:        o.Keys,
+		Type:         o.Type,
+		CacheType:    o.CacheType,
+		CacheSize:    o.CacheSize,
+		Min:          o.Min,
+		Max:          o.Max,
+		Base:         o.Base,
+		Scale:        o.Scale,
+		BitDepth:     uint64(o.BitDepth),
+		TimeQuantum:  string(o.TimeQuantum),
+		Keys:         o.Keys,
+		ForeignIndex: o.ForeignIndex,
 	}
 }
 
@@ -722,17 +780,31 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac
 	return &internal.RecalculateCaches{}
 }
 
+func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest {
+	return &internal.TranslateKeysRequest{
+		Index: request.Index,
+		Field: request.Field,
+		Keys:  request.Keys,
+	}
+}
+
 func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse {
 	return &internal.TranslateKeysResponse{
 		IDs: response.IDs,
 	}
 }
 
-func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest {
-	return &internal.TranslateKeysRequest{
+func encodeTranslateIDsRequest(request *pilosa.TranslateIDsRequest) *internal.TranslateIDsRequest {
+	return &internal.TranslateIDsRequest{
 		Index: request.Index,
 		Field: request.Field,
-		Keys:  request.Keys,
+		IDs:   request.IDs,
+	}
+}
+
+func encodeTranslateIDsResponse(response *pilosa.TranslateIDsResponse) *internal.TranslateIDsResponse {
+	return &internal.TranslateIDsResponse{
+		Keys: response.Keys,
 	}
 }
 
@@ -808,9 +880,11 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions)
 	m.Min = options.Min
 	m.Max = options.Max
 	m.Base = options.Base
+	m.Scale = options.Scale
 	m.BitDepth = uint(options.BitDepth)
 	m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
 	m.Keys = options.Keys
+	m.ForeignIndex = options.ForeignIndex
 }
 
 func decodeNodes(a []*internal.Node, m []*pilosa.Node) {
@@ -963,6 +1037,10 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
 	m.Remote = pb.Remote
 	m.ExcludeRowAttrs = pb.ExcludeRowAttrs
 	m.ExcludeColumns = pb.ExcludeColumns
+	m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
+	for i := range pb.EmbeddedData {
+		m.EmbeddedData[i] = decodeRow(pb.EmbeddedData[i])
+	}
 }
 
 func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) {
@@ -983,6 +1061,8 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV
 	m.ColumnIDs = pb.ColumnIDs
 	m.ColumnKeys = pb.ColumnKeys
 	m.Values = pb.Values
+	m.FloatValues = pb.FloatValues
+	m.StringValues = pb.StringValues
 }
 
 func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {
@@ -994,6 +1074,14 @@ func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.Imp
 	m.Views = views
 }
 
+func decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) {
+	m.Index = pb.Index
+	m.Shard = pb.Shard
+	m.AttrKey = pb.AttrKey
+	m.AttrVals = pb.AttrVals
+	m.ColumnIDs = pb.ColumnIDs
+}
+
 func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) {
 	m.Err = pb.Err
 }
@@ -1052,11 +1140,22 @@ func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.T
 	m.IDs = pb.IDs
 }
 
+func decodeTranslateIDsRequest(pb *internal.TranslateIDsRequest, m *pilosa.TranslateIDsRequest) {
+	m.Index = pb.Index
+	m.Field = pb.Field
+	m.IDs = pb.IDs
+}
+
+func decodeTranslateIDsResponse(pb *internal.TranslateIDsResponse, m *pilosa.TranslateIDsResponse) {
+	m.Keys = pb.Keys
+}
+
 // QueryResult types.
 const (
 	queryResultTypeNil uint32 = iota
 	queryResultTypeRow
 	queryResultTypePairs
+	queryResultTypePairsField
 	queryResultTypeValCount
 	queryResultTypeUint64
 	queryResultTypeBool
@@ -1064,14 +1163,20 @@ const (
 	queryResultTypeGroupCounts
 	queryResultTypeRowIdentifiers
 	queryResultTypePair
+	queryResultTypePairField
+	queryResultTypeSignedRow
 )
 
 func decodeQueryResult(pb *internal.QueryResult) interface{} {
 	switch pb.Type {
+	case queryResultTypeSignedRow:
+		return decodeSignedRow(pb.SignedRow)
 	case queryResultTypeRow:
 		return decodeRow(pb.Row)
 	case queryResultTypePairs:
 		return decodePairs(pb.Pairs)
+	case queryResultTypePairsField:
+		return decodePairsField(pb.PairsField)
 	case queryResultTypeValCount:
 		return decodeValCount(pb.ValCount)
 	case queryResultTypeUint64:
@@ -1088,6 +1193,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
 		return decodeGroupCounts(pb.GroupCounts)
 	case queryResultTypePair:
 		return decodePair(pb.Pairs[0])
+	case queryResultTypePairField:
+		return decodePairField(pb.Pairs[0])
 	}
 	panic(fmt.Sprintf("unknown type: %d", pb.Type))
 }
@@ -1095,14 +1202,31 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} {
 // DecodeRow converts r from its internal representation.
 func decodeRow(pr *internal.Row) *pilosa.Row {
 	if pr == nil {
-		return nil
+		return pilosa.NewRow()
 	}
 
-	r := pilosa.NewRow()
+	var r *pilosa.Row
+	if len(pr.Roaring) > 0 {
+		r = pilosa.NewRowFromRoaring(pr.Roaring)
+	} else {
+		r = pilosa.NewRow()
+		for _, v := range pr.Columns {
+			r.SetBit(v)
+		}
+	}
 	r.Attrs = decodeAttrs(pr.Attrs)
 	r.Keys = pr.Keys
-	for _, v := range pr.Columns {
-		r.SetBit(v)
+
+	return r
+}
+
+func decodeSignedRow(pr *internal.SignedRow) pilosa.SignedRow {
+	if pr == nil {
+		return pilosa.SignedRow{}
+	}
+	r := pilosa.SignedRow{
+		Pos: decodeRow(pr.Pos),
+		Neg: decodeRow(pr.Neg),
 	}
 	return r
 }
@@ -1151,6 +1275,7 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount {
 		other[i] = pilosa.GroupCount{
 			Group: decodeFieldRows(a[i].Group),
 			Count: a[i].Count,
+			Sum:   a[i].Sum,
 		}
 	}
 	return other
@@ -1178,6 +1303,17 @@ func decodePairs(a []*internal.Pair) []pilosa.Pair {
 	return other
 }
 
+func decodePairsField(a *internal.PairsField) *pilosa.PairsField {
+	other := &pilosa.PairsField{
+		Pairs: make([]pilosa.Pair, len(a.Pairs)),
+	}
+	for i := range a.Pairs {
+		other.Pairs[i] = decodePair(a.Pairs[i])
+	}
+	other.Field = a.Field
+	return other
+}
+
 func decodePair(pb *internal.Pair) pilosa.Pair {
 	return pilosa.Pair{
 		ID:    pb.ID,
@@ -1186,10 +1322,22 @@ func decodePair(pb *internal.Pair) pilosa.Pair {
 	}
 }
 
+func decodePairField(pb *internal.Pair) pilosa.PairField {
+	return pilosa.PairField{
+		Pair: pilosa.Pair{
+			ID:    pb.ID,
+			Key:   pb.Key,
+			Count: pb.Count,
+		},
+		//Field: pb.Field, // TODO: in order to have this, we need PairField in QueryResponse.
+	}
+}
+
 func decodeValCount(pb *internal.ValCount) pilosa.ValCount {
 	return pilosa.ValCount{
-		Val:   pb.Val,
-		Count: pb.Count,
+		Val:      pb.Val,
+		FloatVal: pb.FloatVal,
+		Count:    pb.Count,
 	}
 }
 
@@ -1209,16 +1357,29 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
 	}
 }
 
+func encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow {
+	ir := &internal.SignedRow{
+		Pos: encodeRow(r.Pos),
+		Neg: encodeRow(r.Neg),
+	}
+	return ir
+}
+
 func encodeRow(r *pilosa.Row) *internal.Row {
 	if r == nil {
 		return nil
 	}
 
-	return &internal.Row{
-		Columns: r.Columns(),
-		Keys:    r.Keys,
-		Attrs:   encodeAttrs(r.Attrs),
+	ir := &internal.Row{
+		Keys:  r.Keys,
+		Attrs: encodeAttrs(r.Attrs),
 	}
+	if true {
+		ir.Columns = r.Columns()
+	} else {
+		ir.Roaring = r.Roaring()
+	}
+	return ir
 }
 
 func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers {
@@ -1235,6 +1396,7 @@ func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount {
 		result[i] = &internal.GroupCount{
 			Group: encodeFieldRows(counts[i].Group),
 			Count: counts[i].Count,
+			Sum:   counts[i].Sum,
 		}
 	}
 	return result
@@ -1267,6 +1429,17 @@ func encodePairs(a pilosa.Pairs) []*internal.Pair {
 	return other
 }
 
+func encodePairsField(a *pilosa.PairsField) *internal.PairsField {
+	other := &internal.PairsField{
+		Pairs: make([]*internal.Pair, len(a.Pairs)),
+	}
+	for i := range a.Pairs {
+		other.Pairs[i] = encodePair(a.Pairs[i])
+	}
+	other.Field = a.Field
+	return other
+}
+
 func encodePair(p pilosa.Pair) *internal.Pair {
 	return &internal.Pair{
 		ID:    p.ID,
@@ -1275,10 +1448,22 @@ func encodePair(p pilosa.Pair) *internal.Pair {
 	}
 }
 
+func encodePairField(p pilosa.PairField) *internal.Pair {
+	/*
+		// TODO: in order to have this, we need PairField in QueryResponse.
+		return &internal.Pair{
+			Pair:  encodePair(p.Pair),
+			Field: p.Field,
+		}
+	*/
+	return encodePair(p.Pair)
+}
+
 func encodeValCount(vc pilosa.ValCount) *internal.ValCount {
 	return &internal.ValCount{
-		Val:   vc.Val,
-		Count: vc.Count,
+		Val:      vc.Val,
+		FloatVal: vc.FloatVal,
+		Count:    vc.Count,
 	}
 }
 

executor_internal_test.go

@@ -15,9 +15,11 @@
 package pilosa
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -25,8 +27,13 @@ import (
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
+	holder := NewHolder(DefaultPartitionN)
+
+	cluster := NewTestCluster(1)
+
 	e := &executor{
-		Holder: NewHolder(),
+		Holder:  holder,
+		Cluster: cluster,
 	}
 	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
 	err := e.Holder.Open()
@@ -46,12 +53,12 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		t.Fatalf("creating fields %v, %v, %v", erra, errb, errc)
 	}
 
-	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`)
+	query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`)
 	if err != nil {
 		t.Fatalf("parsing query: %v", err)
 	}
 	c := query.Calls[0]
-	err = e.translateGroupByCall("i", idx, c)
+	err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 	if err != nil {
 		t.Fatalf("translating call: %v", err)
 	}
@@ -64,6 +71,19 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 		}
 	}
 
+	if having, hok := c.Args["having"].(*pql.Call); !hok {
+		t.Fatal("expected having to be a call")
+	} else if cond, cok := having.Args["count"].(*pql.Condition); !cok {
+		t.Fatal("expected condition to be a count")
+	} else if cond.Op != pql.GT {
+		t.Fatal("expected condition op to be >")
+	} else {
+		val, ok := cond.Uint64Value()
+		if !ok || val != uint64(10) {
+			t.Fatal("expected condition val to be uint64(10)")
+		}
+	}
+
 	errTests := []struct {
 		pql string
 		err string
@@ -102,7 +122,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) {
 				t.Fatalf("parsing query: %v", err)
 			}
 			c := query.Calls[0]
-			err = e.translateGroupByCall("i", idx, c)
+			err = e.translateCall(context.Background(), "i", c, make(map[string]map[string]uint64))
 			if err == nil {
 				t.Fatalf("expected error, but translated call is '%s", c)
 			}
@@ -222,3 +242,200 @@ func TestFieldRowMarshalJSON(t *testing.T) {
 		t.Fatalf("unexpected json: %s", b)
 	}
 }
+
+func TestExecutor_GroupCountCondition(t *testing.T) {
+	t.Run("satisfiesCondition", func(t *testing.T) {
+		type condCheck struct {
+			cond string
+			exp  bool
+		}
+		tests := []struct {
+			groupCount GroupCount
+			checks     []condCheck
+		}{
+			{
+				groupCount: GroupCount{Count: 100},
+				checks: []condCheck{
+					{cond: "count == 99", exp: false},
+					{cond: "count != 99", exp: true},
+					{cond: "count < 99", exp: false},
+					{cond: "count <= 99", exp: false},
+					{cond: "count > 99", exp: true},
+					{cond: "count >= 99", exp: true},
+
+					{cond: "count == 100", exp: true},
+					{cond: "count != 100", exp: false},
+					{cond: "count < 100", exp: false},
+					{cond: "count <= 100", exp: true},
+					{cond: "count > 100", exp: false},
+					{cond: "count >= 100", exp: true},
+
+					{cond: "count == 101", exp: false},
+					{cond: "count != 101", exp: true},
+					{cond: "count < 101", exp: true},
+					{cond: "count <= 101", exp: true},
+					{cond: "count > 101", exp: false},
+					{cond: "count >= 101", exp: false},
+
+					{cond: "98 < count < 100", exp: false},
+					{cond: "98 < count <= 100", exp: true},
+					{cond: "98 < count < 101", exp: true},
+					{cond: "100 <= count < 102", exp: true},
+					{cond: "100 < count < 102", exp: false},
+					{cond: "98 <= count <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: 100},
+				checks: []condCheck{
+					{cond: "sum == 99", exp: false},
+					{cond: "sum != 99", exp: true},
+					{cond: "sum < 99", exp: false},
+					{cond: "sum <= 99", exp: false},
+					{cond: "sum > 99", exp: true},
+					{cond: "sum >= 99", exp: true},
+
+					{cond: "sum == 100", exp: true},
+					{cond: "sum != 100", exp: false},
+					{cond: "sum < 100", exp: false},
+					{cond: "sum <= 100", exp: true},
+					{cond: "sum > 100", exp: false},
+					{cond: "sum >= 100", exp: true},
+
+					{cond: "sum == 101", exp: false},
+					{cond: "sum != 101", exp: true},
+					{cond: "sum < 101", exp: true},
+					{cond: "sum <= 101", exp: true},
+					{cond: "sum > 101", exp: false},
+					{cond: "sum >= 101", exp: false},
+
+					{cond: "98 < sum < 100", exp: false},
+					{cond: "98 < sum <= 100", exp: true},
+					{cond: "98 < sum < 101", exp: true},
+					{cond: "100 <= sum < 102", exp: true},
+					{cond: "100 < sum < 102", exp: false},
+					{cond: "98 <= sum <= 102", exp: true},
+				},
+			},
+			{
+				groupCount: GroupCount{Sum: -100},
+				checks: []condCheck{
+					{cond: "sum == -99", exp: false},
+					{cond: "sum != -99", exp: true},
+					{cond: "sum < -99", exp: true},
+					{cond: "sum <= -99", exp: true},
+					{cond: "sum > -99", exp: false},
+					{cond: "sum >= -99", exp: false},
+
+					{cond: "sum == -100", exp: true},
+					{cond: "sum != -100", exp: false},
+					{cond: "sum < -100", exp: false},
+					{cond: "sum <= -100", exp: true},
+					{cond: "sum > -100", exp: false},
+					{cond: "sum >= -100", exp: true},
+
+					{cond: "sum == -101", exp: false},
+					{cond: "sum != -101", exp: true},
+					{cond: "sum < -101", exp: false},
+					{cond: "sum <= -101", exp: false},
+					{cond: "sum > -101", exp: true},
+					{cond: "sum >= -101", exp: true},
+
+					{cond: "-100 < sum < -98", exp: false},
+					{cond: "-100 <= sum < -98", exp: true},
+					{cond: "-101 < sum < -98", exp: true},
+					{cond: "-102 < sum <= -100", exp: true},
+					{cond: "-102 < sum < -100", exp: false},
+					{cond: "-102 <= sum <= -98", exp: true},
+				},
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test (#%d):", i), func(t *testing.T) {
+				for j, check := range test.checks {
+					t.Run(fmt.Sprintf("check (#%d):", j), func(t *testing.T) {
+
+						query, err := pql.ParseString(fmt.Sprintf("GroupBy(Rows(a), having=Condition(%s))", check.cond))
+						if err != nil {
+							t.Fatalf("parsing query: %v", err)
+						}
+						c := query.Calls[0]
+						having := c.Args["having"].(*pql.Call)
+
+						var got bool
+						for subj, cond := range having.Args {
+							switch subj {
+							case "count", "sum":
+								condition, ok := cond.(*pql.Condition)
+								if !ok {
+									t.Fatalf("not a valid condition")
+								}
+								got = test.groupCount.satisfiesCondition(subj, condition)
+							}
+						}
+
+						if got != check.exp {
+							t.Fatalf("expected: %v, but got: %v", check.exp, got)
+						}
+					})
+				}
+			})
+		}
+	})
+}
+
+func TestValCountComparisons(t *testing.T) {
+	tests := []struct {
+		name       string
+		vc         ValCount
+		other      ValCount
+		expLarger  ValCount
+		expSmaller ValCount
+	}{
+		{
+			name: "zero",
+		},
+		{
+			name:       "ints",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 3, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 1},
+			expSmaller: ValCount{Val: 3, Count: 2},
+		},
+		{
+			name:       "floats",
+			vc:         ValCount{FloatVal: 10.2, Count: 1},
+			other:      ValCount{FloatVal: 3.4, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.2, Count: 1},
+			expSmaller: ValCount{FloatVal: 3.4, Count: 2},
+		},
+		{
+			name:       "intsEquality",
+			vc:         ValCount{Val: 10, Count: 1},
+			other:      ValCount{Val: 10, Count: 2},
+			expLarger:  ValCount{Val: 10, Count: 3},
+			expSmaller: ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:       "floatsEquality",
+			vc:         ValCount{FloatVal: 10.7, Count: 1},
+			other:      ValCount{FloatVal: 10.7, Count: 2},
+			expLarger:  ValCount{FloatVal: 10.7, Count: 3},
+			expSmaller: ValCount{FloatVal: 10.7, Count: 3},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			gotLarger := test.vc.larger(test.other)
+			if gotLarger != test.expLarger {
+				t.Fatalf("larger failed, expected:\n%+v\ngot:\n%+v", test.expLarger, gotLarger)
+			}
+
+			gotSmaller := test.vc.smaller(test.other)
+			if gotSmaller != test.expSmaller {
+				t.Fatalf("smaller failed, expected:\n%+v\ngot:\n%+v", test.expSmaller, gotSmaller)
+			}
+		})
+	}
+}

extension.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+
+	"github.com/molecula/ext"
+	"github.com/pilosa/pilosa/v2/roaring"
+)
+
+// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
+func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
+	return wrappedBitmap{bm}
+}
+
+// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
+// an extension Bitmap.
+type wrappedBitmap struct{ *roaring.Bitmap }
+
+// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
+func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
+	if inner, ok := bm.(wrappedBitmap); ok {
+		if inner.Bitmap != nil {
+			return inner.Bitmap
+		}
+		return roaring.NewFileBitmap()
+	}
+	return roaring.NewFileBitmap()
+}
+
+func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
+	return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
+}
+
+func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
+}
+
+func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
+	shifted, err := b.Bitmap.Shift(n)
+	return wrappedBitmap{shifted}, err
+}
+
+func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
+	return wrappedBitmap{b.Bitmap.Flip(start, last)}
+}
+
+func (b wrappedBitmap) New() ext.Bitmap {
+	return WrapBitmap(roaring.NewFileBitmap())
+}
+
+// ContainerBits tries to get one container's worth of bits.
+func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
+	// it's an error to call this with a non-container-aligned offset
+	if offset&0xFFFF != 0 {
+		return nil
+	}
+	if b.Bitmap == nil {
+		fmt.Printf("ContainerBits on bitmap with no contents\n")
+		return nil
+	}
+	if b.Bitmap.Containers == nil {
+		fmt.Printf("ContainerBits on bitmap with nil Containers\n")
+		return nil
+	}
+	c := b.Bitmap.Containers.Get(offset >> 16)
+	if c == nil {
+		return nil
+	}
+	return c.AsBitmap(target)
+}

extensions/distinct.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build plugindistinct
+
+package extensions
+
+import (
+	_ "github.com/molecula/extensions/distinct"
+)

extensions/dummy.go

@@ -0,0 +1,18 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package contains only things which are conditional on build
+// tags.
+
+package extensions

field.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
@@ -54,11 +55,12 @@ const (
 
 // Field types.
 const (
-	FieldTypeSet   = "set"
-	FieldTypeInt   = "int"
-	FieldTypeTime  = "time"
-	FieldTypeMutex = "mutex"
-	FieldTypeBool  = "bool"
+	FieldTypeSet     = "set"
+	FieldTypeInt     = "int"
+	FieldTypeTime    = "time"
+	FieldTypeMutex   = "mutex"
+	FieldTypeBool    = "bool"
+	FieldTypeDecimal = "decimal"
 )
 
 // Field represents a container for views.
@@ -73,15 +75,21 @@ type Field struct {
 	// Row attribute storage and cache
 	rowAttrStore AttrStore
 
-	// Key/ID translation store.
-	translateStore TranslateStore
-
 	broadcaster broadcaster
 	Stats       stats.StatsClient
 
 	// Field options.
 	options FieldOptions
 
+	// finalOptions is used with a final call to applyOptions.
+	// The initial call to applyOptions is made with options
+	// loaded from the meta file on disk (in the case when
+	// a field is being re-opened). If the field creator calls
+	// setOptions before calling Open(), then those options
+	// will be held in finalOptions, and applied instead of
+	// those from the meta file.
+	finalOptions *FieldOptions
+
 	bsiGroups []*bsiGroup
 
 	// Shards with data on any node in the cluster, according to this node.
@@ -89,10 +97,21 @@ type Field struct {
 
 	logger logger.Logger
 
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
-	// Instantiates new translation store on open.
+	translateStore TranslateStore
+
+	// Instantiates new translation stores
 	OpenTranslateStore OpenTranslateStoreFunc
+
+	// Used for looking up a foreign index.
+	holder *Holder
+
+	// Stores whether or not the field has keys enabled.
+	// This is most helpful for cases where the keys are
+	// based on a foreign index; this prevents having to
+	// call holder.index.Keys() every time.
+	usesKeys bool
 }
 
 // FieldOption is a functional option type for pilosa.fieldOptions.
@@ -107,6 +126,17 @@ func OptFieldKeys() FieldOption {
 	}
 }
 
+// OptFieldForeignIndex marks this field as a foreign key to another
+// index. That is, the values of this field should be interpreted as
+// referencing records (Pilosa columns) in another index. TODO explain
+// where/how this is used by Pilosa.
+func OptFieldForeignIndex(index string) FieldOption {
+	return func(fo *FieldOptions) error {
+		fo.ForeignIndex = index
+		return nil
+	}
+}
+
 // OptFieldTypeDefault is a functional option on FieldOptions
 // used to set the field type and cache setting to the default values.
 func OptFieldTypeDefault() FieldOption {
@@ -155,6 +185,32 @@ func OptFieldTypeInt(min, max int64) FieldOption {
 	}
 }
 
+func OptFieldTypeDecimal(scale int64, minmax ...int64) FieldOption {
+	return func(fo *FieldOptions) error {
+		if fo.Type != "" {
+			return errors.Errorf("can't set field type to 'decimal', already set to: %s", fo.Type)
+		}
+		fo.Min = math.MinInt64
+		fo.Max = math.MaxInt64
+		if len(minmax) == 2 {
+			min, max := minmax[0], minmax[1]
+			if min > max {
+				return errors.Errorf("decimal field min cannot be greater than max, got %d, %d", min, max)
+			}
+			fo.Min = min
+			fo.Max = max
+		} else if len(minmax) > 2 {
+			return errors.Errorf("unknown extra parameters beyond min and max: %v", minmax)
+		} else if len(minmax) == 1 {
+			fo.Min = minmax[0]
+		}
+		fo.Type = FieldTypeDecimal
+		fo.Base = bsiBase(fo.Min, fo.Max)
+		fo.Scale = scale
+		return nil
+	}
+}
+
 // OptFieldTypeTime is a functional option on FieldOptions
 // used to specify the field as being type `time` and to
 // provide any respective configuration values.
@@ -203,6 +259,11 @@ func OptFieldTypeBool() FieldOption {
 }
 
 // NewField returns a new instance of field.
+// NOTE: This function is only used in tests, which is why
+// it only takes a single `FieldOption` (the assumption being
+// that it's of the type `OptFieldType*`). This means
+// this function couldn't be used to set, for example,
+// `FieldOptions.Keys`.
 func NewField(path, index, name string, opts FieldOption) (*Field, error) {
 	err := validateName(name)
 	if err != nil {
@@ -233,7 +294,7 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) {
 		broadcaster: NopBroadcaster,
 		Stats:       stats.NopStatsClient,
 
-		options: applyDefaultOptions(fo),
+		options: *applyDefaultOptions(&fo),
 
 		remoteAvailableShards: roaring.NewBitmap(),
 
@@ -253,12 +314,19 @@ func (f *Field) Index() string { return f.index }
 // Path returns the path the field was initialized with.
 func (f *Field) Path() string { return f.path }
 
+// TranslateStorePath returns the translation database path for the field.
+func (f *Field) TranslateStorePath() string {
+	return filepath.Join(f.path, "keys")
+}
+
+// TranslateStore returns the field's translation store.
+func (f *Field) TranslateStore() TranslateStore {
+	return f.translateStore
+}
+
 // RowAttrStore returns the attribute storage.
 func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore }
 
-// TranslateStore returns the underlying translation store for the field.
-func (f *Field) TranslateStore() TranslateStore { return f.translateStore }
-
 // AvailableShards returns a bitmap of shards that contain data.
 func (f *Field) AvailableShards() *roaring.Bitmap {
 	f.mu.RLock()
@@ -288,18 +356,30 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
 
 // loadAvailableShards reads remoteAvailableShards data for the field, if any.
 func (f *Field) loadAvailableShards() error {
-	bm := roaring.NewBitmap()
 	// Read data from meta file.
 	path := filepath.Join(f.path, ".available.shards")
 	buf, err := ioutil.ReadFile(path)
+	// doesn't exist: this is fine
 	if os.IsNotExist(err) {
 		return nil
-	} else if err != nil {
-		return errors.Wrap(err, "reading available shards")
-	} else {
-		if err := bm.UnmarshalBinary(buf); err != nil {
-			return errors.Wrap(err, "unmarshaling")
+	}
+	// some other problem:
+	if err != nil {
+		f.logger.Printf("available shards file present but unreadable, discarding: %v", err)
+		err = os.Remove(path)
+		if err != nil {
+			return errors.Wrap(err, "deleting corrupt available shards list")
 		}
+		return nil
+	}
+	bm := roaring.NewBitmap()
+	if err = bm.UnmarshalBinary(buf); err != nil {
+		f.logger.Printf("available shards file corrupt, discarding: %v", err)
+		err = os.Remove(path)
+		if err != nil {
+			return errors.Wrap(err, "deleting corrupt available shards list")
+		}
+		return nil
 	}
 	// Merge bitmap from file into field.
 	f.mergeRemoteAvailableShards(bm)
@@ -418,7 +498,13 @@ func (f *Field) Open() error {
 			return errors.Wrap(err, "loading available shards")
 		}
 
-		// Apply the field options loaded from meta.
+		// If options were provided using setOptions(), then
+		// use those instead of the options from the meta file.
+		if f.finalOptions != nil {
+			f.options = *f.finalOptions
+		}
+
+		// Apply the field options loaded from meta (or set via setOptions()).
 		f.logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
 		if err := f.applyOptions(f.options); err != nil {
 			return errors.Wrap(err, "applying options")
@@ -434,9 +520,17 @@ func (f *Field) Open() error {
 			return errors.Wrap(err, "opening attrstore")
 		}
 
-		// Instantiate & open translation store.
-		if f.translateStore, err = f.OpenTranslateStore(filepath.Join(f.path, "keys"), f.index, f.name); err != nil {
-			return errors.Wrap(err, "opening translate store")
+		// Apply the field-specific translateStore.
+		if err := f.applyTranslateStore(); err != nil {
+			return errors.Wrap(err, "applying translate store")
+		}
+
+		// If the field has a foreign index, make sure the index
+		// exists.
+		if f.options.ForeignIndex != "" {
+			if err := f.holder.checkForeignIndex(f); err != nil {
+				return errors.Wrap(err, "checking foreign index")
+			}
 		}
 
 		return nil
@@ -449,6 +543,49 @@ func (f *Field) Open() error {
 	return nil
 }
 
+// applyTranslateStore opens the configured translate store.
+func (f *Field) applyTranslateStore() error {
+	// Instantiate & open translation store.
+	var err error
+	f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1)
+	if err != nil {
+		return errors.Wrap(err, "opening field translate store")
+	}
+	f.usesKeys = f.options.Keys
+
+	// In the case where the field has a foreign index, set
+	// the usesKeys value accordingly.
+	if foreignIndexName := f.ForeignIndex(); foreignIndexName != "" {
+		if foreignIndex := f.holder.indexes[foreignIndexName]; foreignIndex != nil {
+			f.usesKeys = foreignIndex.Keys()
+		}
+	}
+	return nil
+}
+
+// applyForeignIndex used to set the field's translateStore to
+// that of the foreign index, but since moving to partitioned
+// translate stores on indexes, that doesn't happen anymore.
+// So now all this method does is check that the foreign index
+// actually exists. If we decided this was unnecessary (which
+// it kind of is), we could remove the field.holder and all
+// the logic which does this check on holder open after all
+// indexes have opened.
+func (f *Field) applyForeignIndex() error {
+	foreignIndex := f.holder.Index(f.options.ForeignIndex)
+	if foreignIndex == nil {
+		return errors.Wrapf(ErrForeignIndexNotFound, "%s", f.options.ForeignIndex)
+	}
+	f.usesKeys = foreignIndex.Keys()
+	return nil
+}
+
+// ForeignIndex returns the foreign index name attached to the field.
+// Returns blank string if no foreign index exists.
+func (f *Field) ForeignIndex() string {
+	return f.options.ForeignIndex
+}
+
 var fieldQueue = make(chan struct{}, 16)
 
 // openViews opens and initializes the views inside the field.
@@ -550,10 +687,12 @@ func (f *Field) loadMeta() error {
 	f.options.Min = pb.Min
 	f.options.Max = pb.Max
 	f.options.Base = pb.Base
+	f.options.Scale = pb.Scale
 	f.options.BitDepth = uint(pb.BitDepth)
 	f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
 	f.options.Keys = pb.Keys
 	f.options.NoStandardView = pb.NoStandardView
+	f.options.ForeignIndex = pb.ForeignIndex
 
 	return nil
 }
@@ -584,6 +723,11 @@ func (f *Field) saveMeta() error {
 	return nil
 }
 
+// setOptions saves options for final application during Open().
+func (f *Field) setOptions(opts *FieldOptions) {
+	f.finalOptions = applyDefaultOptions(opts)
+}
+
 // applyOptions configures the field based on opt.
 func (f *Field) applyOptions(opt FieldOptions) error {
 	switch opt.Type {
@@ -596,12 +740,10 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 		if opt.CacheType != "" {
 			f.options.CacheType = opt.CacheType
 		}
-		if opt.CacheSize != 0 {
-			if opt.CacheType == CacheTypeNone {
-				f.options.CacheSize = 0
-			} else {
-				f.options.CacheSize = opt.CacheSize
-			}
+		if opt.CacheType == CacheTypeNone {
+			f.options.CacheSize = 0
+		} else if opt.CacheSize != 0 {
+			f.options.CacheSize = opt.CacheSize
 		}
 		f.options.Min = 0
 		f.options.Max = 0
@@ -609,16 +751,19 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 		f.options.BitDepth = 0
 		f.options.TimeQuantum = ""
 		f.options.Keys = opt.Keys
-	case FieldTypeInt:
+		f.options.ForeignIndex = ""
+	case FieldTypeInt, FieldTypeDecimal:
 		f.options.Type = opt.Type
 		f.options.CacheType = CacheTypeNone
 		f.options.CacheSize = 0
 		f.options.Min = opt.Min
 		f.options.Max = opt.Max
 		f.options.Base = opt.Base
+		f.options.Scale = opt.Scale
 		f.options.BitDepth = opt.BitDepth
 		f.options.TimeQuantum = ""
 		f.options.Keys = opt.Keys
+		f.options.ForeignIndex = opt.ForeignIndex
 
 		// Create new bsiGroup.
 		bsig := &bsiGroup{
@@ -627,6 +772,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 			Min:      opt.Min,
 			Max:      opt.Max,
 			Base:     opt.Base,
+			Scale:    opt.Scale,
 			BitDepth: opt.BitDepth,
 		}
 		// Validate bsiGroup.
@@ -651,6 +797,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 			f.Close()
 			return errors.Wrap(err, "setting time quantum")
 		}
+		f.options.ForeignIndex = ""
 	case FieldTypeBool:
 		f.options.Type = FieldTypeBool
 		f.options.CacheType = CacheTypeNone
@@ -661,6 +808,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 		f.options.BitDepth = 0
 		f.options.TimeQuantum = ""
 		f.options.Keys = false
+		f.options.ForeignIndex = ""
 	default:
 		return errors.New("invalid field type")
 	}
@@ -678,6 +826,13 @@ func (f *Field) Close() error {
 		_ = f.rowAttrStore.Close()
 	}
 
+	// Close field translation store.
+	if f.translateStore != nil {
+		if err := f.translateStore.Close(); err != nil {
+			return err
+		}
+	}
+
 	// Close all views.
 	for _, view := range f.viewMap {
 		if err := view.close(); err != nil {
@@ -686,20 +841,14 @@ func (f *Field) Close() error {
 	}
 	f.viewMap = make(map[string]*view)
 
-	if f.translateStore != nil {
-		if err := f.translateStore.Close(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
-// keys returns true if the field uses string keys.
-func (f *Field) keys() bool {
+// Keys returns true if the field uses string keys.
+func (f *Field) Keys() bool {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
-	return f.options.Keys
+	return f.usesKeys
 }
 
 // bsiGroup returns a bsiGroup by name.
@@ -883,7 +1032,9 @@ func (f *Field) newView(path, name string) *view {
 	view.rowAttrStore = f.rowAttrStore
 	view.stats = f.Stats
 	view.broadcaster = f.broadcaster
-	view.snapshotQueue = f.snapshotQueue
+	if f.snapshotQueue != nil {
+		view.snapshotQueue = f.snapshotQueue
+	}
 	return view
 }
 
@@ -1051,6 +1202,36 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*view) {
 	return me
 }
 
+// StringValue reads an integer field value for a column, and converts
+// it to a string based on a foreign index string key.
+func (f *Field) StringValue(columnID uint64) (value string, exists bool, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return value, false, ErrBSIGroupNotFound
+	}
+
+	val, exists, err := f.Value(columnID)
+	if exists {
+		value, err = f.translateStore.TranslateID(uint64(val))
+	}
+	return value, exists, err
+}
+
+// FloatValue reads an integer field value for a column, and converts
+// it to a float based on the configured scale.
+func (f *Field) FloatValue(columnID uint64) (value float64, exists bool, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return 0, false, ErrBSIGroupNotFound
+	}
+
+	val, exists, err := f.Value(columnID)
+	if exists {
+		value = float64(val) / math.Pow10(int(bsig.Scale))
+	}
+	return value, exists, err
+}
+
 // Value reads a field value for a column.
 func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
 	bsig := f.bsiGroup(f.name)
@@ -1073,6 +1254,18 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
 	return int64(v) + bsig.Base, true, nil
 }
 
+// SetFloatValue takes a floating point value, and converts it to an
+// integer based on the field's configured scale, before setting that
+// integer via SetValue.
+func (f *Field) SetFloatValue(columnID uint64, value float64) (changed bool, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return false, ErrBSIGroupNotFound
+	}
+	val := int64(float64(value) * math.Pow10(int(bsig.Scale)))
+	return f.SetValue(columnID, val)
+}
+
 // SetValue sets a field value for a column.
 func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) {
 	// Fetch bsiGroup & validate min/max.
@@ -1114,10 +1307,46 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error)
 	if err != nil {
 		return false, errors.Wrap(err, "creating view")
 	}
-
 	return view.setValue(columnID, bsig.BitDepth, baseValue)
 }
 
+// ClearValue removes a field value for a column.
+func (f *Field) ClearValue(columnID uint64) (changed bool, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return false, ErrBSIGroupNotFound
+	}
+	// Fetch target view.
+	view := f.view(viewBSIGroupPrefix + f.name)
+	if view == nil {
+		return false, nil
+	}
+	value, exists, err := view.value(columnID, bsig.BitDepth)
+	if err != nil {
+		return false, err
+	}
+	if exists {
+		return view.clearValue(columnID, bsig.BitDepth, value)
+	}
+	return false, nil
+}
+
+// FloatSum performs a Sum query and converts the result to a float
+// based on the field's configured scale.
+func (f *Field) FloatSum(filter *Row, name string) (sum float64, count int64, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return 0, 0, ErrBSIGroupNotFound
+	}
+
+	sumI, count, err := f.Sum(filter, name)
+	if err == nil {
+		sum = float64(sumI) / math.Pow10(int(bsig.Scale))
+	}
+	return sum, count, err
+
+}
+
 // Sum returns the sum and count for a field.
 // An optional filtering row can be provided.
 func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) {
@@ -1138,6 +1367,22 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) {
 	return int64(vsum) + (int64(vcount) * bsig.Base), int64(vcount), nil
 }
 
+// FloatMin performs a Min query and converts the result to a float
+// based on the field's configured scale.
+// TODO: this and Min are probably worthless
+func (f *Field) FloatMin(filter *Row, name string) (min float64, count int64, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return 0, 0, ErrBSIGroupNotFound
+	}
+
+	minI, count, err := f.Min(filter, name)
+	if err == nil {
+		min = float64(minI) / math.Pow10(int(bsig.Scale))
+	}
+	return min, count, err
+}
+
 // Min returns the min for a field.
 // An optional filtering row can be provided.
 func (f *Field) Min(filter *Row, name string) (min, count int64, err error) {
@@ -1158,6 +1403,93 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) {
 	return int64(vmin) + bsig.Base, int64(vcount), nil
 }
 
+// FloatMax performs a max query and converts the result to a float
+// based on the field's configured scale.
+//
+// TODO, this isn't really used, because it's kind of useless. It will
+// only get the max among shards on this node, but all query execution
+// already happens at the shard level and bypasses this entirely
+// calling fragment.max instead.
+func (f *Field) FloatMax(filter *Row, name string) (max float64, count int64, err error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return 0, 0, ErrBSIGroupNotFound
+	}
+
+	maxI, count, err := f.Max(filter, name)
+	if err == nil {
+		max = float64(maxI) / math.Pow10(int(bsig.Scale))
+	}
+	return max, count, err
+}
+
+func (f *Field) MaxForShard(shard uint64, filter *Row) (ValCount, error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return ValCount{}, ErrBSIGroupNotFound
+	}
+
+	view := f.view(viewBSIGroupPrefix + f.name)
+	if view == nil {
+		return ValCount{}, nil
+	}
+
+	fragment := view.Fragment(shard)
+	if fragment == nil {
+		return ValCount{}, nil
+	}
+
+	max, cnt, err := fragment.max(filter, bsig.BitDepth)
+	if err != nil {
+		return ValCount{}, errors.Wrap(err, "calling fragment.max")
+	}
+
+	valCount := ValCount{Count: int64(cnt)}
+
+	if f.Options().Type == FieldTypeDecimal {
+		valCount.FloatVal = float64(max) / math.Pow10(int(bsig.Scale))
+	} else {
+		valCount.Val = max
+	}
+
+	return valCount, nil
+}
+
+// MinForShard returns the minimum value which appears in this shard
+// (this field must be an Int or Decimal field). It also returns the
+// number of times the minimum value appears.
+func (f *Field) MinForShard(shard uint64, filter *Row) (ValCount, error) {
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return ValCount{}, ErrBSIGroupNotFound
+	}
+
+	view := f.view(viewBSIGroupPrefix + f.name)
+	if view == nil {
+		return ValCount{}, nil
+	}
+
+	fragment := view.Fragment(shard)
+	if fragment == nil {
+		return ValCount{}, nil
+	}
+
+	min, cnt, err := fragment.min(filter, bsig.BitDepth)
+	if err != nil {
+		return ValCount{}, errors.Wrap(err, "calling fragment.min")
+	}
+
+	valCount := ValCount{Count: int64(cnt)}
+
+	if f.Options().Type == FieldTypeDecimal {
+		valCount.FloatVal = float64(min) / math.Pow10(int(bsig.Scale))
+	} else {
+		valCount.Val = min
+	}
+
+	return valCount, nil
+}
+
 // Max returns the max for a field.
 // An optional filtering row can be provided.
 func (f *Field) Max(filter *Row, name string) (max, count int64, err error) {
@@ -1283,6 +1615,21 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts
 	return nil
 }
 
+func (f *Field) importFloatValue(columnIDs []uint64, values []float64, options *ImportOptions) error {
+	// convert values to int64 values based on scale
+	ivalues := make([]int64, len(values))
+	bsig := f.bsiGroup(f.name)
+	if bsig == nil {
+		return errors.Wrap(ErrBSIGroupNotFound, f.name)
+	}
+	mult := math.Pow10(int(bsig.Scale))
+	for i, fval := range values {
+		ivalues[i] = int64(fval * mult)
+	}
+	// then call importValue
+	return f.importValue(columnIDs, ivalues, options)
+}
+
 // importValue bulk imports range-encoded value data.
 func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportOptions) error {
 	viewName := viewBSIGroupPrefix + f.name
@@ -1292,34 +1639,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
 		return errors.Wrap(ErrBSIGroupNotFound, f.name)
 	}
 
-	// Find the lowest/highest values.
+	// We want to determine the required bit depth, in case the field doesn't
+	// have as many bits currently as would be needed to represent these values,
+	// but only if the values are in-range for the field.
 	var min, max int64
-	for i, value := range values {
-		if i == 0 || value < min {
-			min = value
-		}
-		if i == 0 || value > max {
-			max = value
-		}
-	}
-
-	// Determine the highest bit depth required by the min & max.
-	requiredDepth := bitDepthInt64(min - bsig.Base)
-	if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
-		requiredDepth = v
-	}
-
-	// Increase bit depth if required.
-	if requiredDepth > bsig.BitDepth {
-		if err := func() error {
-			f.mu.Lock()
-			defer f.mu.Unlock()
-			bsig.BitDepth = requiredDepth
-			f.options.BitDepth = requiredDepth
-			return f.saveMeta()
-		}(); err != nil {
-			return errors.Wrap(err, "increasing bsi bit depth")
-		}
+	if len(values) > 0 {
+		min, max = values[0], values[0]
 	}
 
 	// Split import data by fragment.
@@ -1331,6 +1656,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
 		} else if value < bsig.Min {
 			return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value)
 		}
+		if value > max {
+			max = value
+		}
+		if value < min {
+			min = value
+		}
 
 		// Attach value to each bsiGroup view.
 		for _, name := range []string{viewName} {
@@ -1342,6 +1673,28 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
 		}
 	}
 
+	// Determine the highest bit depth required by the min & max.
+	requiredDepth := bitDepthInt64(min - bsig.Base)
+	if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
+		requiredDepth = v
+	}
+	// Increase bit depth if required.
+	if err := func() error {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		bitDepth := bsig.BitDepth
+		if requiredDepth > bitDepth {
+			bsig.BitDepth = requiredDepth
+			f.options.BitDepth = requiredDepth
+			return f.saveMeta()
+		} else {
+			requiredDepth = bitDepth
+		}
+		return nil
+	}(); err != nil {
+		return errors.Wrap(err, "increasing bsi bit depth")
+	}
+
 	// Import into each fragment.
 	for key, data := range dataByFragment {
 		// The view must already exist (i.e. we can't create it)
@@ -1419,24 +1772,48 @@ type FieldOptions struct {
 	BitDepth       uint        `json:"bitDepth,omitempty"`
 	Min            int64       `json:"min,omitempty"`
 	Max            int64       `json:"max,omitempty"`
+	Scale          int64       `json:"scale,omitempty"`
 	Keys           bool        `json:"keys"`
 	NoStandardView bool        `json:"noStandardView,omitempty"`
 	CacheSize      uint32      `json:"cacheSize,omitempty"`
 	CacheType      string      `json:"cacheType,omitempty"`
 	Type           string      `json:"type,omitempty"`
 	TimeQuantum    TimeQuantum `json:"timeQuantum,omitempty"`
+	ForeignIndex   string      `json:"foreignIndex"`
 }
 
-// applyDefaultOptions returns a new FieldOptions object
-// with default values if o does not contain a valid type.
-func applyDefaultOptions(o FieldOptions) FieldOptions {
-	if o.Type == "" {
-		return FieldOptions{
-			Type:      DefaultFieldType,
-			CacheType: DefaultCacheType,
-			CacheSize: DefaultCacheSize,
+// newFieldOptions returns a new instance of FieldOptions
+// with applied and validated functional options.
+func newFieldOptions(opts ...FieldOption) (*FieldOptions, error) {
+	fo := FieldOptions{}
+	for _, opt := range opts {
+		err := opt(&fo)
+		if err != nil {
+			return nil, err
 		}
 	}
+
+	if fo.Keys {
+		switch fo.Type {
+		case FieldTypeInt:
+			return nil, ErrIntFieldWithKeys
+
+		case FieldTypeDecimal:
+			return nil, ErrDecimalFieldWithKeys
+		}
+	}
+
+	return &fo, nil
+}
+
+// applyDefaultOptions updates FieldOptions with the default
+// values if o does not contain a valid type.
+func applyDefaultOptions(o *FieldOptions) *FieldOptions {
+	if o.Type == "" {
+		o.Type = DefaultFieldType
+		o.CacheType = DefaultCacheType
+		o.CacheSize = DefaultCacheSize
+	}
 	return o
 }
 
@@ -1454,12 +1831,14 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
 		CacheType:      o.CacheType,
 		CacheSize:      o.CacheSize,
 		Base:           o.Base,
+		Scale:          o.Scale,
 		BitDepth:       uint64(o.BitDepth),
 		Min:            o.Min,
 		Max:            o.Max,
 		TimeQuantum:    string(o.TimeQuantum),
 		Keys:           o.Keys,
 		NoStandardView: o.NoStandardView,
+		ForeignIndex:   o.ForeignIndex,
 	}
 }
 
@@ -1481,9 +1860,28 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
 			o.Keys,
 		})
 	case FieldTypeInt:
+		return json.Marshal(struct {
+			Type         string `json:"type"`
+			Base         int64  `json:"base"`
+			BitDepth     uint   `json:"bitDepth"`
+			Min          int64  `json:"min"`
+			Max          int64  `json:"max"`
+			Keys         bool   `json:"keys"`
+			ForeignIndex string `json:"foreignIndex"`
+		}{
+			o.Type,
+			o.Base,
+			o.BitDepth,
+			o.Min,
+			o.Max,
+			o.Keys,
+			o.ForeignIndex,
+		})
+	case FieldTypeDecimal:
 		return json.Marshal(struct {
 			Type     string `json:"type"`
 			Base     int64  `json:"base"`
+			Scale    int64  `json:"scale"`
 			BitDepth uint   `json:"bitDepth"`
 			Min      int64  `json:"min"`
 			Max      int64  `json:"max"`
@@ -1491,6 +1889,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
 		}{
 			o.Type,
 			o.Base,
+			o.Scale,
 			o.BitDepth,
 			o.Min,
 			o.Max,
@@ -1563,6 +1962,7 @@ type bsiGroup struct {
 	Min      int64  `json:"min,omitempty"`
 	Max      int64  `json:"max,omitempty"`
 	Base     int64  `json:"base,omitempty"`
+	Scale    int64  `json:"scale,omitempty"`
 	BitDepth uint   `json:"bitDepth,omitempty"`
 }
 

field_internal_test.go

@@ -19,7 +19,9 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
@@ -366,6 +368,62 @@ func TestField_PersistAvailableShards(t *testing.T) {
 
 }
 
+func TestField_CorruptAvailableShards(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := avail.Write([]byte{23})
+	if err != nil || n != 1 {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
+func TestField_TruncatedAvailableShards(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.AddRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+
+	path := filepath.Join(f.path, ".available.shards")
+
+	avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	avail.Close()
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) {
+		t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice())
+	}
+}
+
 // Ensure that persisting available shards having a smaller footprint (for example,
 // when going from a bitmap to a smaller, RLE representation) succeeds.
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
@@ -438,3 +496,236 @@ func TestBSIGroup_BaseDefaultValue(t *testing.T) {
 		}
 	}
 }
+
+func TestField_ApplyOptions(t *testing.T) {
+	for i, tt := range []struct {
+		opts    FieldOptions
+		expOpts FieldOptions
+	}{
+		{
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+			FieldOptions{
+				Type:      FieldTypeSet,
+				CacheType: CacheTypeNone,
+				CacheSize: 0,
+			},
+		},
+	} {
+
+		fld := &Field{}
+		fld.options = *applyDefaultOptions(&FieldOptions{})
+
+		if err := fld.applyOptions(tt.opts); err != nil {
+			t.Fatal(err)
+		}
+
+		if fld.options.CacheType != tt.expOpts.CacheType {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheType value. expected: %s, but got: %s", i, tt.expOpts.CacheType, fld.options.CacheType)
+		} else if fld.options.CacheSize != tt.expOpts.CacheSize {
+			t.Fatalf("test %d, unexpected FieldOptions.CacheSize value. expected: %d, but got: %d", i, tt.expOpts.CacheSize, fld.options.CacheSize)
+		}
+	}
+}
+
+// Ensure that importValue handles requiredDepth correctly.
+// This test sets the same column value to 1, then 8, then 1.
+// A previous bug was incorrectly determining bitDepth based
+// on the values in the import, and not taking existing values
+// into consideration. This would cause an import of 1/8/1
+// to result in a value of 9 instead of 1.
+func TestBSIGroup_importValue(t *testing.T) {
+	f := MustOpenField(OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, tt := range []struct {
+		columnIDs []uint64
+		values    []int64
+		checkVal  int64
+		expCols   []uint64
+	}{
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{8},
+			8,
+			[]uint64{100},
+		},
+		{
+			[]uint64{100},
+			[]int64{1},
+			1,
+			[]uint64{100},
+		},
+	} {
+		if err := f.importValue(tt.columnIDs, tt.values, options); err != nil {
+			t.Fatalf("test %d, importing values: %s", i, err.Error())
+		}
+
+		if row, err := f.Range(f.name, pql.EQ, tt.checkVal); err != nil {
+			t.Fatalf("test %d, getting range: %s", i, err.Error())
+		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
+			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
+		}
+	}
+}
+
+func TestIntField_MinMaxForShard(t *testing.T) {
+	f := MustOpenField(OptFieldTypeInt(-100, 200))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []int64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []int64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []int64{10},
+			expMax:    ValCount{Val: 10, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []int64{10, 20},
+			expMax:    ValCount{Val: 20, Count: 1},
+			expMin:    ValCount{Val: 10, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []int64{10, 20, 10, 10, 20},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []int64{10, 20, 10, 10, 20, 11, 12, 11, 13, 11, 44, 1},
+			expMax:    ValCount{Val: 20, Count: 2},
+			expMin:    ValCount{Val: 10, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}
+
+func TestDecimalField_MinMaxForShard(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDecimal(3))
+
+	options := &ImportOptions{}
+	for i, test := range []struct {
+		name      string
+		columnIDs []uint64
+		values    []float64
+		expMax    ValCount
+		expMin    ValCount
+	}{
+		{
+			name:      "zero",
+			columnIDs: []uint64{},
+			values:    []float64{},
+		},
+		{
+			name:      "single",
+			columnIDs: []uint64{1},
+			values:    []float64{10.1},
+			expMax:    ValCount{FloatVal: 10.1, Count: 1},
+			expMin:    ValCount{FloatVal: 10.1, Count: 1},
+		},
+		{
+			name:      "twovals",
+			columnIDs: []uint64{1, 2},
+			values:    []float64{10.1, 20.2},
+			expMax:    ValCount{FloatVal: 20.2, Count: 1},
+			expMin:    ValCount{FloatVal: 10.1, Count: 1},
+		},
+		{
+			name:      "multiplecounts",
+			columnIDs: []uint64{1, 2, 3, 4, 5},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2},
+			expMax:    ValCount{FloatVal: 20.2, Count: 2},
+			expMin:    ValCount{FloatVal: 10.1, Count: 3},
+		},
+		{
+			name:      "middlevals",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
+			expMax:    ValCount{FloatVal: 20.2, Count: 2},
+			expMin:    ValCount{FloatVal: 10.1, Count: 3},
+		},
+		{
+			name:      "another shard",
+			columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100000000, 100000001},
+			values:    []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11, 44.39, 0.23},
+			expMax:    ValCount{FloatVal: 20.2, Count: 2},
+			expMin:    ValCount{FloatVal: 10.1, Count: 3},
+		},
+	} {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			if err := f.importFloatValue(test.columnIDs, test.values, options); err != nil {
+				t.Fatalf("test %d, importing values: %s", i, err.Error())
+			}
+
+			maxvc, err := f.MaxForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting max for shard: %v", err)
+			}
+			if maxvc != test.expMax {
+				t.Fatalf("max expected:\n%+v\ngot:\n%+v", test.expMax, maxvc)
+			}
+
+			minvc, err := f.MinForShard(0, nil)
+			if err != nil {
+				t.Fatalf("getting min for shard: %v", err)
+			}
+			if minvc != test.expMin {
+				t.Fatalf("min expected:\n%+v\ngot:\n%+v", test.expMin, minvc)
+			}
+		})
+	}
+}

field_test.go

@@ -158,6 +158,7 @@ func TestField_NameValidation(t *testing.T) {
 		"under_score",
 		"abc123",
 		"trailing_",
+		"charact2301234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
 	}
 	invalidFieldNames := []string{
 		"",
@@ -168,7 +169,7 @@ func TestField_NameValidation(t *testing.T) {
 		"abc def",
 		"camelCase",
 		"UPPERCASE",
-		"a12345678901234567890123456789012345678901234567890123456789012345",
+		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
 	path, err := ioutil.TempDir("", "pilosa-field-")
@@ -227,3 +228,44 @@ func TestField_AvailableShards(t *testing.T) {
 		t.Fatal(diff)
 	}
 }
+
+func TestField_ClearValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on field.
+		if changed, err := f.SetValue(100, 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		if changed, err := f.ClearValue(100); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal(err)
+		}
+
+		// Read value.
+		if _, exists, err := f.Value(100); err != nil {
+			t.Fatal(err)
+		} else if exists {
+			t.Fatal("expected value to not exist")
+		}
+	})
+}

fragment_internal_test.go

@@ -596,6 +596,21 @@ func TestFragment_Range(t *testing.T) {
 		}
 	})
 
+	t.Run("LTRegression", func(t *testing.T) {
+		f := mustOpenFragment("i", "f", viewStandard, 0, "")
+		defer f.Clean(t)
+
+		if _, err := f.setValue(1, 1, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if b, err := f.rangeOp(pql.LT, 1, 2); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Columns(), []uint64{1}) {
+			t.Fatalf("unepxected coulmns: %+v", b.Columns())
+		}
+	})
+
 	t.Run("GT", func(t *testing.T) {
 		f := mustOpenFragment("i", "f", viewStandard, 0, "")
 		defer f.Clean(t)
@@ -1382,6 +1397,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 
 	// Read into another fragment.
 	f1 := mustOpenFragment("i", "f", viewStandard, 0, "")
+	defer f1.Clean(t)
 	if rn, err := f1.ReadFrom(&buf); err != nil {
 		t.Fatal(err)
 	} else if wn != rn {
@@ -2054,7 +2070,11 @@ func BenchmarkImportRoaring(b *testing.B) {
 					b.StartTimer()
 					err := f.importRoaringT(data, false)
 					if err != nil {
-						f.awaitSnapshot()
+						// we don't actually particularly
+						// care whether this succeeds,
+						// but if it's happening we want
+						// it to be done.
+						_ = f.snapshotQueue.Await(f)
 						f.Clean(b)
 						b.Fatalf("import error: %v", err)
 					}
@@ -2093,7 +2113,9 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 							j := j
 							eg.Go(func() error {
 								err := frags[j].importRoaringT(data[j], false)
-								frags[j].awaitSnapshot()
+								// error unimportant if it happened, but we want
+								// any snapshots to have finished.
+								_ = frags[j].snapshotQueue.Await(frags[j])
 								return err
 							})
 						}
@@ -2131,11 +2153,13 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								// is excessive. force storage into snapshotted state, then use import
 								// to generate an op log and/or snapshot.
 								_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
-								frags[j].enqueueSnapshot()
-								frags[j].awaitSnapshot()
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
 								}
+								err = frags[j].snapshotQueue.Immediate(frags[j])
+								if err != nil {
+									b.Fatalf("snapshot after import: %v", err)
+								}
 							}
 							eg := errgroup.Group{}
 							b.StartTimer()
@@ -2143,7 +2167,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								j := j
 								eg.Go(func() error {
 									err := frags[j].importRoaringT(updata, false)
-									frags[j].awaitSnapshot()
+									err2 := frags[j].snapshotQueue.Await(frags[j])
+									if err == nil {
+										err = err2
+									}
 									return err
 								})
 							}
@@ -2205,20 +2232,38 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						// is excessive. force storage into snapshotted state, then use import
 						// to generate an op log and/or snapshot.
 						_, _, err := f.storage.ImportRoaringBits(data, false, false, 0)
-						f.enqueueSnapshot()
-						f.awaitSnapshot()
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Immediate(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StartTimer()
 						err = f.importRoaringT(updata, false)
-						f.awaitSnapshot()
 						if err != nil {
 							f.Clean(b)
 							b.Errorf("import error: %v", err)
 						}
+						err = f.snapshotQueue.Await(f)
+						if err != nil {
+							b.Errorf("snapshot after import error: %v", err)
+						}
 						b.StopTimer()
-						stat, _ := f.file.Stat()
+						var stat os.FileInfo
+						var statTarget io.Writer
+						err = f.gen.Transaction(&statTarget, func() error {
+							targetFile, ok := statTarget.(*os.File)
+							if ok {
+								stat, _ = targetFile.Stat()
+							} else {
+								b.Errorf("couldn't stat file")
+							}
+							return nil
+						})
+						if err != nil {
+							b.Errorf("transaction error: %v", err)
+						}
 						fileSize[name] = stat.Size()
 						f.Clean(b)
 					}
@@ -2367,6 +2412,7 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
 			t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
 		}
 	}
+	f.Clean(t)
 }
 
 // getZipfRowsSliceRoaring generates a random fragment with the given number of
@@ -2512,16 +2558,28 @@ func (f *fragment) sanityCheck(t testing.TB) {
 }
 
 func (f *fragment) Clean(t testing.TB) {
-	f.awaitSnapshot()
+	f.mu.Lock()
+	err := f.snapshotQueue.Await(f)
+	f.mu.Unlock()
+	if err != nil {
+		t.Fatalf("snapshot failed before sanity check: %v", err)
+	}
 	f.sanityCheck(t)
+	if f.storage != nil && f.storage.Source != nil {
+		if f.storage.Source.Dead() {
+			t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+		}
+	}
 	errc := f.Close()
+	// prevent double-closes of generation during testing.
+	f.gen = nil
 	errf := os.Remove(f.path)
 	errp := os.Remove(f.cachePath())
 	if errc != nil || errf != nil {
 		t.Fatal("cleaning up fragment: ", errc, errf, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -2544,7 +2602,7 @@ func (f *fragment) CleanKeep(t testing.TB) {
 		t.Fatal("closing fragment: ", errc, errp)
 	}
 	if f.snapshotQueue != nil {
-		close(f.snapshotQueue)
+		f.snapshotQueue.Stop()
 		f.snapshotQueue = nil
 	}
 	// not all fragments have cache files
@@ -3032,10 +3090,10 @@ func TestFragmentRowIterator(t *testing.T) {
 
 func TestUnionInPlaceMapped(t *testing.T) {
 	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	// note: clean has to be deferred first, because it has to run with
+	// the lock *not* held, because it is sometimes so it has to grab the
+	// lock...
 	defer f.Clean(t)
-	// I know this doesn't actually matter in our current context, but
-	// strictly speaking, we do say you have to hold the lock while calling
-	// unprotectedWriteToFragment...
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	r0 := rand.New(rand.NewSource(2))
@@ -3068,8 +3126,15 @@ func TestUnionInPlaceMapped(t *testing.T) {
 	f.storage.UnionInPlace(setBM1)
 	countUnion := f.storage.Count()
 	// UnionInPlace produces no ops log, we have to make it snapshot, to
-	// ensure that the on-disk representation is correct.
-	f.enqueueSnapshot()
+	// ensure that the on-disk representation is correct. Note, UIP is
+	// not used for things that are modifying real fragments, usually;
+	// it's used only in computation of things that usually don't go to
+	// disk, which is why we handle this specially in testing and not
+	// generically.
+	err = f.snapshotQueue.Immediate(f)
+	if err != nil {
+		t.Fatalf("snapshot after union-in-place: %v", err)
+	}
 
 	if count0 != countF {
 		t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF)
@@ -3178,6 +3243,25 @@ func TestFragmentPositionsForValue(t *testing.T) {
 	}
 }
 
+func TestIntLTRegression(t *testing.T) {
+	f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	defer f.Clean(t)
+
+	_, err := f.setValue(1, 6, 33)
+	if err != nil {
+		t.Fatalf("setting value: %v", err)
+	}
+
+	row, err := f.rangeOp(pql.LT, 6, 33)
+	if err != nil {
+		t.Fatalf("doing range of: %v", err)
+	}
+
+	if !row.IsEmpty() {
+		t.Errorf("expected nothing, but got: %v", row.Columns())
+	}
+}
+
 func TestImportClearRestart(t *testing.T) {
 	tests := []struct {
 		rows []uint64
@@ -3257,7 +3341,7 @@ func TestImportClearRestart(t *testing.T) {
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
-				err = f.closeStorage(true)
+				err = f.closeStorage()
 				if err != nil {
 					t.Fatalf("closing storage: %v", err)
 				}
@@ -3291,7 +3375,7 @@ func TestImportClearRestart(t *testing.T) {
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
-				err = f2.closeStorage(true)
+				err = f2.closeStorage()
 				if err != nil {
 					t.Fatalf("f2 closing storage: %v", err)
 				}
@@ -3339,6 +3423,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) {
 
 func TestImportValueConcurrent(t *testing.T) {
 	f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+	defer f.Clean(t)
 	eg := &errgroup.Group{}
 	for i := 0; i < 4; i++ {
 		i := i
@@ -3363,7 +3448,7 @@ func TestImportMultipleValues(t *testing.T) {
 		cols      []uint64
 		vals      []int64
 		checkCols []uint64
-		checkVals []uint64
+		checkVals []int64
 		depth     uint
 	}{
 		{
@@ -3371,7 +3456,7 @@ func TestImportMultipleValues(t *testing.T) {
 			vals:      []int64{97, 100},
 			depth:     7,
 			checkCols: []uint64{0},
-			checkVals: []uint64{100},
+			checkVals: []int64{100},
 		},
 	}
 
@@ -3395,7 +3480,7 @@ func TestImportMultipleValues(t *testing.T) {
 					if !exists {
 						t.Errorf("column %d should exist", cc)
 					}
-					if n != 100 {
+					if n != cv {
 						t.Errorf("wrong value: %d is not %d", n, cv)
 					}
 				}
@@ -3405,6 +3490,66 @@ func TestImportMultipleValues(t *testing.T) {
 	}
 }
 
+func TestImportValueRowCache(t *testing.T) {
+	type testCase struct {
+		cols      []uint64
+		vals      []int64
+		checkCols []uint64
+		depth     uint
+	}
+	tests := []struct {
+		tc1 testCase
+		tc2 testCase
+	}{
+		{
+			tc1: testCase{
+				cols:      []uint64{2},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2},
+			},
+			tc2: testCase{
+				cols:      []uint64{1000},
+				vals:      []int64{1},
+				depth:     1,
+				checkCols: []uint64{2, 1000},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		for _, maxOpN := range []int{1, 10000} {
+			t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) {
+				f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
+				f.MaxOpN = maxOpN
+				defer f.Clean(t)
+
+				// First import (tc1)
+				if err := f.importValue(test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc1.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns())
+				}
+
+				// Second import (tc2)
+				if err := f.importValue(test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil {
+					t.Fatalf("importing values: %v", err)
+				}
+
+				if r, err := f.rangeOp(pql.GT, test.tc2.depth, 0); err != nil {
+					t.Error("getting range of values")
+				} else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) {
+					t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns())
+				}
+			})
+		}
+	}
+}
+
 func TestFragmentConcurrentReadWrite(t *testing.T) {
 	f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
 	defer f.Clean(t)
@@ -3431,3 +3576,47 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 
 	t.Logf("%d", acc)
 }
+
+func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+	b := []byte{60, 48, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
+	defer f.Clean(t)
+	err := f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check the bit
+	res := f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("expecting 1 got: %v", res)
+	}
+	//clear the bit
+	changed, _ := f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+	// import again
+	err = f.importRoaringT(b, false)
+	if err != nil {
+		t.Fatalf("importing roaring: %v", err)
+	}
+	//check
+	res = f.row(1).Columns()
+	if len(res) < 1 || f.row(1).Columns()[0] != 1 {
+		t.Fatalf("again expecting 1 got: %v", res)
+	}
+	changed, _ = f.clearBit(1, 1)
+	if !changed {
+		t.Fatalf("again expected change got %v", changed)
+	}
+	//check missing
+	res = f.row(1).Columns()
+	if len(res) != 0 {
+		t.Fatalf("expected nothing got %v", res)
+	}
+}

gendebug_test.go

@@ -0,0 +1,39 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func examineResults() {
+	results := reportGenerations()
+	if len(results) > 0 {
+		fmt.Printf("generations:\n")
+		for _, res := range results {
+			fmt.Printf("  %s\n", res)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	ret := m.Run()
+	examineResults()
+	os.Exit(ret)
+}

generation.go

@@ -0,0 +1,439 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pkg/errors"
+)
+
+// generation represents one "generation" of opening a data file.
+// This is what determines when it's safe to unmap a data file, if it
+// got mapped, and handles closing/reopening files if we need to
+// manage file handle availability. It's an interface because this
+// lets us write simpler code for specific cases, rather than handling
+// the whole matrix of mapped/unmapped, staying open/being reopened,
+// etcetera.
+//
+// You create a generation by calling newGeneration with a file
+// path. If it succeeds in opening that path, it calls a provided
+// setup function with the data from the generation, and a flag
+// indicating whether the data is mmapped. If the setup function
+// fails, newGeneration cleans things up and closes. Otherwise,
+// it returns a generation.
+//
+// The generation itself uses runtime.SetFinalizer to clean up when
+// the last reference to it goes away. You should store a pointer
+// to the generation in any object which is reliant on the generation.
+//
+// When you anticipate a generation should be done (for instance,
+// opening a new generation), the old one gets marked done, which
+// stashes a timestamp in it. Later operations can check whether
+// the timestamp is a while back, and if so, complain that something
+// might be wrong.
+//
+// In some cases, we don't have enough open file limit to keep every
+// file actually open. To address this, use the `Transaction` function,
+// which ensures that the file is open, stores a reference to it in
+// a provided `*io.Writer`, and then restores the previous value of
+// the io.Writer when it's done. For instance, for a bitmap, this might
+// be used with `&b.OpWriter`.
+//
+// newGeneration takes an optional previous generation; it calls
+// that generation's Done function after running the provided setup,
+// and bumps the generation count.
+type generation interface {
+	// Transaction runs the given transaction with the generation's
+	// file open. If the **os.File parameter is
+	// non-nil, the generation's file will be open, and stored
+	// into that pointer, during the execution of func, after
+	// which the previous contents are restored. Otherwise
+	// the file may or may not be open during the operation.
+	Transaction(*io.Writer, func() error) error
+	// Done() should be called exactly once, to indicate that a
+	// generation is expected not to be in use for long -- for instance,
+	// when a new generation replaces it.
+	Done()
+	// Generation count.
+	Generation() int64
+	// ID indicates the source -- path and generation number -- that
+	// this generation represents.
+	ID() string
+	// Dead indicates whether this generation is Done.
+	Dead() bool
+	// Bytes reports the storage associated with this generation, if any.
+	// DO NOT USE THIS. Except if you're debugging mmap segfaults.
+	Bytes() []byte
+}
+
+type mmapGeneration struct {
+	mu         sync.Mutex // mutex guards modifiers of generation, not of data
+	transMu    sync.Mutex // guards transactions, specifically
+	path       string
+	id         string
+	file       *os.File
+	data       []byte
+	generation int64     // generation counter
+	dead       bool      // we think this generation is dead
+	deadSince  time.Time // when this generation was marked dead
+	retries    int       // for cases where we're retrying
+	logger     logger.Logger
+}
+
+func (m *mmapGeneration) Dead() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.dead
+}
+
+func (m *mmapGeneration) ID() string {
+	return m.id
+}
+
+func (m *mmapGeneration) Generation() int64 {
+	return m.generation
+}
+
+// Transaction runs an exclusive call, ensuring that the file is open if
+// the *io.Writer parameter is present.
+func (m *mmapGeneration) Transaction(fileP *io.Writer, fn func() error) (transactionErr error) {
+	m.transMu.Lock()
+	defer m.transMu.Unlock()
+	// HEY LOOK CAREFULLY AT THIS BIT:
+	// We can't just defer this unlock. We specifically want to be
+	// sure to unlock the regular mutex *before* this function is over,
+	// and if we error out trying to open the file, we want to do it
+	// even sooner. If we deferred this, the transaction would block
+	// *everything*, including things like sanity checks against the
+	// generation being Dead(), but also including the deferred
+	// re-close-the-file.
+	m.mu.Lock()
+	// if we've been asked for a file pointer, we need to ensure that
+	// our file is open, and that the file pointer to it is stored in
+	// the requested location, then revert that when we're done.
+	// if we aren't asked for a file pointer, nothing needs the file
+	// open.
+	if m.dead {
+		elapsed := time.Since(m.deadSince)
+		m.logger.Printf("WARNING: transaction against %s, which has been dead for %v\n", m.id, elapsed)
+	}
+	if fileP != nil {
+		if m.file == nil {
+			// we ignore the shouldClose response here; if this
+			// fragment was previously not being kept open, we're
+			// going to stick with that.
+			_, err := m.openFile()
+			if err != nil {
+				m.mu.Unlock()
+				return err
+			}
+			defer func() {
+				// report a close error if we have no other error to report
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				err := m.closeFile()
+				if transactionErr == nil {
+					transactionErr = err
+				}
+			}()
+		}
+		var fileStash io.Writer
+		fileStash, *fileP = *fileP, m.file
+		defer func() {
+			*fileP = fileStash
+		}()
+	}
+	// We are done locking the generation itself for now.
+	m.mu.Unlock()
+	wouldPanic := debug.SetPanicOnFault(true)
+	defer func() {
+		debug.SetPanicOnFault(wouldPanic)
+		if r := recover(); r != nil {
+			if err, ok := r.(error); ok {
+				// special case: if we caught a page fault, we diagnose that directly. sadly,
+				// we can't see the actual values that were used to generate this, probably.
+				if err.Error() == "runtime error: invalid memory address or nil pointer dereference" {
+					if transactionErr == nil {
+						transactionErr = errors.New("invalid memory access during transaction")
+					} else {
+						transactionErr = fmt.Errorf("invalid memory access during transaction, previous error %v", transactionErr)
+					}
+					return
+				}
+			}
+			if transactionErr == nil {
+				transactionErr = fmt.Errorf("panic during transaction: %v", r)
+			} else {
+				transactionErr = fmt.Errorf("panic during erroring transaction: panic %v, previous error %v", r, transactionErr)
+			}
+		}
+	}()
+	return fn()
+}
+
+func (m *mmapGeneration) Bytes() []byte {
+	return m.data
+}
+
+// Done marks the generation done, and closes its file, but may not unmap it.
+// It's still conceptually possible to end up doing a Transaction against a
+// done generation, but it's a red flag.
+func (m *mmapGeneration) Done() {
+	if m == nil {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.dead {
+		oops := fmt.Sprintf("generation %s, marked done again at %v, previously marked dead at %v",
+			m.id, time.Now(), m.deadSince)
+		panic(oops)
+	}
+	m.dead = true
+	m.deadSince = time.Now()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error closing generation %s: %v", m.id, err)
+	}
+	// If we're not debugging, the finalizer won't have been enabled
+	// previously. Finalizers have non-zero cost, so having them not be
+	// created until they're needed seems rewarding?
+	if !generationDebug {
+		runtime.SetFinalizer(m, generationFinalizer)
+	}
+	endGeneration(m.id)
+	// note, Done() doesn't close the file; only the finalizer actually
+	// does the shutdown.
+}
+
+// Try to close the file if it's currently open.
+func (m *mmapGeneration) closeFile() error {
+	var lastErr error
+	// report the most serious error encountered, but still close
+	// file even if something else failed.
+	if m.file != nil {
+		if err := m.file.Sync(); err != nil {
+			lastErr = fmt.Errorf("sync: %s", err)
+		}
+		if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_UN); err != nil {
+			lastErr = fmt.Errorf("unlock: %s", err)
+		}
+		if err := syswrap.CloseFile(m.file); err != nil {
+			lastErr = fmt.Errorf("close file: %s", err)
+		}
+		m.file = nil
+	}
+	return lastErr
+}
+
+// openFile ensures the file is open and locked, or fails. If it does
+// open the file, it will also report the "you need to close this file
+// when you're done" flag from syswrap.
+func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
+	if m.file != nil {
+		return false, nil
+	}
+	m.file, shouldClose, err = syswrap.OpenFile(m.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return false, err
+	}
+	// do we actually want this in every openFile? I don't know.
+	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		m.file.Close()
+		m.file = nil
+		return false, fmt.Errorf("flock: %s", err)
+	}
+	return shouldClose, nil
+}
+
+func generationFinalizer(m *mmapGeneration) {
+	m.mu.Lock()
+	if !m.dead {
+		m.logger.Printf("finalizing generation %s which isn't dead yet\n",
+			m.id)
+	}
+	m.mu.Unlock()
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("finalizing generation, closing file: %v\n", err)
+	}
+	if m.data != nil {
+		err := syswrap.Munmap(m.data)
+		if err != nil {
+			m.logger.Printf("finalizing generation, munmap: %v\n", err)
+		}
+		m.data = nil
+	}
+	finalizeGeneration(m.id)
+}
+
+// Cancel closes a generation out entirely. It cancels any finalizer,
+// unmaps any data, ends generation tracking, and closes any files.
+// It does each of these separately whether or not the others need to be done,
+// or succeed. It's used to handle failures from newGeneration; it makes sure
+// the generation isn't holding any resources and doesn't need to be cleaned
+// up otherwise.
+//
+// Mostly a helper function because there's several cases where newGeneration
+// might fail.
+func (m *mmapGeneration) Cancel() {
+	if m.data != nil {
+		_ = syswrap.Munmap(m.data)
+		m.data = nil
+	}
+	err := m.closeFile()
+	if err != nil {
+		m.logger.Printf("error cancelling generation %s: %v", m.id, err)
+	}
+	runtime.SetFinalizer(m, nil)
+	m.dead = true
+	m.deadSince = time.Now()
+	cancelGeneration(m.id)
+}
+
+// newGeneration creates a new generation using the given file path. It
+// then calls the provided setup function with the allocated storage, a
+// file handle, the new generation, and a flag indicatting whether the storage
+// is memory-mapped. If the setup function returns a non-nil error, the
+// generation is cleaned up, and newGeneration fails. The setup function
+// also returns a boolean indicating whether it used the mapping; if it
+// didn't, newGeneration discards the mapping and returns a nil generation.
+//
+// If generationDebug is enabled, we track the generation even if no mapping
+// is actually in use, so we can verify that the tracking is working.
+//
+// On failure, newGeneration returns nil values for generation and func,
+// and an error. On success, the func returned is the close func to use
+// when the generation is no longer needed by the caller.
+func newGeneration(existing generation, path string, readData bool, setup func([]byte, *os.File, generation, bool) (bool, error), logger logger.Logger) (generation, error) {
+	m := mmapGeneration{path: path, logger: logger}
+	if existing != nil {
+		m.generation = existing.Generation() + 1
+		// we might keep a previous generation around just for its generation count.
+		if !existing.Dead() {
+			defer existing.Done()
+		}
+	}
+	shouldClose, err := m.openFile()
+	if err != nil {
+		return nil, err
+	}
+	m.id = fmt.Sprintf("%s:%d", m.path, m.generation)
+	// possibly assign new generation ID if this one's been used, which can
+	// happen with reopens, especially during testing.
+	m.id = registerGeneration(m.id)
+	// if debugging, we always want the finalizer on so we notice if a
+	// generation is finalized without being closed. for non-debugging
+	// use, we only need it when the generation is closed.
+	if generationDebug {
+		runtime.SetFinalizer(&m, generationFinalizer)
+	}
+	// Mmap the underlying file so it can be zero copied.
+	var mapped bool
+	var data []byte
+	fi, err := m.file.Stat()
+	if err == nil && fi.Size() > 0 {
+		data, err = syswrap.Mmap(int(m.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
+		if err == syswrap.ErrMaxMapCountReached {
+			// I have no idea where/how to display this message.
+			m.logger.Printf("maximum number of maps reached, reading file '%s' instead", m.path)
+		} else if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "mmap failed")
+		} else {
+			mapped = true
+		}
+	}
+	if data == nil && readData {
+		data, err = ioutil.ReadAll(m.file)
+		if err != nil {
+			m.Cancel()
+			return nil, errors.Wrap(err, "failure file readall")
+		}
+	}
+	// if we got here, data's the expected data, so let's try to use it
+	mappedAny, err := setup(data, m.file, &m, mapped)
+
+	// if the setup failed, we unmap data if we previously mapped it,
+	// and exit. Note that having no data, or having only trivial
+	// data (like a zero-container Roaring file) isn't "failed".
+	if err != nil {
+		m.Cancel()
+		// Unless, that is, we think the file probably ought to
+		// be truncated: For instance, if a bitmap has a corrupted
+		// ops log, we could truncate that part of it and retry.
+		if err, ok := err.(roaring.FileShouldBeTruncatedError); ok && m.retries < 1 {
+			m.logger.Printf("file %s read partially, but should-be-truncated at %d bytes\n", m.path, err.SuggestedLength())
+			// close this generation, then try again. once.
+			m.retries++
+			err := os.Truncate(m.path, err.SuggestedLength())
+			if err != nil {
+				m.logger.Printf("truncating file failed [but retrying anyway]: %v\n", err)
+			}
+			return newGeneration(&m, path, readData, setup, logger)
+		}
+		return nil, err
+	}
+
+	if mapped {
+		// when generationDebug is on, we want to track this even
+		// if it's not being used.
+		if generationDebug || mappedAny {
+			// Advise the kernel that the mmap is accessed randomly.
+			// We don't care much about errors with this.
+			_ = madvise(data, syscall.MADV_RANDOM)
+			// store the data, so we can unmap it when this generation
+			// gets finalized.
+			m.data = data
+		} else {
+			// unmap the data and don't stash the pointer in this
+			// generation. It's not being used. This generation
+			// doesn't need to exist, yay.
+			unmapErr := syswrap.Munmap(data)
+			if unmapErr != nil {
+				m.logger.Printf("error unmapping (probably harmless): %v", unmapErr)
+			}
+		}
+	}
+	// shouldClose comes from underlying syswrap.OpenFile, which checks
+	// a count of open files to hint at us when we need to start closing
+	// files to preserve open file descriptor limit.
+	if shouldClose {
+		err := m.closeFile()
+		if err != nil {
+			m.logger.Printf("closing file to preserve open files failed: %v\n", err)
+		}
+	}
+	// It's possible that the generation has no actual data to track,
+	// because nothing's mapped, in which case there won't be any bitmap
+	// sources following this, just the fragment source. (Bitmaps won't
+	// be attached to the source unless they're actually mapped to it,
+	// or generationDebug is true). That's okay. We pay a tiny cost
+	// for the finalizer, but we also get higher confidence that it really
+	// does get cleaned up.
+	return &m, nil
+}

generation_debug.go

@@ -0,0 +1,160 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"sort"
+	"sync"
+	"time"
+)
+
+const generationDebug = true
+
+type lifespan struct {
+	from, to, finalized time.Time
+}
+
+var knownGenerations map[string]lifespan
+var knownGenerationLock sync.Mutex
+
+var timeZero time.Time
+
+func registerGeneration(id string) string {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	if knownGenerations == nil {
+		knownGenerations = make(map[string]lifespan)
+	}
+	newSpan := lifespan{from: time.Now()}
+	origId := id
+
+	// if you have more than 65k of the same file open, maybe you have bigger
+	// problems than this.
+	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
+		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
+		if span.finalized != timeZero {
+			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
+				id, suffix, span.from, span.to, span.finalized)
+		} else {
+			if span.to != timeZero {
+				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
+			} else {
+				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
+			}
+		}
+		id = origId + suffix
+	}
+	fmt.Printf("new generation %s\n", id)
+	knownGenerations[id] = newSpan
+	return id
+}
+
+func endGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("ending generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero || span.to != timeZero {
+		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
+		panic(oops)
+	}
+	span.to = time.Now()
+	knownGenerations[id] = span
+}
+
+// cancelGeneration marks the generation as finalized. In principle it's
+// only used in cases where we just started a generation but something
+// went wrong. it's not fancier than this because of the weird cases
+// where the same generation shows up again, such as when closing and
+// reopening an index so we don't know about previous instances of the
+// same files.
+func cancelGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if exists {
+		span.finalized = time.Now()
+		span.to = span.finalized
+		knownGenerations[id] = span
+	}
+}
+
+func finalizeGeneration(id string) {
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	span, exists := knownGenerations[id]
+	if !exists {
+		oops := fmt.Sprintf("finalizing generation %s: unknown", id)
+		panic(oops)
+	}
+	if span.finalized != timeZero {
+		var oops string
+		if span.to != timeZero {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
+		} else {
+			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
+		}
+		panic(oops)
+	}
+	span.finalized = time.Now()
+	knownGenerations[id] = span
+}
+
+func reportGenerations() []string {
+	runtime.GC()
+	knownGenerationLock.Lock()
+	defer knownGenerationLock.Unlock()
+	var surviving []string
+	times := make([]int64, 0, len(knownGenerations))
+	for id, span := range knownGenerations {
+		if span.to == timeZero {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
+			} else {
+				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
+			}
+		} else {
+			if span.finalized == timeZero {
+				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
+			} else {
+				times = append(times, int64(span.finalized.Sub(span.to)))
+			}
+		}
+	}
+	if len(times) > 0 {
+		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
+		var total int64
+		for _, d := range times {
+			total += d
+		}
+		var mean, median, p90, p99, worst int64
+		mean = total / int64(len(times))
+		median = times[len(times)/2]
+		p90 = times[(len(times)*9)/10]
+		p99 = times[(len(times)*99)/100]
+		worst = times[len(times)-1]
+		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+	}
+	return surviving
+}

generation_nodebug.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package pilosa
+
+const generationDebug = false
+
+func registerGeneration(id string) string {
+	return id
+}
+
+func endGeneration(id string) {
+}
+
+func cancelGeneration(id string) {
+}
+
+func finalizeGeneration(id string) {
+}
+
+//lint:ignore U1000 this is conditional on a build flag, see generation_test.go.
+func reportGenerations() []string { //nolint:unused,deadcode
+	return nil
+}

generation_test.go

@@ -0,0 +1,67 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// +build generationparanoia
+
+package pilosa
+
+import (
+	"runtime"
+	"testing"
+	"unsafe"
+)
+
+func TestGenerationPanic(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "none")
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	_ = f.Snapshot()
+	_ = f.row(0)
+	var prevData []byte
+
+	if f.gen.(*mmapGeneration).data == nil {
+		t.Fatalf("generation code didn't create a mapping, apparently?")
+	}
+	prevData = f.gen.(*mmapGeneration).data
+	f.mu.Lock()
+	_ = f.snapshotQueue.Immediate(f)
+	f.mu.Unlock()
+	runtime.GC()
+	for i := 0; i < (f.MaxOpN / 2); i++ {
+		_, _ = f.setBit(0, uint64(i*32)+23)
+	}
+	f.mu.Lock()
+	f.snapshotQueue.Await(f)
+	f.mu.Unlock()
+	runtime.GC()
+	newData := f.gen.(*mmapGeneration).data
+	if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
+		t.Fatalf("test can't run usefully, didn't get new data pointer")
+	}
+
+	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+		prevData[0] = 0x3c
+		return nil
+	})
+	if err == nil {
+		t.Fatalf("expected a panic to get caught, but nothing happened")
+	}
+	if err.Error() != "invalid memory access during transaction" {
+		t.Fatalf("expected \"invalid memory access during transaction\", got %q", err.Error())
+	}
+}

go.mod

@@ -3,7 +3,6 @@ module github.com/pilosa/pilosa/v2
 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -13,12 +12,14 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
-	github.com/golang/protobuf v1.3.1
+	github.com/golang/protobuf v1.3.2
 	github.com/google/go-cmp v0.2.0
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
+	github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
 	github.com/pkg/errors v0.8.1
@@ -36,12 +37,13 @@ require (
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
-	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect
+	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
 	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/grpc v1.24.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 )
 
-go 1.11
+go 1.13

go.sum

@@ -1,3 +1,4 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
@@ -20,6 +21,7 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -39,10 +41,15 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -80,6 +87,14 @@ github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
+github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
+github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 h1:XOImsA5XhGklFj8Y0TxSm1qWZzEwYxom2JOXiu9GMq0=
+github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
+github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@@ -90,6 +105,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -153,13 +169,16 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -180,12 +199,20 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=

handler.go

@@ -16,6 +16,9 @@ package pilosa
 
 import (
 	"encoding/json"
+
+	"github.com/pilosa/pilosa/v2/tracing"
+	"github.com/pkg/errors"
 )
 
 // QueryRequest represent a request to process a query.
@@ -42,6 +45,13 @@ type QueryRequest struct {
 	// If true, indicates that query is part of a larger distributed query.
 	// If false, this request is on the originating node.
 	Remote bool
+
+	// Should we profile this query?
+	Profile bool
+
+	// Additional data associated with the query, in cases where there's
+	// row-style inputs for precomputed values.
+	EmbeddedData []*Row
 }
 
 // QueryResponse represent a response from a processed query.
@@ -55,6 +65,9 @@ type QueryResponse struct {
 
 	// Error during parsing or execution.
 	Err error
+
+	// Profiling data, if any
+	Profile *tracing.Profile
 }
 
 // MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
@@ -68,9 +81,11 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
 	return json.Marshal(struct {
 		Results        []interface{}    `json:"results"`
 		ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
+		Profile        *tracing.Profile `json:"profile,omitempty"`
 	}{
 		Results:        resp.Results,
 		ColumnAttrSets: resp.ColumnAttrSets,
+		Profile:        resp.Profile,
 	})
 }
 
@@ -97,12 +112,63 @@ var NopHandler Handler = nopHandler{}
 // ImportValueRequest describes the import request structure
 // for a value (BSI) import.
 type ImportValueRequest struct {
-	Index      string
-	Field      string
-	Shard      uint64
-	ColumnIDs  []uint64
-	ColumnKeys []string
-	Values     []int64
+	Index string
+	Field string
+	// if Shard is MaxUint64 (an impossible shard value), this
+	// indicates that the column IDs may come from multiple shards.
+	Shard        uint64
+	ColumnIDs    []uint64
+	ColumnKeys   []string
+	Values       []int64
+	FloatValues  []float64
+	StringValues []string
+}
+
+func (ivr *ImportValueRequest) Len() int           { return len(ivr.ColumnIDs) }
+func (ivr *ImportValueRequest) Less(i, j int) bool { return ivr.ColumnIDs[i] < ivr.ColumnIDs[j] }
+func (ivr *ImportValueRequest) Swap(i, j int) {
+	ivr.ColumnIDs[i], ivr.ColumnIDs[j] = ivr.ColumnIDs[j], ivr.ColumnIDs[i]
+	if len(ivr.Values) > 0 {
+		ivr.Values[i], ivr.Values[j] = ivr.Values[j], ivr.Values[i]
+	} else if len(ivr.FloatValues) > 0 {
+		ivr.FloatValues[i], ivr.FloatValues[j] = ivr.FloatValues[j], ivr.FloatValues[i]
+	} else if len(ivr.StringValues) > 0 {
+		ivr.StringValues[i], ivr.StringValues[j] = ivr.StringValues[j], ivr.StringValues[i]
+	}
+}
+
+// Validate ensures that the payload of the request is valid.
+func (ivr *ImportValueRequest) Validate() error {
+	if ivr.Index == "" || ivr.Field == "" {
+		return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
+	}
+	if len(ivr.ColumnIDs) != 0 && len(ivr.ColumnKeys) != 0 {
+		return errors.Errorf("must pass either column ids or keys, but not both")
+	}
+	var valueSetCount int
+	if len(ivr.Values) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.FloatValues) != 0 {
+		valueSetCount++
+	}
+	if len(ivr.StringValues) != 0 {
+		valueSetCount++
+	}
+	if valueSetCount > 1 {
+		return errors.Errorf("must pass ints, floats, or strings but not multiple")
+	}
+	return nil
+}
+
+// ImportColumnAttrsRequest describes the import request structure
+// for a ColumnAttr import
+type ImportColumnAttrsRequest struct {
+	AttrKey   string
+	ColumnIDs []uint64
+	AttrVals  []string
+	Shard     int64
+	Index     string
 }
 
 // ImportRequest describes the import request structure
@@ -160,3 +226,17 @@ type TranslateKeysRequest struct {
 type TranslateKeysResponse struct {
 	IDs []uint64
 }
+
+// TranslateIDsRequest describes the structure of a request
+// for a batch of id translations.
+type TranslateIDsRequest struct {
+	Index string
+	Field string
+	IDs   []uint64
+}
+
+// TranslateIDsResponse is the structured response of a id
+// translation request.
+type TranslateIDsResponse struct {
+	Keys []string
+}

holder.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -50,6 +51,9 @@ const (
 type Holder struct {
 	mu sync.RWMutex
 
+	// Partition count used by translation.
+	partitionN int
+
 	// Indexes by name.
 	indexes map[string]*Index
 
@@ -75,15 +79,23 @@ type Holder struct {
 
 	Logger logger.Logger
 
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
-	// Manages replication from the primary node.
-	primaryTranslateNode     *Node
-	translateStoreReplicator *holderTranslateStoreReplicator
+	// Instantiates new translation stores
+	OpenTranslateStore  OpenTranslateStoreFunc
+	OpenTranslateReader OpenTranslateReaderFunc
 
-	// Instantiates new translation stores for indexes & fields.
-	OpenTranslateStore  OpenTranslateStoreFunc  // local store
-	OpenTranslateReader OpenTranslateReaderFunc // replication
+	translationSyncer translationSyncer
+
+	// Queue of fields (having a foreign index) which have
+	// opened before their foreign index has opened.
+	foreignIndexFields []*Field
+
+	// opening is set to true while Holder is opening.
+	// It's used to determine if foreign index application
+	// needs to be queued and completed after all indexes
+	// have opened.
+	opening bool
 }
 
 // lockedChan looks a little ridiculous admittedly, but exists for good reason.
@@ -113,10 +125,11 @@ func (lc *lockedChan) Recv() {
 }
 
 // NewHolder returns a new instance of Holder.
-func NewHolder() *Holder {
+func NewHolder(partitionN int) *Holder {
 	return &Holder{
-		indexes: make(map[string]*Index),
-		closing: make(chan struct{}),
+		partitionN: partitionN,
+		indexes:    make(map[string]*Index),
+		closing:    make(chan struct{}),
 
 		opened: lockedChan{ch: make(chan struct{})},
 
@@ -127,14 +140,19 @@ func NewHolder() *Holder {
 
 		cacheFlushInterval: defaultCacheFlushInterval,
 
-		Logger: logger.NopLogger,
-
 		OpenTranslateStore: OpenInMemTranslateStore,
+
+		translationSyncer: NopTranslationSyncer,
+
+		Logger: logger.NopLogger,
 	}
 }
 
 // Open initializes the root data directory for the holder.
 func (h *Holder) Open() error {
+	h.opening = true
+	defer func() { h.opening = false }()
+
 	// Reset closing in case Holder is being reopened.
 	h.closing = make(chan struct{})
 
@@ -167,7 +185,7 @@ func (h *Holder) Open() error {
 	// Run snapshots asynchronously. The snapshotQueue will have a background
 	// task associated with it which flushes it and waits until this channel
 	// is closed, so we should always close this channel when done.
-	h.snapshotQueue = newSnapshotQueue(100, 2, h.Logger)
+	h.snapshotQueue = newSnapshotQueue(10, 2, h.Logger)
 
 	for _, fi := range fis {
 		// Skip files or hidden directories.
@@ -196,6 +214,14 @@ func (h *Holder) Open() error {
 		h.indexes[index.Name()] = index
 		h.mu.Unlock()
 	}
+
+	// If any fields were opened before their foreign index
+	// was opened, it's safe to process those now since all index
+	// opens have completed by this point.
+	if err := h.processForeignIndexFields(); err != nil {
+		return errors.Wrap(err, "processing foreign index fields")
+	}
+
 	h.Logger.Printf("open holder: complete")
 
 	// Periodically flush cache.
@@ -203,8 +229,37 @@ func (h *Holder) Open() error {
 	go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
 
 	h.Stats.Open()
+	h.snapshotQueue.ScanHolder(h)
 
 	h.opened.Close()
+
+	return nil
+}
+
+// checkForeignIndex is a check before applying a foreign
+// index to a field; if the index is not yet available,
+// (because holder is still opening and may not have opened
+// the index yet), this method queues it up to be processed
+// once all indexes have been opened.
+func (h *Holder) checkForeignIndex(f *Field) error {
+	if h.opening {
+		if fi := h.Index(f.options.ForeignIndex); fi == nil {
+			h.foreignIndexFields = append(h.foreignIndexFields, f)
+			return nil
+		}
+	}
+	return f.applyForeignIndex()
+}
+
+// processForeignIndexFields applies a foreign index to any
+// fields which were opened before their foreign index.
+func (h *Holder) processForeignIndexFields() error {
+	for _, f := range h.foreignIndexFields {
+		if err := f.applyForeignIndex(); err != nil {
+			return errors.Wrap(err, "applying foreign index")
+		}
+	}
+	h.foreignIndexFields = h.foreignIndexFields[:0] // reset
 	return nil
 }
 
@@ -222,8 +277,7 @@ func (h *Holder) Close() error {
 		}
 	}
 	if h.snapshotQueue != nil {
-		close(h.snapshotQueue)
-		// assuming the snapshotQueueWorker has already started, this is safe.
+		h.snapshotQueue.Stop()
 		h.snapshotQueue = nil
 	}
 
@@ -232,13 +286,6 @@ func (h *Holder) Close() error {
 	h.opened.ch = make(chan struct{})
 	h.opened.mu.Unlock()
 
-	h.mu.Lock()
-	if h.translateStoreReplicator != nil {
-		h.translateStoreReplicator.Close()
-		h.translateStoreReplicator = nil
-	}
-	h.mu.Unlock()
-
 	return nil
 }
 
@@ -350,7 +397,7 @@ func (h *Holder) applySchema(schema *Schema) error {
 		}
 		// Create fields that don't exist.
 		for _, f := range index.Fields {
-			field, err := idx.createFieldIfNotExists(f.Name, f.Options)
+			field, err := idx.createFieldIfNotExists(f.Name, &f.Options)
 			if err != nil {
 				return errors.Wrap(err, "creating field")
 			}
@@ -398,7 +445,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 	defer h.mu.Unlock()
 
 	// Ensure index doesn't already exist.
-	if h.indexes[name] != nil {
+	if h.index(name) != nil {
 		return nil, newConflictError(ErrIndexExists)
 	}
 	return h.createIndex(name, opt)
@@ -407,21 +454,15 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
 // CreateIndexIfNotExists returns an index by name.
 // The index is created if it does not already exist.
 func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error) {
-	h.mu.RLock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 
-	// Find index in cache first.
-	if index := h.indexes[name]; index != nil {
-		h.mu.RUnlock()
+	// Return index if it exists.
+	if index := h.index(name); index != nil {
 		return index, nil
 	}
 
-	h.mu.RUnlock()
-
-	index, err := h.CreateIndex(name, opt)
-	if _, ok := err.(ConflictError); err != nil && !ok {
-		return nil, err
-	}
-	return index, nil
+	return h.createIndex(name, opt)
 }
 
 func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
@@ -429,11 +470,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 		return nil, errors.New("index name required")
 	}
 
-	// Return index if it exists.
-	if index := h.index(name); index != nil {
-		return index, nil
-	}
-
 	// Otherwise create a new index.
 	index, err := h.newIndex(h.IndexPath(name), name)
 	if err != nil {
@@ -443,23 +479,27 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err := index.Open(); err != nil {
+	if err = index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
-	} else if err := index.saveMeta(); err != nil {
+	}
+	if err = index.saveMeta(); err != nil {
 		return nil, errors.Wrap(err, "meta")
 	}
 
 	// Update options.
 	h.indexes[index.Name()] = index
 
-	// Restart replication.
-	go h.refreshTranslateStoreReplicator()
+	// Since this is a new index, we need to kick off
+	// its translation sync.
+	if err := h.translationSyncer.Reset(); err != nil {
+		return nil, errors.Wrap(err, "resetting translation sync")
+	}
 
 	return index, nil
 }
 
 func (h *Holder) newIndex(path, name string) (*Index, error) {
-	index, err := NewIndex(path, name)
+	index, err := NewIndex(path, name, h.partitionN)
 	if err != nil {
 		return nil, err
 	}
@@ -469,8 +509,9 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
 	index.newAttrStore = h.NewAttrStore
 	index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data"))
 	index.snapshotQueue = h.snapshotQueue
-	index.holder = h
 	index.OpenTranslateStore = h.OpenTranslateStore
+	index.translationSyncer = h.translationSyncer
+	index.holder = h
 	return index, nil
 }
 
@@ -498,7 +539,10 @@ func (h *Holder) DeleteIndex(name string) error {
 	// Remove reference.
 	delete(h.indexes, name)
 
-	return nil
+	// I'm not sure if calling Reset() here is necessary
+	// since closing the index stops its translation
+	// sync processes.
+	return h.translationSyncer.Reset()
 }
 
 // Field returns the field for an index and name.
@@ -667,243 +711,6 @@ func (h *Holder) logStartup() error {
 	return nil
 }
 
-// TranslateStore returns store for the given index or field.
-func (h *Holder) TranslateStore(index, field string) (TranslateStore, error) {
-	if field == "" {
-		idx := h.Index(index)
-		if idx == nil {
-			return nil, ErrIndexNotFound
-		}
-		return idx.TranslateStore(), nil
-	}
-
-	f := h.Field(index, field)
-	if f == nil {
-		return nil, ErrFieldNotFound
-	}
-	return f.TranslateStore(), nil
-}
-
-// TranslateOffsetMap returns a map of offsets for all indexes & fields.
-func (h *Holder) TranslateOffsetMap() (TranslateOffsetMap, error) {
-	m := make(TranslateOffsetMap)
-	for _, idx := range h.Indexes() {
-		id, err := idx.TranslateStore().MaxID()
-		if err != nil {
-			return nil, err
-		}
-		m.SetIndexOffset(idx.Name(), id+1)
-
-		for _, field := range idx.Fields() {
-			id, err := field.TranslateStore().MaxID()
-			if err != nil {
-				return nil, err
-			}
-			m.SetFieldOffset(idx.Name(), field.Name(), id+1)
-		}
-	}
-	return m, nil
-}
-
-func (h *Holder) setTranslateStoreReadOnly(v bool) {
-	for _, idx := range h.Indexes() {
-		idx.TranslateStore().SetReadOnly(v)
-		for _, field := range idx.Fields() {
-			field.TranslateStore().SetReadOnly(v)
-		}
-	}
-}
-
-func (h *Holder) setPrimaryTranslateStore(node *Node) error {
-	if node != nil && h.OpenTranslateReader == nil {
-		return nil
-	}
-	h.mu.Lock()
-	h.primaryTranslateNode = node.Clone()
-	h.mu.Unlock()
-
-	go h.refreshTranslateStoreReplicator()
-	return nil
-}
-
-func (h *Holder) refreshTranslateStoreReplicator() {
-	h.mu.RLock()
-	node := h.primaryTranslateNode
-	h.mu.RUnlock()
-
-	var nodeURL string
-	if node != nil {
-		u := node.URI.URL()
-		nodeURL = u.String()
-	}
-
-	// Stop existing replication, if running.
-	h.mu.Lock()
-	if h.translateStoreReplicator != nil {
-		h.translateStoreReplicator.Close()
-		h.translateStoreReplicator = nil
-	}
-	h.mu.Unlock()
-
-	// Set all stores read only mode based on if we have a primary.
-	h.setTranslateStoreReadOnly(node != nil)
-
-	// Start replication monitor, if needed.
-	h.mu.Lock()
-	defer h.mu.Unlock()
-	if nodeURL != "" {
-		h.translateStoreReplicator = newHolderTranslateStoreReplicator(h, nodeURL)
-		h.translateStoreReplicator.logger = h.Logger
-		if err := h.translateStoreReplicator.Open(); err != nil {
-			h.Logger.Printf("cannot open translate store replicator: %s", err)
-		}
-	}
-}
-
-// TranslateEntryReader returns a reader that merges all index & field reader
-// that are specified in the offsets map.
-func (h *Holder) TranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
-	// Ensure all readers are cleaned up if any error.
-	var a []TranslateEntryReader
-	defer func() {
-		if err != nil {
-			for i := range a {
-				a[i].Close()
-			}
-		}
-	}()
-
-	// Fetch all readers.
-	for indexName, m := range offsets {
-		for fieldName, offset := range m {
-			var store TranslateStore
-
-			idx := h.Index(indexName)
-			if idx == nil {
-				return nil, ErrIndexNotFound
-			}
-
-			// Fetch from index or field store.
-			if fieldName == "" {
-				store = idx.TranslateStore()
-			} else {
-				f := idx.Field(fieldName)
-				if f == nil {
-					return nil, ErrFieldNotFound
-				}
-				store = f.TranslateStore()
-			}
-
-			// Generate reader and append to multireader.
-			r, err := store.EntryReader(ctx, uint64(offset))
-			if err != nil {
-				return nil, errors.Wrap(err, "translate reader")
-			}
-			a = append(a, r)
-		}
-	}
-
-	return NewMultiTranslateEntryReader(ctx, a), nil
-}
-
-// holderTranslateStoreReplicator manages the replication of translation store
-// data from a primary store to the local replica. Continually tries to
-// reconnect on disconnect.
-type holderTranslateStoreReplicator struct {
-	ctx    context.Context
-	cancel func()
-	wg     sync.WaitGroup
-
-	holder  *Holder
-	nodeURL string
-
-	logger logger.Logger
-}
-
-func newHolderTranslateStoreReplicator(h *Holder, nodeURL string) *holderTranslateStoreReplicator {
-	r := &holderTranslateStoreReplicator{
-		holder:  h,
-		nodeURL: nodeURL,
-		logger:  logger.NopLogger,
-	}
-	r.ctx, r.cancel = context.WithCancel(context.Background())
-	return r
-}
-
-// Open starts the background monitoring goroutine.
-func (r *holderTranslateStoreReplicator) Open() error {
-	r.wg.Add(1)
-	go func() { defer r.wg.Done(); r.monitor() }()
-	return nil
-}
-
-// Close stops the replicator.
-func (r *holderTranslateStoreReplicator) Close() error {
-	r.cancel()
-	return nil
-}
-
-// monitor runs in a background goroutine and continually tries to connect and
-// stream translate changes from the primary store.
-func (r *holderTranslateStoreReplicator) monitor() {
-	for {
-		select {
-		case <-r.ctx.Done():
-			return
-		default:
-			if err := r.replicate(); err != nil {
-				r.logger.Printf("cannot replicate: nodeURL=%s err=%s", r.nodeURL, err)
-			}
-			time.Sleep(1 * time.Second)
-		}
-	}
-}
-
-func (r *holderTranslateStoreReplicator) replicate() error {
-	// Determine the offsets of every index & field store.
-	offsets, err := r.holder.TranslateOffsetMap()
-	if err != nil {
-		return err
-	} else if len(offsets) == 0 {
-		return nil
-	}
-
-	// Begin streaming from remote primary.
-	rd, err := r.holder.OpenTranslateReader(r.ctx, r.nodeURL, offsets)
-	if err != nil {
-		return err
-	}
-	defer rd.Close()
-
-	for {
-		var entry TranslateEntry
-		if err := rd.ReadEntry(&entry); err != nil {
-			return err
-		}
-
-		// Find appropriate store.
-		var store TranslateStore
-		if entry.Field == "" {
-			idx := r.holder.Index(entry.Index)
-			if idx == nil {
-				return ErrIndexNotFound
-			}
-			store = idx.TranslateStore()
-		} else {
-			f := r.holder.Field(entry.Index, entry.Field)
-			if f == nil {
-				return ErrFieldNotFound
-			}
-			store = f.TranslateStore()
-		}
-
-		// Apply replication to store.
-		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
-			return err
-		}
-	}
-}
-
 // holderSyncer is an active anti-entropy tool that compares the local holder
 // with a remote holder based on block checksums and resolves differences.
 type holderSyncer struct {
@@ -914,6 +721,9 @@ type holderSyncer struct {
 	Node    *Node
 	Cluster *cluster
 
+	// Translation sync handling.
+	readers []TranslateEntryReader
+
 	// Stats
 	Stats stats.StatsClient
 
@@ -1101,6 +911,16 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
 		return ErrFieldNotFound
 	}
 
+	// TODO: this is a temporary fix put in place to prevent
+	// the syncer from trying to sync replicas of fields
+	// other than `set` or `time` (i.e. `int`, `mutex`, `bool`,
+	// `decimal`) using ImportRoaring, because ImportRoaring
+	// only supports `set` and `time` fields.
+	if f.Type() != FieldTypeSet && f.Type() != FieldTypeTime {
+		s.Holder.Logger.Printf("temporarily skipping fragment sync: %s/%d", field, shard)
+		return nil
+	}
+
 	// Ensure view exists locally.
 	v, err := f.createViewIfNotExists(view)
 	if err != nil {
@@ -1127,6 +947,263 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
 	return nil
 }
 
+// resetTranslationSync reinitializes streaming sync of translation data.
+func (s *holderSyncer) resetTranslationSync() error {
+	// Stop existing streams.
+	if err := s.stopTranslationSync(); err != nil {
+		return errors.Wrap(err, "stop translation sync")
+	}
+
+	// Set read-only flag for all translation stores.
+	s.setTranslateReadOnlyFlags()
+
+	// Connect to each node that has a primary for which we are a replica.
+	if err := s.initializeIndexTranslateReplication(); err != nil {
+		return errors.Wrap(err, "initialize index translate replication")
+	}
+
+	// Connect to coordinator to stream field data.
+	if err := s.initializeFieldTranslateReplication(); err != nil {
+		return errors.Wrap(err, "initialize field translate replication")
+	}
+	return nil
+}
+
+////////////////////////////////////////////////////////////
+
+// translationSyncer provides an interface allowing a function
+// to notify the server that an action has occurred which requires
+// the translation sync process to be reset. In general, this
+// includes anything which modifies schema (add/remove index, etc),
+// or anything that changes the cluster topology (add/remove node).
+// I originally considered leveraging the broadcaster since that was
+// already in place and provides similar event messages, but the
+// broadcaster is really meant for notifiying other nodes, while
+// this is more akin to an internal message bus. In fact, I think
+// a future iteration on this may be to make it more generic so
+// it can act as an internal message bus where one of the messages
+// being published is "translationSyncReset".
+type translationSyncer interface {
+	Reset() error
+}
+
+// NopTranslationSyncer represents a translationSyncer that doesn't do anything.
+var NopTranslationSyncer translationSyncer = &nopTranslationSyncer{}
+
+type nopTranslationSyncer struct{}
+
+// Reset is a no-op implementation of translationSyncer Reset method.
+func (nopTranslationSyncer) Reset() error { return nil }
+
+// activeTranslationSyncer represents a translationSyncer that resets
+// the server's translation syncer.
+type activeTranslationSyncer struct {
+	ch chan struct{}
+}
+
+// newActiveTranslationSyncer returns a new instance of activeTranslationSyncer.
+func newActiveTranslationSyncer(ch chan struct{}) *activeTranslationSyncer {
+	return &activeTranslationSyncer{
+		ch: ch,
+	}
+}
+
+// Reset resets the server's translation syncer.
+func (a *activeTranslationSyncer) Reset() error {
+	a.ch <- struct{}{}
+	return nil
+}
+
+////////////////////////////////////////////////////////////
+
+// stopTranslationSync closes and waits for all outstanding translation readers
+// to complete. This should be called before reconnecting to the cluster in case
+// of a cluster resize or schema change.
+func (s *holderSyncer) stopTranslationSync() error {
+	var g errgroup.Group
+	for i := range s.readers {
+		rd := s.readers[i]
+		g.Go(func() error {
+			return rd.Close()
+		})
+	}
+	return g.Wait()
+}
+
+// setTranslateReadOnlyFlags updates all translation stores to enable or disable
+// writing new translation keys. Index stores are writable if the node owns the
+// partition. Field stores are writable if the node is the coordinator.
+func (s *holderSyncer) setTranslateReadOnlyFlags() {
+	isCoordinator := s.Cluster.isCoordinator()
+
+	for _, index := range s.Holder.Indexes() {
+		// There is a race condition here:
+		// if Indexes() returns idx1, and then in another
+		// process, holder.DeleteIndex(idx1) is called,
+		// then the next step trying to get TranslateStore(partitionID)
+		// for an index that is closed (and therefore its transateStores
+		// no longer exist) will fail with a nil pointer error.
+		// For now, I just checked that the translateStore hasn't been
+		// set to nil before trying to use it, but another option may
+		// be to prevent the translateStores from being zeroed out
+		// while this process is active. Checking for nil as we do
+		// really obviates the need for the RLock around the for loop.
+
+		// Obtain a read lock on index to prevent Index.Close() from
+		// destroying the Index.translateStores map before this is
+		// done using it.
+		index.mu.RLock()
+		for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
+			ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID)
+			if ts := index.TranslateStore(partitionID); ts != nil {
+				ts.SetReadOnly(!ownsPartition)
+			}
+		}
+		index.mu.RUnlock()
+
+		for _, field := range index.Fields() {
+			field.TranslateStore().SetReadOnly(!isCoordinator)
+		}
+	}
+}
+
+// initializeIndexTranslateReplication connects to each node that is the
+// primary for a partition that we are a replica of.
+func (s *holderSyncer) initializeIndexTranslateReplication() error {
+	for _, node := range s.Cluster.Nodes() {
+		// Skip local node.
+		if node.ID == s.Node.ID {
+			continue
+		}
+
+		// Build a map of partition offsets to stream from.
+		m := make(TranslateOffsetMap)
+		for _, index := range s.Holder.Indexes() {
+			if !index.Keys() {
+				continue
+			}
+
+			for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
+				partitionNodes := s.Cluster.partitionNodes(partitionID)
+				isPrimary := partitionNodes[0].ID == node.ID                 // remote is primary?
+				isReplica := Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica?
+				if !isPrimary || !isReplica {
+					continue
+				}
+
+				store := index.TranslateStore(partitionID)
+				offset, err := store.MaxID()
+				if err != nil {
+					return errors.Wrapf(err, "cannot determine max id for %q", index.Name())
+				}
+				m.SetIndexPartitionOffset(index.Name(), partitionID, offset)
+			}
+		}
+
+		// Skip if no replication required.
+		if len(m) == 0 {
+			continue
+		}
+
+		// Connect to remote node and begin streaming.
+		rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
+		if err != nil {
+			return err
+		}
+		s.readers = append(s.readers, rd)
+
+		go func() { defer rd.Close(); s.readIndexTranslateReader(rd) }()
+	}
+
+	return nil
+}
+
+// initializeFieldTranslateReplication connects the coordinator to stream field data.
+func (s *holderSyncer) initializeFieldTranslateReplication() error {
+	// Skip if coordinator.
+	if s.Cluster.isCoordinator() {
+		return nil
+	}
+
+	// Build a map of partition offsets to stream from.
+	m := make(TranslateOffsetMap)
+	for _, index := range s.Holder.Indexes() {
+		for _, field := range index.Fields() {
+			store := field.TranslateStore()
+			offset, err := store.MaxID()
+			if err != nil {
+				return errors.Wrapf(err, "cannot determine max id for %q/%q", index.Name(), field.Name())
+			}
+			m.SetFieldOffset(index.Name(), field.Name(), offset)
+		}
+	}
+
+	// Skip if no replication required.
+	if len(m) == 0 {
+		return nil
+	}
+
+	// Connect to coordinator and begin streaming.
+	coordinator := s.Cluster.coordinatorNode()
+	rd, err := s.Holder.OpenTranslateReader(context.Background(), coordinator.URI.String(), m)
+	if err != nil {
+		return err
+	}
+	s.readers = append(s.readers, rd)
+
+	go func() { defer rd.Close(); s.readFieldTranslateReader(rd) }()
+
+	return nil
+}
+
+func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
+	for {
+		var entry TranslateEntry
+		if err := rd.ReadEntry(&entry); err != nil {
+			s.Holder.Logger.Printf("cannot read index translate entry: %s", err)
+			return
+		}
+
+		// Find appropriate store.
+		idx := s.Holder.Index(entry.Index)
+		if idx == nil {
+			s.Holder.Logger.Printf("index not found: %q", entry.Index)
+			return
+		}
+
+		// Apply replication to store.
+		store := idx.TranslateStore(s.Cluster.keyPartition(entry.Index, entry.Key))
+		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
+			s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key)
+			return
+		}
+	}
+}
+
+func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) {
+	for {
+		var entry TranslateEntry
+		if err := rd.ReadEntry(&entry); err != nil {
+			s.Holder.Logger.Printf("cannot read field translate entry: %s", err)
+			return
+		}
+
+		// Find appropriate store.
+		f := s.Holder.Field(entry.Index, entry.Field)
+		if f == nil {
+			s.Holder.Logger.Printf("field not found: %q/%q", entry.Index, entry.Field)
+			return
+		}
+
+		// Apply replication to store.
+		store := f.TranslateStore()
+		if err := store.ForceSet(entry.ID, entry.Key); err != nil {
+			s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key)
+			return
+		}
+	}
+}
+
 // holderCleaner removes fragments and data files that are no longer used.
 type holderCleaner struct {
 	Node *Node

holder_internal_test.go

@@ -39,7 +39,7 @@ func (h *tHolder) Close() error {
 // Note that the holder must be Closed first.
 func (h *tHolder) Reopen() error {
 	path, logger := h.Path, h.Holder.Logger
-	h.Holder = NewHolder()
+	h.Holder = NewHolder(DefaultPartitionN)
 	h.Holder.Path = path
 	h.Holder.Logger = logger
 	return h.Holder.Open()
@@ -51,7 +51,7 @@ func newHolder() *tHolder {
 		panic(err)
 	}
 
-	h := &tHolder{Holder: NewHolder()}
+	h := &tHolder{Holder: NewHolder(DefaultPartitionN)}
 	h.Path = path
 	return h
 }
@@ -302,7 +302,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
 
 // Ensure holder can reopen.
 func TestHolderCleaner_Reopen(t *testing.T) {
-	h := NewHolder()
+	h := NewHolder(DefaultPartitionN)
 	h.Path = "path"
 	err := h.Open()
 	if err != nil {

holder_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pkg/errors"
 )
 
 func TestHolder_Open(t *testing.T) {
@@ -197,7 +198,110 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	})
+	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		h := test.MustOpenHolder()
+		defer h.Close()
 
+		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+			t.Fatal(err)
+		} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
+			t.Fatal(err)
+		} else if _, err := field.SetBit(0, 0, nil); err != nil {
+			t.Fatal(err)
+		} else if err := h.Holder.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := h.Reopen(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ForeignIndex", func(t *testing.T) {
+		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else {
+				_, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("nonexistent"))
+				if err == nil {
+					t.Fatalf("expected error: %s", pilosa.ErrForeignIndexNotFound)
+				} else if errors.Cause(err) != pilosa.ErrForeignIndexNotFound {
+					t.Fatalf("expected error: %s, but got: %s", pilosa.ErrForeignIndexNotFound, err)
+				}
+			}
+		})
+
+		// Foreign index zzz is opened after foo/bar.
+		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("zzz")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Foreign index aaa is opened before foo/bar.
+		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
+				t.Fatal(err)
+			} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100), pilosa.OptFieldForeignIndex("aaa")); err != nil {
+				t.Fatal(err)
+			} else if err := h.Holder.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := h.Reopen(); err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		})
+
+		// Try to re-create existing index
+		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
+			h := test.MustOpenHolder()
+			defer h.Close()
+
+			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err = h.CreateIndex("aaa", pilosa.IndexOptions{}); err == nil {
+				t.Fatalf("expected: ConflictError, got: nil")
+			} else if _, ok := err.(pilosa.ConflictError); !ok {
+				t.Fatalf("expected: ConflictError, got: %s", err)
+			}
+
+			idx2, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if idx1 != idx2 {
+				t.Fatalf("expected the same indexes, got: %s and %s", idx1.Name(), idx2.Name())
+			}
+		})
+	})
 }
 
 func TestHolder_HasData(t *testing.T) {
@@ -419,3 +523,40 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 		}
 	}
 }
+
+// Ensure holder can sync integer views with a remote holder.
+func TestHolderSyncer_IntField(t *testing.T) {
+	c := test.MustNewCluster(t, 2)
+	c[0].Config.Cluster.ReplicaN = 2
+	c[0].Config.AntiEntropy.Interval = 0
+	c[1].Config.Cluster.ReplicaN = 2
+	c[1].Config.AntiEntropy.Interval = 0
+	err := c.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer c.Close()
+
+	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index i: %v", err)
+	}
+	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+	if err != nil {
+		t.Fatalf("creating field f: %v", err)
+	}
+
+	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
+	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+
+	// Set data on the local holder for node0.
+	hldr0.SetValue("i", "f", 1, 1)
+
+	// Set data on node1.
+	hldr1.SetValue("i", "f", 2, 2)
+
+	err = c[0].Server.SyncData()
+	if err != nil {
+		t.Fatalf("syncing node 0: %v", err)
+	}
+}

http/client.go

@@ -553,6 +553,35 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
 	return nil
 }
 
+// ImportValue2 is a simplified ImportValue method which just uses the
+// ImportValueRequest instead of splitting up ImportValue and
+// ImportValueK... it also supports importing float values. The idea
+// being that (assuming it works) this will become the default (and be
+// renamed) for 2.0, and we can deprecate the other methods.
+func (c *InternalClient) ImportValue2(ctx context.Context, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.NewImportValue")
+	defer span.Finish()
+
+	buf, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Errorf("marshal import request: %s", err)
+	}
+
+	// Retrieve a list of nodes that own the shard.
+	nodes, err := c.FragmentNodes(ctx, req.Index, req.Shard)
+	if err != nil {
+		return errors.Errorf("shard nodes: %s", err)
+	}
+
+	// Import to each node.
+	for _, node := range nodes {
+		if err := c.importNode(ctx, node, req.Index, req.Field, buf, options); err != nil {
+			return errors.Errorf("import node: host=%s, err=%s", node.URI, err)
+		}
+	}
+	return nil
+}
+
 // ImportValueK bulk imports keyed field values to a host.
 func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
@@ -667,6 +696,56 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
 	return nil
 }
 
+// ImportColumnAttrs does bulk import of column attrs
+func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
+	defer span.Finish()
+
+	if index == "" {
+		return pilosa.ErrIndexRequired
+	}
+	if uri == nil {
+		uri = c.defaultURI
+	}
+
+	url := fmt.Sprintf("%s/index/%s/import-column-attrs", uri, index)
+
+	// Marshal data to protobuf.
+	data, err := c.serializer.Marshal(req)
+	if err != nil {
+		return errors.Wrap(err, "marshal import-column-attrs request")
+	}
+
+	// Generate HTTP request.
+	httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+	httpReq.Header.Set("Content-Type", "application/x-protobuf")
+	httpReq.Header.Set("Accept", "application/x-protobuf")
+	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(httpReq.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	dec := json.NewDecoder(resp.Body)
+	rbody := &pilosa.ImportResponse{}
+	err = dec.Decode(rbody)
+	// Decode can return EOF when no error occurred. helpful!
+	if err != nil && err != io.EOF {
+		return errors.Wrap(err, "decoding response body")
+	}
+	if rbody.Err != "" {
+		return errors.Wrap(errors.New(rbody.Err), "importing roaring")
+	}
+
+	return nil
+}
+
 // ExportCSV bulk exports data for a single shard from a host to CSV format.
 func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
@@ -791,18 +870,28 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
 	}
 
 	// convert pilosa.FieldOptions to fieldOptions
+	//
+	// TODO this kind of sucks because it's one more place that needs
+	// changes when we change anything with field options (and there
+	// are a lot of places already). It's not clear to me that this is
+	// providing a lot of value, but I think this kind of validation
+	// should probably happen in the field anyway??
 	fieldOpt := fieldOptions{
 		Type: opt.Type,
 		Keys: &opt.Keys,
 	}
-	if fieldOpt.Type == "set" {
+	if fieldOpt.Type == pilosa.FieldTypeSet {
 		fieldOpt.CacheType = &opt.CacheType
 		fieldOpt.CacheSize = &opt.CacheSize
-	} else if fieldOpt.Type == "int" {
+	} else if fieldOpt.Type == pilosa.FieldTypeInt {
 		fieldOpt.Min = &opt.Min
 		fieldOpt.Max = &opt.Max
-	} else if fieldOpt.Type == "time" {
+	} else if fieldOpt.Type == pilosa.FieldTypeTime {
 		fieldOpt.TimeQuantum = &opt.TimeQuantum
+	} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
+		fieldOpt.Min = &opt.Min
+		fieldOpt.Max = &opt.Max
+		fieldOpt.Scale = &opt.Scale
 	}
 
 	// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
@@ -1032,7 +1121,109 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
 	if err != nil {
 		return errors.Wrap(err, "executing request")
 	}
-	return errors.Wrap(resp.Body.Close(), "closing response body")
+	defer resp.Body.Close()
+	_, err = io.Copy(ioutil.Discard, resp.Body)
+	return errors.Wrap(err, "draining SendMessage response body")
+}
+
+// TranslateKeysNode sends a key translation request to a specific node.
+func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string) ([]uint64, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{
+		Index: index,
+		Field: field,
+		Keys:  keys,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateKeysRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/keys")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateKeysResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.IDs, nil
+}
+
+// TranslateIDsNode sends an id translation request to a specific node.
+func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode")
+	defer span.Finish()
+
+	if index == "" {
+		return nil, pilosa.ErrIndexRequired
+	}
+
+	buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{
+		Index: index,
+		Field: field,
+		IDs:   ids,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "marshaling TranslateIDsRequest")
+	}
+
+	// Create HTTP request.
+	u := uri.Path("/internal/translate/ids")
+	req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Accept", "application/x-protobuf")
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+
+	// Execute request against the host.
+	resp, err := c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Read body and unmarshal response.
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading")
+	}
+
+	tkresp := &pilosa.TranslateIDsResponse{}
+	if err := c.serializer.Unmarshal(body, tkresp); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %s", err)
+	}
+	return tkresp.Keys, nil
 }
 
 // executeRequest executes the given request and checks the Response. For

http/client_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	gohttp "net/http"
 	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
@@ -147,7 +148,8 @@ func TestClient_MultiNode(t *testing.T) {
 	}
 
 	// Test must return exactly N results.
-	if len(result.Results[0].([]pilosa.Pair)) != topN {
+	pairsField := result.Results[0].(*pilosa.PairsField)
+	if len(pairsField.Pairs) != topN {
 		t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
 	}
 	p := []pilosa.Pair{
@@ -157,7 +159,7 @@ func TestClient_MultiNode(t *testing.T) {
 		{ID: 99, Count: 7}}
 
 	// Valdidate the Top 4 result counts.
-	if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
+	if !reflect.DeepEqual(pairsField.Pairs, p) {
 		t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
 	}
 
@@ -292,22 +294,29 @@ func TestClient_Export(t *testing.T) {
 		buf := bytes.NewBuffer(nil)
 		bw := bufio.NewWriter(buf)
 
-		// Send export request.
-		if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", 0, bw); err != nil {
-			t.Fatal(err)
+		// Send export request for every partition.
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil {
+				t.Fatal(err)
+			}
 		}
 
 		got := buf.String()
 
-		// Expected output.
-		exp := ""
-		for _, bit := range data {
-			exp += fmt.Sprintf("%d,%s\n", bit.RowID, bit.ColumnKey)
-		}
+		// Expected output is not sorted because of key sharding.
+		exp := "" +
+			"2,col200\n" +
+			"2,col201\n" +
+			"2,col202\n" +
+			"2,col203\n" +
+			"1,col103\n" +
+			"1,col102\n" +
+			"1,col101\n" +
+			"1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 
@@ -327,21 +336,28 @@ func TestClient_Export(t *testing.T) {
 		bw := bufio.NewWriter(buf)
 
 		// Send export request.
-		if err := c.ExportCSV(context.Background(), "keyed", "keyedf", 0, bw); err != nil {
-			t.Fatal(err)
+		for i := 0; i < pilosa.DefaultPartitionN; i++ {
+			if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil {
+				t.Fatal(err)
+			}
 		}
 
 		got := buf.String()
 
-		// Expected output.
-		exp := ""
-		for _, bit := range data {
-			exp += fmt.Sprintf("%s,%s\n", bit.RowKey, bit.ColumnKey)
-		}
+		// Expected output is unsorted because of key sharding.
+		exp := "" +
+			"row2,col200\n" +
+			"row2,col201\n" +
+			"row2,col202\n" +
+			"row2,col203\n" +
+			"row1,col103\n" +
+			"row1,col102\n" +
+			"row1,col101\n" +
+			"row1,col100\n"
 
 		// Verify data.
 		if got != exp {
-			t.Fatalf("unexpected export data: %s", got)
+			t.Fatalf("unexpected export data: %q, expected %q", got, exp)
 		}
 	})
 }
@@ -394,6 +410,60 @@ func TestClient_Import(t *testing.T) {
 	}
 }
 
+// Ensure client can bulk import column attrs.
+func TestClient_ImportColumnAttrs(t *testing.T) {
+	cluster := test.MustNewCluster(t, 2)
+	for _, c := range cluster {
+		c.Config.Cluster.ReplicaN = 2
+	}
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer cluster.Close()
+
+	ctx := context.Background()
+	_, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+	_, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+
+	attrKey := "k"
+	// Send import request.
+	host := cluster[0].URL()
+	c := MustNewClient(host, http.GetHTTPClient(nil))
+	colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
+	if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data.
+	pql := "Options(Row(f=0), columnAttrs=true)"
+	res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res.ColumnAttrSets) != 5 {
+		t.Fatal("incorrect number of column attrs set")
+	}
+
+	for _, v := range res.ColumnAttrSets {
+		attrVal := attrFun(v.ID)
+		if attrVal != v.Attrs[attrKey] {
+			t.Fatal(err)
+		}
+	}
+
+}
+
 // Ensure client can bulk import data.
 func TestClient_ImportRoaring(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
@@ -550,14 +620,14 @@ func TestClient_ImportKeys(t *testing.T) {
 				Index: "keyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -577,14 +647,14 @@ func TestClient_ImportKeys(t *testing.T) {
 				Index: "keyed",
 				Query: "TopN(unkeyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{ID: 1, Count: 3},
 				{ID: 2, Count: 2},
 				{ID: 3, Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -604,14 +674,14 @@ func TestClient_ImportKeys(t *testing.T) {
 				Index: "unkeyed",
 				Query: "TopN(keyedf)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 	})
@@ -649,14 +719,14 @@ func TestClient_ImportKeys(t *testing.T) {
 				Index: "keyed",
 				Query: "TopN(keyedf0)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %v", pairs)
+				t.Fatalf("unexpected topn result: %v", pairs.Pairs)
 			}
 		})
 
@@ -681,14 +751,14 @@ func TestClient_ImportKeys(t *testing.T) {
 				Index: "keyed",
 				Query: "TopN(keyedf1)",
 			})
-			if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok {
+			if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok {
 				t.Fatalf("unexpected response type %T", resp.Results[0])
-			} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+			} else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{
 				{Key: "green", Count: 3},
 				{Key: "blue", Count: 2},
 				{Key: "purple", Count: 1},
 			}) {
-				t.Fatalf("unexpected topn result: %#v", pairs)
+				t.Fatalf("unexpected topn result: %#v", pairs.Pairs)
 			}
 		})
 	})
@@ -777,6 +847,72 @@ func TestClient_ImportKeys(t *testing.T) {
 	})
 }
 
+func TestClient_ImportIDs(t *testing.T) {
+	// Ensure that running a query between two imports does
+	// not affect the result set. It turns out, this is caused
+	// by the fragment.rowCache failing to be cleared after an
+	// importValue. This ensures that the rowCache is cleared
+	// after an import.
+	t.Run("ImportRangeImport", func(t *testing.T) {
+		cluster := test.MustRunCluster(t, 1)
+		defer cluster.Close()
+		cmd := cluster[0]
+		host := cmd.URL()
+		holder := cmd.Server.Holder()
+		hldr := test.Holder{Holder: holder}
+
+		idxName := "i"
+		fldName := "f"
+
+		// Load bitmap into cache to ensure cache gets updated.
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{Keys: false})
+		_, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-10000, 10000))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Send import request.
+		c := MustNewClient(host, http.GetHTTPClient(nil))
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 2, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		queryRequest := &pilosa.QueryRequest{
+			Query:  fmt.Sprintf(`Row(%s>0)`, fldName),
+			Remote: false,
+		}
+
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+
+		// Send import request.
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 1000, Value: 1},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify range.
+		if result, err := c.Query(context.Background(), idxName, queryRequest); err != nil {
+			t.Fatal(err)
+		} else {
+			res := result.Results[0].(*pilosa.Row).Columns()
+			if !reflect.DeepEqual(res, []uint64{2, 1000}) {
+				t.Fatalf("unexpected column ids: %v", res)
+			}
+		}
+	})
+}
+
 // Ensure client can bulk import value data.
 func TestClient_ImportValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
@@ -998,6 +1134,114 @@ func TestClient_FragmentBlocks(t *testing.T) {
 	}
 }
 
+func TestClient_CreateDecimalField(t *testing.T) {
+	cluster := test.MustRunCluster(t, 1)
+	defer cluster.Close()
+	cmd := cluster[0]
+
+	c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
+
+	index := "cdf"
+	err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	field := "dfield"
+	err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeDecimal, Scale: 1, Min: -1000, Max: 1000})
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	fld, err := cmd.API.Field(context.Background(), index, field)
+	if err != nil {
+		t.Fatalf("getting field: %v", err)
+	}
+	if fld.Options().Scale != 1 {
+		t.Fatalf("expected Scale 1, got: %+v", fld.Options())
+	}
+
+	err = c.ImportValue2(context.Background(), &pilosa.ImportValueRequest{Index: index, Field: field, ColumnIDs: []uint64{1, 2, 3}, Shard: 0, FloatValues: []float64{1.1, 2.2, 3.3}}, &pilosa.ImportOptions{})
+	if err != nil {
+		t.Fatalf("importing float values: %v", err)
+	}
+
+	// Integer predicate.
+	resp, err := c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicate.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.1)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Integer predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1<dfield<3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	// Float predicates.
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<=dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(1.1<dfield<=3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield<3.3)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{1, 2}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+
+	resp, err = c.Query(context.Background(), index, &pilosa.QueryRequest{Index: index, Query: "Row(dfield>=2.2)"})
+	if err != nil {
+		t.Fatalf("querying: %v", err)
+	}
+	if !reflect.DeepEqual(resp.Results[0].(*pilosa.Row).Columns(), []uint64{2, 3}) {
+		t.Fatalf("unexpected results: %v", resp.Results[0].(*pilosa.Row).Columns())
+	}
+}
+
 // Client represents a test wrapper for pilosa.Client.
 type Client struct {
 	*http.InternalClient
@@ -1021,3 +1265,23 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring
 		},
 	}
 }
+
+func attrFun(id uint64) string {
+	return strconv.FormatInt(int64(id), 10)
+}
+
+func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pilosa.ImportColumnAttrsRequest {
+	colIDs := make([]uint64, 0, 5)
+	attrVals := make([]string, 0, 5)
+	for n := uint64(0); n < 5; n++ {
+		colIDs = append(colIDs, n)
+		attrVals = append(attrVals, attrFun(n))
+	}
+	return &pilosa.ImportColumnAttrsRequest{
+		Index:     index,
+		Shard:     shard,
+		AttrKey:   attrKey,
+		ColumnIDs: colIDs,
+		AttrVals:  attrVals,
+	}
+}

http/handler.go

@@ -187,7 +187,7 @@ func (h *Handler) populateValidators() {
 	h.validators["DeleteField"] = queryValidationSpecRequired()
 	h.validators["PostImport"] = queryValidationSpecRequired().Optional("clear", "ignoreKeyCheck")
 	h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote", "clear")
-	h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
+	h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns", "profile")
 	h.validators["GetInfo"] = queryValidationSpecRequired()
 	h.validators["RecalculateCaches"] = queryValidationSpecRequired()
 	h.validators["GetSchema"] = queryValidationSpecRequired()
@@ -282,10 +282,15 @@ func newRouter(handler *Handler) *mux.Router {
 	router.Handle("/metrics", promhttp.Handler())
 	router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
 	router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
+	router.HandleFunc("/index", handler.handlePostIndex).Methods("POST").Name("PostIndex")
+	router.HandleFunc("/index/", handler.handlePostIndex).Methods("POST").Name("PostIndex")
 	router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
 	router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
 	router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
 	//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
+	router.HandleFunc("/index/{index}/import-column-attrs", handler.handlePostImportColumnAttrs).Methods("POST").Name("PostImportColumnAttrs")
+	router.HandleFunc("/index/{index}/field", handler.handlePostField).Methods("POST").Name("PostField")
+	router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField")
 	router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
 	router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
 	router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
@@ -308,6 +313,7 @@ func newRouter(handler *Handler) *mux.Router {
 	router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
 	router.HandleFunc("/internal/translate/data", handler.handlePostTranslateData).Methods("POST").Name("PostTranslateData")
 	router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys")
+	router.HandleFunc("/internal/translate/ids", handler.handlePostTranslateIDs).Methods("POST").Name("PostTranslateIDs")
 	router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
 	router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE")
 	router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
@@ -682,7 +688,11 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
-	indexName := mux.Vars(r)["index"]
+	indexName, ok := mux.Vars(r)["index"]
+	if !ok {
+		http.Error(w, "index name is required", http.StatusBadRequest)
+		return
+	}
 
 	resp := successResponse{h: h}
 
@@ -750,8 +760,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
 		return
 	}
-	indexName := mux.Vars(r)["index"]
-	fieldName := mux.Vars(r)["field"]
+
+	indexName, ok := mux.Vars(r)["index"]
+	if !ok {
+		http.Error(w, "index name is required", http.StatusBadRequest)
+		return
+	}
+
+	fieldName, ok := mux.Vars(r)["field"]
+	if !ok {
+		http.Error(w, "field name is required", http.StatusBadRequest)
+		return
+	}
 
 	resp := successResponse{h: h}
 
@@ -776,7 +796,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 	switch req.Options.Type {
 	case pilosa.FieldTypeSet:
 		fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize))
-	case pilosa.FieldTypeInt:
+	case pilosa.FieldTypeInt, pilosa.FieldTypeDecimal:
 		if req.Options.Min == nil {
 			min := int64(math.MinInt64)
 			req.Options.Min = &min
@@ -785,7 +805,15 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 			max := int64(math.MaxInt64)
 			req.Options.Max = &max
 		}
-		fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
+		if req.Options.Type == pilosa.FieldTypeDecimal {
+			scale := int64(0)
+			if req.Options.Scale != nil {
+				scale = *req.Options.Scale
+			}
+			fos = append(fos, pilosa.OptFieldTypeDecimal(scale, *req.Options.Min, *req.Options.Max))
+		} else {
+			fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
+		}
 	case pilosa.FieldTypeTime:
 		fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView))
 	case pilosa.FieldTypeMutex:
@@ -798,6 +826,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
 			fos = append(fos, pilosa.OptFieldKeys())
 		}
 	}
+	if req.Options.ForeignIndex != nil {
+		fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex))
+	}
 
 	_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
 	if _, ok := err.(pilosa.BadRequestError); ok {
@@ -819,9 +850,11 @@ type fieldOptions struct {
 	CacheSize      *uint32             `json:"cacheSize,omitempty"`
 	Min            *int64              `json:"min,omitempty"`
 	Max            *int64              `json:"max,omitempty"`
+	Scale          *int64              `json:"scale,omitempty"`
 	TimeQuantum    *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
 	Keys           *bool               `json:"keys,omitempty"`
 	NoStandardView bool                `json:"noStandardView,omitempty"`
+	ForeignIndex   *string             `json:"foreignIndex,omitempty"`
 }
 
 func (o *fieldOptions) validate() error {
@@ -849,14 +882,18 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type set"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("set field cannot be a foreign key"))
 		}
-	case pilosa.FieldTypeInt:
+	case pilosa.FieldTypeInt, pilosa.FieldTypeDecimal:
 		if o.CacheType != nil {
 			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int"))
 		} else if o.CacheSize != nil {
 			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
+		} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
+			return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeTime:
 		if o.CacheType != nil {
@@ -869,6 +906,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type time"))
 		} else if o.TimeQuantum == nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("time field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeMutex:
 		if o.CacheType == nil {
@@ -883,6 +922,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex"))
 		} else if o.TimeQuantum != nil {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("mutex field cannot be a foreign key"))
 		}
 	case pilosa.FieldTypeBool:
 		if o.CacheType != nil {
@@ -897,6 +938,8 @@ func (o *fieldOptions) validate() error {
 			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
 		} else if o.Keys != nil {
 			return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key"))
 		}
 	default:
 		return errors.Errorf("invalid field type: %s", o.Type)
@@ -1021,9 +1064,20 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
 		return nil, errors.New("invalid shard argument")
 	}
 
+	// Optional profiling
+	profile := false
+	profileString := q.Get("profile")
+	if profileString != "" {
+		profile, err = strconv.ParseBool(q.Get("profile"))
+		if err != nil {
+			return nil, fmt.Errorf("invalid profile argument: '%s' (should be true/false)", profileString)
+		}
+	}
+
 	return &pilosa.QueryRequest{
 		Query:           query,
 		Shards:          shards,
+		Profile:         profile,
 		ColumnAttrs:     q.Get("columnAttrs") == "true",
 		ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
 		ExcludeColumns:  q.Get("excludeColumns") == "true",
@@ -1101,7 +1155,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Unmarshal request based on field type.
-	if field.Type() == pilosa.FieldTypeInt {
+	if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
 		// Field type: Int
 		// Marshal into request object.
 		req := &pilosa.ImportValueRequest{}
@@ -1601,6 +1655,50 @@ func GetHTTPClient(t *tls.Config) *http.Client {
 	return &http.Client{Transport: transport}
 }
 
+// handlePostImportColumnAttrs
+func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) {
+	// Verify that request is only communicating over protobufs.
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
+		return
+	} else if r.Header.Get("Accept") != "application/x-protobuf" {
+		http.Error(w, "Not acceptable", http.StatusNotAcceptable)
+		return
+	}
+
+	opts := []pilosa.ImportOption{}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	req := &pilosa.ImportColumnAttrsRequest{}
+	if err := h.api.Serializer.Unmarshal(body, req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Marshal response object.
+	buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
+	if e != nil {
+		http.Error(w, fmt.Sprintf("marshal import-column-attrs response"), http.StatusInternalServerError)
+		return
+	}
+
+	// Write response.
+	_, err = w.Write(buf)
+	if err != nil {
+		h.logger.Printf("writing import-column-attrs response: %v", err)
+	}
+}
+
 // handlPostRoaringImport
 func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
 	// Verify that request is only communicating over protobufs.
@@ -1665,7 +1763,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 	// Marshal response object.
 	buf, err := h.api.Serializer.Marshal(resp)
 	if err != nil {
-		http.Error(w, fmt.Sprintf("marshal import response: %v", err), http.StatusInternalServerError)
+		http.Error(w, fmt.Sprintf("marshal import-roaring response: %v", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -1687,11 +1785,35 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	buf, err := h.api.TranslateKeys(r.Body)
+	buf, err := h.api.TranslateKeys(r.Context(), r.Body)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError)
 	}
 
+	// Write response.
+	_, err = w.Write(buf)
+	if err != nil {
+		h.logger.Printf("writing translate keys response: %v", err)
+		return
+	}
+}
+
+func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) {
+	// Verify that request is only communicating over protobufs.
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
+		return
+	} else if r.Header.Get("Accept") != "application/x-protobuf" {
+		http.Error(w, "Not acceptable", http.StatusNotAcceptable)
+		return
+	}
+
+	buf, err := h.api.TranslateIDs(r.Context(), r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("translate ids: %v", err), http.StatusInternalServerError)
+		return
+	}
+
 	// Write response.
 	_, err = w.Write(buf)
 	if err != nil {

index.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
@@ -44,6 +45,9 @@ type Index struct {
 	trackExistence bool
 	existenceFld   *Field
 
+	// Partitions used by translation.
+	partitionN int
+
 	// Fields by name.
 	fields map[string]*Field
 
@@ -52,32 +56,36 @@ type Index struct {
 	// Column attribute storage and cache.
 	columnAttrs AttrStore
 
-	translateStore TranslateStore
-
 	broadcaster broadcaster
 	Stats       stats.StatsClient
 
 	logger        logger.Logger
-	snapshotQueue chan *fragment
+	snapshotQueue snapshotQueue
 
-	// Used for notifying holder when a field is added.
+	// Passed to field for foreign-index lookup.
 	holder *Holder
 
-	// Instantiates new translation stores for fields.
+	// Per-partition translation stores
+	translateStores map[int]TranslateStore
+
+	translationSyncer translationSyncer
+
+	// Instantiates new translation stores
 	OpenTranslateStore OpenTranslateStoreFunc
 }
 
 // NewIndex returns a new instance of Index.
-func NewIndex(path, name string) (*Index, error) {
+func NewIndex(path, name string, partitionN int) (*Index, error) {
 	err := validateName(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "validating name")
 	}
 
 	return &Index{
-		path:   path,
-		name:   name,
-		fields: make(map[string]*Field),
+		path:       path,
+		name:       name,
+		partitionN: partitionN,
+		fields:     make(map[string]*Field),
 
 		newAttrStore: newNopAttrStore,
 		columnAttrs:  nopStore,
@@ -87,6 +95,10 @@ func NewIndex(path, name string) (*Index, error) {
 		logger:         logger.NopLogger,
 		trackExistence: true,
 
+		translateStores: make(map[int]TranslateStore),
+
+		translationSyncer: NopTranslationSyncer,
+
 		OpenTranslateStore: OpenInMemTranslateStore,
 	}, nil
 }
@@ -97,15 +109,22 @@ func (i *Index) Name() string { return i.name }
 // Path returns the path the index was initialized with.
 func (i *Index) Path() string { return i.path }
 
+// TranslateStorePath returns the translation database path for a partition.
+func (i *Index) TranslateStorePath(partitionID int) string {
+	return filepath.Join(i.path, translateStoreDir, strconv.Itoa(partitionID))
+}
+
+// TranslateStore returns the translation store for a given partition.
+func (i *Index) TranslateStore(partitionID int) TranslateStore {
+	return i.translateStores[partitionID]
+}
+
 // Keys returns true if the index uses string keys.
 func (i *Index) Keys() bool { return i.keys }
 
 // ColumnAttrStore returns the storage for column attributes.
 func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs }
 
-// TranslateStore returns the underlying translation store for the index.
-func (i *Index) TranslateStore() TranslateStore { return i.translateStore }
-
 // Options returns all options for this index.
 func (i *Index) Options() IndexOptions {
 	i.mu.RLock()
@@ -149,9 +168,27 @@ func (i *Index) Open() (err error) {
 		return errors.Wrap(err, "opening attrstore")
 	}
 
-	// Instantiate & open translation store.
-	if i.translateStore, err = i.OpenTranslateStore(filepath.Join(i.path, "keys"), i.name, ""); err != nil {
-		return errors.Wrap(err, "opening translate store")
+	i.logger.Debugf("open translate store for index: %s", i.name)
+
+	var g errgroup.Group
+	var mu sync.Mutex
+	for partitionID := 0; partitionID < i.partitionN; partitionID++ {
+		partitionID := partitionID
+
+		g.Go(func() error {
+			store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.partitionN)
+			if err != nil {
+				return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
+			}
+
+			mu.Lock()
+			defer mu.Unlock()
+			i.translateStores[partitionID] = store
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return err
 	}
 
 	return nil
@@ -197,6 +234,10 @@ fileLoop:
 					return errors.Wrapf(ErrName, "'%s'", fi.Name())
 				}
 
+				// Pass holder through to the field for use in looking
+				// up a foreign index.
+				fld.holder = i.holder
+
 				if err := fld.Open(); err != nil {
 					return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
 				}
@@ -213,7 +254,7 @@ fileLoop:
 
 // openExistenceField gets or creates the existence field and associates it to the index.
 func (i *Index) openExistenceField() error {
-	f, err := i.createFieldIfNotExists(existenceFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
+	f, err := i.createFieldIfNotExists(existenceFieldName, &FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
 	if err != nil {
 		return errors.Wrap(err, "creating existence field")
 	}
@@ -271,6 +312,14 @@ func (i *Index) Close() error {
 	// Close the attribute store.
 	i.columnAttrs.Close()
 
+	// Close partitioned translation stores.
+	for _, store := range i.translateStores {
+		if err := store.Close(); err != nil {
+			return errors.Wrap(err, "closing translation store")
+		}
+	}
+	i.translateStores = make(map[int]TranslateStore)
+
 	// Close all fields.
 	for _, f := range i.fields {
 		if err := f.Close(); err != nil {
@@ -279,12 +328,6 @@ func (i *Index) Close() error {
 	}
 	i.fields = make(map[string]*Field)
 
-	if i.translateStore != nil {
-		if err := i.translateStore.Close(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -362,13 +405,10 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 		return nil, newConflictError(ErrFieldExists)
 	}
 
-	// Apply functional options.
-	fo := FieldOptions{}
-	for _, opt := range opts {
-		err := opt(&fo)
-		if err != nil {
-			return nil, errors.Wrap(err, "applying option")
-		}
+	// Apply and validate functional options.
+	fo, err := newFieldOptions(opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
 	}
 
 	return i.createField(name, fo)
@@ -389,19 +429,16 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field
 		return f, nil
 	}
 
-	// Apply functional options.
-	fo := FieldOptions{}
-	for _, opt := range opts {
-		err := opt(&fo)
-		if err != nil {
-			return nil, errors.Wrap(err, "applying option")
-		}
+	// Apply and validate functional options.
+	fo, err := newFieldOptions(opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "applying option")
 	}
 
 	return i.createField(name, fo)
 }
 
-func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createFieldIfNotExists(name string, opt *FieldOptions) (*Field, error) {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -413,7 +450,7 @@ func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, e
 	return i.createField(name, opt)
 }
 
-func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
+func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
 	if name == "" {
 		return nil, errors.New("field name required")
 	} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
@@ -426,17 +463,17 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 		return nil, errors.Wrap(err, "initializing")
 	}
 
+	// Pass holder through to the field for use in looking
+	// up a foreign index.
+	f.holder = i.holder
+
+	f.setOptions(opt)
+
 	// Open field.
 	if err := f.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 
-	// Apply field options.
-	if err := f.applyOptions(opt); err != nil {
-		f.Close()
-		return nil, errors.Wrap(err, "applying options")
-	}
-
 	if err := f.saveMeta(); err != nil {
 		f.Close()
 		return nil, errors.Wrap(err, "saving meta")
@@ -445,9 +482,9 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 	// Add to index's field lookup.
 	i.fields[name] = f
 
-	// Update replication, if needed.
-	if i.holder != nil {
-		go i.holder.refreshTranslateStoreReplicator()
+	// Kick off the field's translation sync process.
+	if err := i.translationSyncer.Reset(); err != nil {
+		return nil, errors.Wrap(err, "resetting translation syncer")
 	}
 
 	return f, nil
@@ -462,7 +499,9 @@ func (i *Index) newField(path, name string) (*Field, error) {
 	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
-	f.snapshotQueue = i.snapshotQueue
+	if i.snapshotQueue != nil {
+		f.snapshotQueue = i.snapshotQueue
+	}
 	f.OpenTranslateStore = i.OpenTranslateStore
 	return f, nil
 }
@@ -503,7 +542,7 @@ func (i *Index) DeleteField(name string) error {
 	// Remove reference.
 	delete(i.fields, name)
 
-	return nil
+	return i.translationSyncer.Reset()
 }
 
 type indexSlice []*Index

index_internal_test.go

@@ -25,7 +25,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, "i")
+	index, err := NewIndex(path, "i", DefaultPartitionN)
 	if err != nil {
 		panic(err)
 	}

index_test.go

@@ -185,6 +185,30 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 		*/
 	})
+
+	t.Run("WithKeys", func(t *testing.T) {
+		// Don't allow an int field to be created with keys=true
+		t.Run("IntField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrIntFieldWithKeys {
+				t.Fatal("int field cannot be created with keys=true")
+			}
+		})
+
+		// Don't allow a decimal field to be created with keys=true
+		t.Run("DecimalField", func(t *testing.T) {
+			index := test.MustOpenIndex()
+			defer index.Close()
+
+			_, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, -1, 1), pilosa.OptFieldKeys())
+			if errors.Cause(err) != pilosa.ErrDecimalFieldWithKeys {
+				t.Fatal("decimal field cannot be created with keys=true")
+			}
+		})
+	})
 }
 
 // Ensure index can delete a field.
@@ -217,7 +241,7 @@ func TestIndex_InvalidName(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(path, "ABC")
+	index, err := pilosa.NewIndex(path, "ABC", pilosa.DefaultPartitionN)
 	if err == nil {
 		t.Fatalf("should have gotten an error on index name with caps")
 	}

internal/clustertests/docker-compose.yml

@@ -51,6 +51,6 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command:
-      - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
+      - "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
 networks: 
   pilosanet:

internal/private.proto

@@ -18,6 +18,8 @@ message FieldOptions {
 	bool NoStandardView = 12;
 	int64 Base = 13;
 	uint64 BitDepth = 14;
+	int64 Scale = 15;
+	string ForeignIndex = 16;
 }
 
 message ImportResponse {

internal/public.proto

@@ -6,6 +6,12 @@ message Row {
 	repeated uint64 Columns = 1;
 	repeated string Keys = 3;
 	repeated Attr Attrs = 2;
+	bytes Roaring = 4;
+}
+
+message SignedRow {
+	Row Pos = 1;
+	Row Neg = 2;
 }
 
 message RowIdentifiers {
@@ -19,6 +25,16 @@ message Pair {
 	uint64 Count = 2;
 }
 
+message PairField {
+	Pair Pair = 1;
+	string Field = 2;
+}
+
+message PairsField {
+	repeated Pair Pairs = 1;
+	string Field = 2;
+}
+
 message FieldRow{
 	string Field = 1;
 	uint64 RowID = 2;
@@ -28,11 +44,13 @@ message FieldRow{
 message GroupCount{
 	repeated FieldRow Group = 1;
 	uint64 Count = 2;
+	int64 Sum = 3;
 }
 
 message ValCount {
 	int64 Val = 1;
 	int64 Count = 2;
+	double FloatVal = 3;
 }
 
 message ColumnAttrSet {
@@ -61,6 +79,7 @@ message QueryRequest {
 	bool Remote = 5;
 	bool ExcludeRowAttrs = 6;
 	bool ExcludeColumns = 7;
+	repeated Row EmbeddedData = 8;
 }
 
 message QueryResponse {
@@ -79,6 +98,8 @@ message QueryResult {
 	repeated uint64 RowIDs = 7;
 	repeated GroupCount GroupCounts = 8;
 	RowIdentifiers RowIdentifiers = 9;
+	SignedRow SignedRow = 10;
+	PairsField PairsField = 11;
 }
 
 message ImportRequest {
@@ -99,6 +120,8 @@ message ImportValueRequest {
 	repeated uint64 ColumnIDs = 5;
 	repeated string ColumnKeys = 7;
 	repeated int64 Values = 6;
+	repeated double FloatValues = 8;
+	repeated string StringValues = 9;
 }
 
 message TranslateKeysRequest {
@@ -111,6 +134,16 @@ message TranslateKeysResponse {
 	repeated uint64 IDs = 3;
 }
 
+message TranslateIDsRequest {
+	string Index = 1;
+	string Field = 2;
+	repeated uint64 IDs = 3;
+}
+
+message TranslateIDsResponse {
+	repeated string Keys = 3;
+}
+
 message ImportRoaringRequestView {
 	string Name = 1;
 	bytes Data = 2;
@@ -119,4 +152,12 @@ message ImportRoaringRequestView {
 message ImportRoaringRequest {
 	bool Clear = 1;
 	repeated ImportRoaringRequestView views = 2;
-}
+}
+
+message ImportColumnAttrsRequest {
+	string Index = 1;
+	int64 Shard = 2;
+	string AttrKey = 3;
+	repeated string AttrVals = 4;
+	repeated uint64 ColumnIDs = 5;
+}

license.exceptions

@@ -0,0 +1,10 @@
+# names of files which we do not expect to have our license header
+./apimethod_string.go
+./pql/pql.peg.go
+./internal/private.pb.go
+./internal/public.pb.go
+./lru/lru.go
+./enterprise/enterprise.go
+./roaring/btree.go
+./roaring/btree_test.go
+./proto/pilosa.pb.go

logger/logger.go

@@ -105,3 +105,27 @@ func (cl *CaptureLogger) Printf(format string, v ...interface{}) {
 func (cl *CaptureLogger) Debugf(format string, v ...interface{}) {
 	cl.Debugs = append(cl.Debugs, fmt.Sprintf(format, v...))
 }
+
+// Logfer is a thing that has only a Logf() method, like for instance,
+// testing.T or testing.B.
+type Logfer interface {
+	Logf(format string, v ...interface{})
+}
+
+// LogfLogger is a logger that wraps something that has a Logf interface
+// and makes it act like our logger.
+type LogfLogger struct {
+	wrapped Logfer
+}
+
+func (ll *LogfLogger) Printf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func (ll *LogfLogger) Debugf(format string, v ...interface{}) {
+	ll.wrapped.Logf(format, v...)
+}
+
+func NewLogfLogger(l Logfer) *LogfLogger {
+	return &LogfLogger{wrapped: l}
+}

mmap_test.go

@@ -0,0 +1,102 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/syswrap"
+)
+
+type cv struct {
+	cols []uint64
+	vals []int64
+}
+
+func forceSnapshotsCheckMapping(t *testing.T) {
+	depth := uint(6)
+	f := mustOpenBSIFragment("i", "f", viewStandard, 0)
+	f.Logger = logger.NewLogfLogger(t)
+	defer f.Clean(t)
+
+	for i := 0; i < f.MaxOpN; i++ {
+		_, _ = f.setBit(0, uint64(i*32))
+	}
+	// force snapshot so we get a mmapped row...
+	err := f.Snapshot()
+	if err != nil {
+		t.Fatalf("initial snapshot error: %v", err)
+	}
+
+	values := make([]cv, 1024)
+	for i := range values {
+		cols := make([]uint64, 128)
+		vals := make([]int64, 128)
+		for j := range cols {
+			// pick values in the first 16 cols of each of the 16
+			// shards in a default shardwidth, so each set will
+			// probably change some values from the previous one.
+			cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16))
+			vals[j] = int64(rand.Int63n(1 << depth))
+		}
+		values[i] = cv{cols, vals}
+	}
+
+	// modify the original bitmap, until it causes a snapshot, which
+	// then invalidates the other map...
+	for i := 0; i < 32; i++ {
+		cv := values[i%len(values)]
+		// periodically force gc, so if we have a small pool of maps
+		// we'll go in and out of mapping mode
+		if i%5 == 0 {
+			runtime.GC()
+		}
+		err := f.importValue(cv.cols, cv.vals, depth, (i%3 == 1))
+		if err != nil {
+			t.Fatalf("importValue[%d]: %v", i, err)
+		}
+		err = f.Snapshot()
+		if err != nil {
+			t.Fatalf("snapshot[%d]: %v", i, err)
+		}
+	}
+}
+
+// This test should basically never fail, but it might if you were running
+// out of available mmaps. Which you can fake up by adding '&& false' to the test
+// in newGeneration in generation.go. So this is probably useless but it's
+// a failure mode we've been bitten by once...
+func TestMmapBehavior(t *testing.T) {
+	var changed bool
+	var original uint64
+	defer func() {
+		syswrap.SetMaxMapCount(original)
+	}()
+
+	for _, mmapMaxVal := range []uint64{0, 3} {
+		prev := syswrap.SetMaxMapCount(mmapMaxVal)
+		if !changed {
+			original = prev
+			changed = true
+		}
+		t.Run(fmt.Sprintf("maps%d", mmapMaxVal), func(t *testing.T) {
+			forceSnapshotsCheckMapping(t)
+		})
+	}
+}

mock/translator.go

@@ -25,6 +25,7 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil)
 type TranslateStore struct {
 	CloseFunc         func() error
 	MaxIDFunc         func() (uint64, error)
+	PartitionIDFunc   func() int
 	ReadOnlyFunc      func() bool
 	SetReadOnlyFunc   func(v bool)
 	TranslateKeyFunc  func(key string) (uint64, error)
@@ -43,6 +44,10 @@ func (s *TranslateStore) MaxID() (uint64, error) {
 	return s.MaxIDFunc()
 }
 
+func (s *TranslateStore) PartitionID() int {
+	return s.PartitionIDFunc()
+}
+
 func (s *TranslateStore) ReadOnly() bool {
 	return s.ReadOnlyFunc()
 }

pilosa.go

@@ -29,6 +29,8 @@ var (
 	ErrIndexExists   = errors.New("index already exists")
 	ErrIndexNotFound = errors.New("index not found")
 
+	ErrForeignIndexNotFound = errors.New("foreign index not found")
+
 	// ErrFieldRequired is returned when no field is specified.
 	ErrFieldRequired = errors.New("field required")
 	ErrFieldExists   = errors.New("field already exists")
@@ -48,7 +50,7 @@ var (
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")
 
-	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters")
+	ErrName  = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters")
 	ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
 
 	// ErrFragmentNotFound is returned when a fragment does not exist.
@@ -69,6 +71,9 @@ var (
 	ErrNotImplemented            = errors.New("not implemented")
 	ErrFieldsArgumentRequired    = errors.New("fields argument required")
 	ErrExpectedFieldListArgument = errors.New("expected field list argument")
+
+	ErrIntFieldWithKeys     = errors.New("int field cannot be created with 'keys=true' option")
+	ErrDecimalFieldWithKeys = errors.New("decimal field cannot be created with 'keys=true' option")
 )
 
 // apiMethodNotAllowedError wraps an error value indicating that a particular
@@ -118,7 +123,7 @@ func newNotFoundError(err error) NotFoundError {
 }
 
 // Regular expression to validate index and field names.
-var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
+var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
 
 // ColumnAttrSet represents a set of attributes for a vertical column in an index.
 // Can have a set of attributes attached to it.

pilosa_internal_test.go

@@ -21,7 +21,7 @@ import (
 func TestValidateName(t *testing.T) {
 	names := []string{
 		"a", "ab", "ab1", "b-c", "d_e", "exists",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"longbutnottoolongaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa12345689012345689012345678901234567890",
 	}
 	for _, name := range names {
 		if validateName(name) != nil {
@@ -33,7 +33,7 @@ func TestValidateName(t *testing.T) {
 func TestValidateNameInvalid(t *testing.T) {
 	names := []string{
 		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
+		"long123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
 	}
 	for _, name := range names {
 		if validateName(name) == nil {

pql/ast.go

@@ -17,10 +17,13 @@ package pql
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/molecula/ext"
 )
 
 // Query represents a PQL query.
@@ -84,19 +87,26 @@ func (q *Query) endConditional() {
 	if len(q.conditional) != 5 {
 		panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional))
 	}
-	low, _ := strconv.ParseInt(q.conditional[0], 10, 64)
+	low := parseNum(q.conditional[0])
 	field := q.conditional[2]
-	high, _ := strconv.ParseInt(q.conditional[4], 10, 64)
+	high := parseNum(q.conditional[4])
 
-	if q.conditional[1] == "<" {
-		low++
-	}
-	if q.conditional[3] == "<" {
-		high--
+	var op Token
+	switch q.conditional[1] + q.conditional[3] {
+	case "<<":
+		op = BTWN_LT_LT
+	case "<=<":
+		op = BTWN_LTE_LT
+	case "<<=":
+		op = BTWN_LT_LTE
+	case "<=<=":
+		op = BETWEEN
+	default:
+		panic(fmt.Sprintf("impossible conditional ops: '%s' and '%s'", q.conditional[1], q.conditional[3]))
 	}
 
 	elem := q.lastCallStackElem()
-	elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
+	elem.call.Args[field] = &Condition{Op: op, Value: []interface{}{low, high}}
 
 	q.conditional = nil
 }
@@ -152,16 +162,7 @@ func (q *Query) addNumVal(val string) {
 	if elem == nil || elem.lastField == "" {
 		panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
 	}
-	var ival interface{}
-	var err error
-	if strings.Contains(val, ".") {
-		ival, err = strconv.ParseFloat(val, 64)
-	} else {
-		ival, err = strconv.ParseInt(val, 10, 64)
-	}
-	if err != nil {
-		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
-	}
+	ival := parseNum(val)
 	if elem.inList {
 		if elem.lastCond != ILLEGAL {
 			list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{})
@@ -259,11 +260,261 @@ type callStackElem struct {
 	inList    bool
 }
 
-// Call represents a function call in the AST.
+// Some call types may require special handling, which needs to occur
+// before distributing processing to individual shards.
+type CallType byte
+
+const (
+	// Normal calls can be executed per shard.
+	PrecallNone = CallType(iota)
+	// PreCallGlobal indicates a call which must be run globally *before*
+	// distributing the call to other shards. Example: A Distinct query,
+	// where every shard could potentially produce results for any shard,
+	// so you have to produce the results up front.
+	PrecallGlobal
+	// PreCallPerNode indicates a call which needs to be run per-shard
+	// in a way that lets it be done on each shard, but where it should
+	// be done prior to spawning per-shard goroutines. Example:
+	// A cross-index query, where each local shard may or may not need
+	// to get data from a remote node, but batches of shards can
+	// probably be gotten from the same remote node.
+	PrecallPerNode
+)
+
+// Call represents a function call in the AST. The Precomputed field
+// is used by the executor to handle non-standard call types; it does
+// these by actually executing them separately, then replacing them
+// in the call tree with a new call using the special precomputed
+// type, with the Precomputed field set to a map from shards to results.
 type Call struct {
-	Name     string
-	Args     map[string]interface{}
-	Children []*Call
+	Name        string
+	Args        map[string]interface{}
+	Children    []*Call
+	Type        CallType
+	Precomputed map[uint64]interface{}
+}
+
+// callInfo defines the arguments allowed for a particular PQL call, and
+// possibly things about its semantics. If allowUnknown is true, unfamiliar
+// non-reserved names are allowed on the assumption that they're field names.
+// Otherwise, only those names explicitly listed are allowed. Reserved args
+// (those with a leading underscore) are never allowed unless explicitly
+// present.
+//
+// The prototypes map maps from argument names to a value. If the value is
+// non-nil, the argument will be checked for type-matching. So, for instance,
+// `x: 10` would indicate that x must be an int.
+type callInfo struct {
+	allowUnknown bool
+	prototypes   map[string]interface{}
+	callType     CallType
+}
+
+// We want to be able to accept either a string or int64 for
+// field names. Special-case type:
+type stringOrInt64Type struct{}
+
+var stringOrInt64 stringOrInt64Type
+
+var allowUnderField = callInfo{
+	allowUnknown: true,
+	prototypes: map[string]interface{}{
+		"_field": "",
+	},
+}
+
+var allowField = callInfo{
+	allowUnknown: false,
+	prototypes: map[string]interface{}{
+		"field": "",
+	},
+}
+
+var callInfoByFunc = map[string]callInfo{
+	// the easy cases: things that take arbitrary inputs, because they're
+	// taking field=value cases
+	"Bitmap": {allowUnknown: true},
+	"Count":  {allowUnknown: true},
+	"Row":    {allowUnknown: true},
+	"Range":  {allowUnknown: true},
+
+	"Distinct": {allowUnknown: true},
+
+	// allow only "field=X" cases with string field names
+	"Max": allowField,
+	"Min": allowField,
+	"Sum": allowField,
+
+	// only take other calls, should never have "args"
+	"Difference": {allowUnknown: false},
+	"Intersect":  {allowUnknown: false},
+	"Not":        {allowUnknown: false},
+	"All": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"limit":  int64(0),
+			"offset": int64(0),
+		},
+	},
+	"ClearRow": {allowUnknown: true},
+	"Store":    {allowUnknown: true},
+	"MinRow":   allowField,
+	"MaxRow":   allowField,
+	"Rows": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"_field":   "",
+			"field":    "",
+			"limit":    int64(0),
+			"column":   nil,
+			"previous": nil,
+			"from":     nil,
+			"to":       nil,
+		},
+	},
+	"Shift": {allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"n": int64(0),
+		},
+	},
+	"Union": {allowUnknown: false},
+	"Xor":   {allowUnknown: false},
+
+	// things that take _field
+	"TopN": allowUnderField,
+	// special cases:
+	"Clear": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"GroupBy": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"filter":    nil,
+			"limit":     int64(0),
+			"previous":  nil,
+			"aggregate": nil,
+			"having":    nil,
+		},
+	},
+	"Options": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"excludeRowAttrs": true,
+			"excludeColumns":  true,
+			"columnAttrs":     true,
+			"shards":          nil,
+		},
+	},
+	"Set": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col":       stringOrInt64,
+			"_timestamp": "",
+		},
+	},
+	"Precomputed": {
+		allowUnknown: true,
+	},
+	"SetBit": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_col": stringOrInt64,
+		},
+	},
+	"SetRowAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_row":   stringOrInt64,
+		},
+	},
+	"SetColumnAttrs": {
+		allowUnknown: true,
+		prototypes: map[string]interface{}{
+			"_field": "",
+			"_col":   stringOrInt64,
+		},
+	},
+	"IncludesColumn": {
+		allowUnknown: false,
+		prototypes: map[string]interface{}{
+			"column": stringOrInt64,
+		},
+	},
+}
+
+// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
+// arg validation.
+func RegisterPluginFuncs(ops []ext.BitmapOp) {
+	for _, op := range ops {
+		// ignore overlap for now. This should change.
+		if _, ok := callInfoByFunc[op.Name]; ok {
+			continue
+		}
+		ci := callInfo{allowUnknown: true}
+		if len(op.Reserved) > 0 {
+			// mark these as valid/known reserved words
+			ci.prototypes = make(map[string]interface{})
+			for _, res := range op.Reserved {
+				ci.prototypes[res] = nil
+			}
+		}
+		t := op.Func.BitmapOpType()
+		if t.Precall == ext.OpPrecallGlobal {
+			ci.callType = PrecallGlobal
+		}
+		callInfoByFunc[op.Name] = ci
+	}
+}
+
+// CheckCallInfo tries to validate that arguments are correct and valid for the
+// given call. It does not guarantee checking all possible errors; for instance,
+// if an argument is a field name, CheckCallInfo can't validate that the field
+// exists. It also updates with information like whether the call is expected
+// to require precalling.
+func (c *Call) CheckCallInfo() error {
+	valid, ok := callInfoByFunc[c.Name]
+	if !ok {
+		return fmt.Errorf("no arg validation for '%s'", c.Name)
+	}
+	c.Type = valid.callType
+	for k, v := range c.Args {
+		acceptable, ok := valid.prototypes[k]
+		if !ok && !valid.allowUnknown {
+			return fmt.Errorf("'%s': unknown arg '%s'", c.String(), k)
+		}
+		if !ok && strings.HasPrefix(k, "_") {
+			return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
+		}
+		if acceptable == nil {
+			continue
+		}
+		// if the types are identical, that's fine
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(v) {
+			continue
+		}
+		if reflect.TypeOf(acceptable) == reflect.TypeOf(stringOrInt64) {
+			switch v.(type) {
+			case string, int64:
+				continue
+			default:
+				return fmt.Errorf("'%s': arg '%s' needed a string or integer value, got %T.",
+					c.String(), k, v)
+			}
+		}
+		return fmt.Errorf("'%s': arg '%s' wrong type (got %T, expected %T)",
+			c.String(), k, v, acceptable)
+	}
+	// call-specific checking
+	for _, child := range c.Children {
+		if err := child.CheckCallInfo(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // FieldArg determines which key-value pair contains the field and rowID,
@@ -283,13 +534,36 @@ func IsReservedArg(name string) bool {
 		return true
 	}
 	switch name {
-	case "from", "to":
+	case "from", "to", "index":
 		return true
 	default:
 		return false
 	}
 }
 
+// CallIndex handles guessing whether we've been asked to apply this to a
+// different index. An empty string means "no".
+func (c *Call) CallIndex() string {
+	if index, ok := c.Args["_index"]; ok {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	if index, ok := c.Args["index"]; ok && index != "" {
+		if index, ok := index.(string); ok {
+			return index
+		}
+	}
+	return ""
+}
+
+// Arg is for reading the value at key from call.Args.
+// If the key is not in Call.Args, the value of the returned bool will be false.
+func (c *Call) Arg(key string) (interface{}, bool) {
+	v, ok := c.Args[key]
+	return v, ok
+}
+
 // BoolArg is for reading the value at key from call.Args as a bool. If the
 // key is not in Call.Args, the value of the returned bool will be false, and
 // the error will be nil. The value is assumed to be a bool. An error is
@@ -477,6 +751,36 @@ func (c *Call) HasConditionArg() bool {
 	return false
 }
 
+// TranslateInfo returns the relevant translation fields.
+func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) {
+	switch c.Name {
+	case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow":
+		// Positional args in new PQL syntax require special handling here.
+		fieldName, _ = c.FieldArg()
+		return "_" + columnLabel, fieldName, fieldName
+	case "SetRowAttrs":
+		// Positional args in new PQL syntax require special handling here.
+		return "", "_" + rowLabel, c.ArgString("_field")
+	case "Rows":
+		return "column", "previous", c.ArgString("_field")
+	case "IncludesColumn":
+		return "column", "", ""
+	case "GroupBy":
+		return "", "", ""
+	default:
+		return "col", "row", c.ArgString("_field")
+	}
+}
+
+func (c *Call) ArgString(key string) string {
+	value, ok := c.Args[key]
+	if !ok {
+		return ""
+	}
+	s, _ := value.(string)
+	return s
+}
+
 // Condition represents an operation & value.
 // When used in an argument map it represents a binary expression.
 type Condition struct {
@@ -489,9 +793,39 @@ func (cond *Condition) String() string {
 	return fmt.Sprintf("%s %s", cond.Op.String(), formatValue(cond.Value))
 }
 
+// StringWithSubj returns the string representation of the condition
+// including the provided subject.
+func (cond *Condition) StringWithSubj(subj string) string {
+	switch cond.Op {
+	case EQ, NEQ, LT, LTE, GT, GTE:
+		return fmt.Sprintf("%s%s", subj, cond.String())
+	case BETWEEN, BTWN_LT_LTE, BTWN_LTE_LT, BTWN_LT_LT:
+		val, ok := cond.Int64SliceValue() // TODO: this should depend on subj type (int64 vs. uint64)
+		if !ok || len(val) < 2 {
+			return ""
+		}
+		if cond.Op == BETWEEN {
+			return fmt.Sprintf("%d<=%s<=%d", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LTE {
+			return fmt.Sprintf("%d<%s<=%d", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LTE_LT {
+			return fmt.Sprintf("%d<=%s<%d", val[0], subj, val[1])
+		} else if cond.Op == BTWN_LT_LT {
+			return fmt.Sprintf("%d<%s<%d", val[0], subj, val[1])
+		}
+	}
+	return ""
+}
+
 // IntSliceValue reads cond.Value as a slice of uint64.
 // If the value is a slice of uint64 it will convert
 // it to []int64. Otherwise, if it is not a []int64 it will return an error.
+//
+// TODO(2.0) this is now only referenced in a test and should probably
+// be removed. The functionality was replaced by getCondIntSlice in
+// pilosa/executor.go which needed to check for floating point values
+// and also have access to the Pilosa field to see if floating point
+// values were valid and how they needed to be scaled.
 func (cond *Condition) IntSliceValue() ([]int64, error) {
 	val := cond.Value
 
@@ -514,6 +848,79 @@ func (cond *Condition) IntSliceValue() ([]int64, error) {
 	}
 }
 
+func (cond *Condition) Uint64Value() (uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		if tval >= 0 {
+			return uint64(tval), true
+		}
+	case uint64:
+		return tval, true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Uint64SliceValue() ([]uint64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]uint64, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = uint64(tv)
+			case uint64:
+				ret[i] = tv
+			default:
+				return nil, false
+			}
+		}
+		return ret, true
+	}
+
+	return nil, false
+}
+
+func (cond *Condition) Int64Value() (int64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case int64:
+		return tval, true
+	case uint64:
+		// TODO: consider overflow?
+		return int64(tval), true
+	}
+
+	return 0, false
+}
+
+func (cond *Condition) Int64SliceValue() ([]int64, bool) {
+	val := cond.Value
+
+	switch tval := val.(type) {
+	case []interface{}:
+		ret := make([]int64, len(tval))
+		for i, v := range tval {
+			switch tv := v.(type) {
+			case int64:
+				ret[i] = tv
+			case uint64:
+				ret[i] = int64(tv)
+			default:
+				return nil, false
+			}
+		}
+		return ret, true
+	}
+
+	return nil, false
+}
+
 func formatValue(v interface{}) string {
 	switch v := v.(type) {
 	case string:
@@ -560,3 +967,17 @@ func joinUint64Slice(a []uint64) string {
 	}
 	return "[" + strings.Join(other, ",") + "]"
 }
+
+func parseNum(val string) interface{} {
+	var ival interface{}
+	var err error
+	if strings.Contains(val, ".") {
+		ival, err = strconv.ParseFloat(val, 64)
+	} else {
+		ival, err = strconv.ParseInt(val, 10, 64)
+	}
+	if err != nil {
+		panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err))
+	}
+	return ival
+}

pql/parser.go

@@ -82,6 +82,14 @@ func (p *parser) Parse() (*Query, error) {
 			panic(v)
 		}
 	}
+	for _, call := range p.Query.Calls {
+		if call == nil {
+			return nil, fmt.Errorf("unexpected nil Call in query's call list")
+		}
+		if err := call.CheckCallInfo(); err != nil {
+			return nil, err
+		}
+	}
 
 	return &p.Query, nil
 }

pql/parser_test.go

@@ -75,12 +75,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with only arguments.
 	t.Run("ArgumentsOnly", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
+		q, err := pql.ParseString(`Row( key= value, foo='bar', age = 12 , bool0=true, bool1=false, x=null, escape="\" \\escape\n\\\\"  )`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key":    "value",
 					"foo":    "bar",
@@ -98,12 +98,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithFloatArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
+		q, err := pql.ParseString(`Row( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": 12.25,
 					"foo": 13.167,
@@ -118,12 +118,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with float arguments.
 	t.Run("WithNegativeArgs", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
+		q, err := pql.ParseString(`Row( key=-12.25, foo= -13)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": -12.25,
 					"foo": int64(-13),
@@ -173,12 +173,12 @@ func TestParser_Parse(t *testing.T) {
 
 	// Parse with condition arguments.
 	t.Run("WithCondition", func(t *testing.T) {
-		q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
+		q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
 		if err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(q.Calls[0],
 			&pql.Call{
-				Name: "MyCall",
+				Name: "Row",
 				Args: map[string]interface{}{
 					"key": "foo",
 					"x":   &pql.Condition{Op: pql.EQ, Value: 12.25},

pql/pql.peg

@@ -32,7 +32,7 @@ COND <- ( '><' { p.addBTWN() }
         )
 
 conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
-condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
+condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])}
 condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
 condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
 
@@ -55,7 +55,7 @@ item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
 doublequotedstring <- ( '\\"' / '\\\\' / [^"] )*
 singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )*
 
-fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' / '-' )*
+fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
 field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
 reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
 posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }

pql/pqlpeg_test.go

@@ -46,7 +46,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
 		t.Fatalf("Failed, got: %s", q)
 	}
 
-	_, err = ParseString("C(a=falsen0)")
+	_, err = ParseString("Row(a=falsen0)")
 	if err != nil {
 		t.Fatalf("falsen0 should have been parsed as a string")
 	}
@@ -109,15 +109,15 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 2},
 		{
 			name:   "SetWithArbCall",
-			input:  "Set(1, a=4)Blerg(z=ha)",
+			input:  "Set(1, a=4)Row(z=ha)",
 			ncalls: 2},
 		{
 			name:   "SetArbSet",
-			input:  "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
+			input:  "Set(1, a=4)Row(z=ha)Set(2, z=99)",
 			ncalls: 3},
 		{
 			name:   "ArbSetArb",
-			input:  "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
+			input:  "Row(q=1, a=4)Set(1, z=9)Row(z=99)",
 			ncalls: 3},
 		{
 			name:   "SetStringArg",
@@ -161,11 +161,11 @@ func TestPEGWorking(t *testing.T) {
 			ncalls: 1},
 		{
 			name:   "double quoted args",
-			input:  `B(a="zm''e")`,
+			input:  `Row(a="zm''e")`,
 			ncalls: 1},
 		{
 			name:   "single quoted args",
-			input:  `B(a='zm""e')`,
+			input:  `Row(a='zm""e')`,
 			ncalls: 1},
 		{
 			name:   "SetRowAttrs",
@@ -320,7 +320,7 @@ func TestPEGErrors(t *testing.T) {
 			input: "Set(, 1, a=4)"},
 		{
 			name:  "StartinCommaArb",
-			input: "Zeeb(, a=4)"},
+			input: "Row(, a=4)"},
 		{
 			name:  "SetRowAttrs0args",
 			input: "SetRowAttrs(blah, 9)"},
@@ -533,8 +533,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(4), int64(8)},
+						Op:    BTWN_LTE_LT,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -545,8 +545,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(8)},
+						Op:    BTWN_LT_LT,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -569,8 +569,8 @@ func TestPQLDeepEquality(t *testing.T) {
 				Name: "Row",
 				Args: map[string]interface{}{
 					"a": &Condition{
-						Op:    BETWEEN,
-						Value: []interface{}{int64(5), int64(9)},
+						Op:    BTWN_LT_LTE,
+						Value: []interface{}{int64(4), int64(9)},
 					},
 				},
 			}},
@@ -585,11 +585,11 @@ func TestPQLDeepEquality(t *testing.T) {
 			}},
 		{
 			name: "Weird dash",
-			call: "Sum(field-=f)",
+			call: "Count(dashy-=f)",
 			exp: &Call{
-				Name: "Sum",
+				Name: "Count",
 				Args: map[string]interface{}{
-					"field-": "f",
+					"dashy-": "f",
 				},
 			}},
 		{
@@ -672,8 +672,8 @@ func TestPQLDeepEquality(t *testing.T) {
 						Name: "Row",
 						Args: map[string]interface{}{
 							"a": &Condition{
-								Op:    BETWEEN,
-								Value: []interface{}{int64(5), int64(8)},
+								Op:    BTWN_LT_LT,
+								Value: []interface{}{int64(4), int64(9)},
 							},
 						},
 					},

pql/token.go

@@ -21,14 +21,24 @@ const (
 	// Special tokens
 	ILLEGAL Token = iota
 
-	ASSIGN  // =
-	EQ      // ==
-	NEQ     // !=
-	LT      // <
-	LTE     // <=
-	GT      // >
-	GTE     // >=
-	BETWEEN // ><
+	ASSIGN // =
+	EQ     // ==
+	NEQ    // !=
+	LT     // <
+	LTE    // <=
+	GT     // >
+	GTE    // >=
+
+	BETWEEN // ><  (this is like a <= x <= b)
+
+	// not used in lexing/parsing, but so that the parser can signal
+	// to the executor how to treat the arguments. We used to just add
+	// 1 to the arguments if they were LT so the executor could assume
+	// it was always <=, <=, but then we needed to support
+	// floats/decimals and couldn't do that any more.
+	BTWN_LT_LTE // a < x <= b
+	BTWN_LTE_LT // a <= x < b
+	BTWN_LT_LT  // a < x < b
 )
 
 var tokens = [...]string{

proto/interface.go

@@ -0,0 +1,267 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// StreamClient is an interface for a stream
+// which can return a RowResponse sent to a
+// stream via Send().
+type StreamClient interface {
+	Recv() (*RowResponse, error)
+}
+
+// StreamServer is an interface for a stream
+// which can accept a RowResponse to be later
+// returned by the stream via Recv().
+type StreamServer interface {
+	Send(*RowResponse) error
+}
+
+// EOF acts as an io.EOF encoded into a RowResponse.
+var EOF *RowResponse = &RowResponse{
+	StatusError: &StatusError{
+		Code:    0,
+		Message: "EOF",
+	},
+}
+
+// Error is a helper function to create a RowResponse
+// based on an error message. If the error is a grpc
+// Status, then the status code is passed through.
+func Error(err error) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrap prepends a message to the existing status
+// error message.
+func ErrorWrap(err error, message string) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: message + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorWrapf prepends a message to the existing status
+// error message with the format specifier.
+func ErrorWrapf(err error, format string, args ...interface{}) *RowResponse {
+	status, _ := status.FromError(err)
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(status.Code()),
+			Message: fmt.Sprintf(format, args...) + ": " + status.Err().Error(),
+		},
+	}
+}
+
+// ErrorCode is a helper function to create a RowResponse
+// based on a grpc status code and an error message.
+func ErrorCode(err error, c codes.Code) *RowResponse {
+	return &RowResponse{
+		StatusError: &StatusError{
+			Code:    uint32(c),
+			Message: err.Error(),
+		},
+	}
+}
+
+// RowResponseSorter implements the sort interface for a
+// provided []RowResponse based on the column index, type,
+// and sort direction.
+type RowResponseSorter struct {
+	colIdx        []int
+	colDescending []bool
+	colType       []string
+
+	rrs []*RowResponse
+}
+
+// NewRowResponseSorter return a new RowResponseSorter. It
+// does input validation and returns an error if the inputs
+// aren't compatible.
+func NewRowResponseSorter(idxs []int, dirs []bool, typs []string, rrs []*RowResponse) (*RowResponseSorter, error) {
+	// Ensure the input slices are non-empty and equal size.
+	if len(idxs) == 0 {
+		return nil, errors.New("index list cannot be empty")
+	}
+	if len(dirs) != len(idxs) || len(typs) != len(idxs) {
+		return nil, errors.New("index, direction, and type lists must be the same size")
+	}
+
+	// Ensure the provided data types are supported by the sorter.
+	for i := range typs {
+		switch typs[i] {
+		case "[]uint64", "[]string", "bool", "float64", "int64", "string", "uint64":
+			// pass
+		default:
+			return nil, fmt.Errorf("unsupported data type: %s", typs[i])
+		}
+	}
+
+	// Ensure max(colIdx) is within size of rr.Columns.
+	if len(rrs) > 0 {
+		var maxColIdx int
+		for i := range idxs {
+			if idxs[i] > maxColIdx {
+				maxColIdx = idxs[i]
+			}
+		}
+		if maxColIdx >= len(rrs[0].Columns) {
+			return nil, fmt.Errorf("column index is out of range: %d", maxColIdx)
+		}
+	}
+
+	return &RowResponseSorter{
+		colIdx:        idxs,
+		colDescending: dirs,
+		colType:       typs,
+		rrs:           rrs,
+	}, nil
+
+}
+
+func (r RowResponseSorter) Len() int      { return len(r.rrs) }
+func (r RowResponseSorter) Swap(i, j int) { r.rrs[i], r.rrs[j] = r.rrs[j], r.rrs[i] }
+func (r RowResponseSorter) Less(i, j int) bool {
+	ri := r.rrs[i]
+	rj := r.rrs[j]
+
+	for i, idx := range r.colIdx {
+		coli := ri.Columns[idx]
+		colj := rj.Columns[idx]
+		var comp int
+		switch r.colType[i] {
+		case "[]uint64":
+			ai := coli.GetUint64ArrayVal().Vals
+			aj := colj.GetUint64ArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					piv := ai[ii]
+					pjv := aj[ii]
+					if piv == pjv {
+						continue
+					} else if piv < pjv {
+						return -1
+					} else {
+						return 1
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "[]string":
+			ai := coli.GetStringArrayVal().Vals
+			aj := colj.GetStringArrayVal().Vals
+			comp = func() int {
+				for ii := 0; ii < len(ai); ii++ {
+					if len(aj) == ii {
+						return 1
+					}
+					sComp := strings.Compare(ai[ii], aj[ii])
+					if sComp == 0 {
+						continue
+					} else {
+						return sComp
+					}
+				}
+				if len(aj) > len(ai) {
+					return -1
+				}
+				return 0
+			}()
+		case "bool":
+			bi := coli.GetBoolVal()
+			bj := colj.GetBoolVal()
+			if bi == bj {
+				comp = 0
+			} else if !bi && bj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "float64":
+			fi := coli.GetFloat64Val()
+			fj := colj.GetFloat64Val()
+			if fi == fj {
+				comp = 0
+			} else if fi < fj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "int64":
+			ni := coli.GetInt64Val()
+			nj := colj.GetInt64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		case "string":
+			comp = strings.Compare(coli.GetStringVal(), colj.GetStringVal())
+		case "uint64":
+			ni := coli.GetUint64Val()
+			nj := colj.GetUint64Val()
+			if ni == nj {
+				comp = 0
+			} else if ni < nj {
+				comp = -1
+			} else {
+				comp = 1
+			}
+		}
+
+		isDescending := r.colDescending[i]
+
+		switch comp {
+		case 0:
+			continue
+		case -1:
+			if isDescending {
+				return false
+			}
+			return true
+		case 1:
+			if isDescending {
+				return true
+			}
+			return false
+		}
+	}
+	return false
+}

proto/pilosa.proto

@@ -0,0 +1,64 @@
+syntax = "proto3";
+package pilosa;
+
+message QueryPQLRequest {
+  string index = 1;
+  string pql = 2;
+}
+
+message StatusError{
+  uint32 Code = 1;
+  string Message = 2;
+}
+
+message RowResponse{
+  repeated ColumnInfo headers = 1;
+  repeated ColumnResponse columns = 2;
+  StatusError StatusError = 3;
+}
+
+message ColumnInfo {
+  string name = 1;
+  string datatype = 2;
+}
+
+message ColumnResponse{
+  oneof columnVal {
+    string stringVal = 1;
+    uint64 uint64Val = 2;
+    int64 int64Val = 3;
+    bool boolVal = 4;
+    bytes blobVal = 5;
+    Uint64Array uint64ArrayVal = 6;
+    StringArray stringArrayVal = 7;
+    double float64Val = 8;
+  }
+}
+
+message InspectRequest {
+  string index = 1;
+  IdsOrKeys columns = 2;
+  repeated string filterFields = 3;
+  uint64 limit = 4;
+  uint64 offset = 5;
+}
+
+message Uint64Array {
+  repeated uint64 vals = 1;
+}
+
+message StringArray {
+  repeated string vals = 1;
+}
+
+message IdsOrKeys {
+  oneof type {
+    Uint64Array ids = 1;
+    StringArray keys = 2;
+  }
+}
+
+service Pilosa {
+  rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {};
+  rpc Inspect(InspectRequest) returns (stream RowResponse) {};
+}

roaring/container_stash.go

@@ -267,9 +267,6 @@ func (c *Container) Freeze() *Container {
 	if c.flags&flagFrozen != 0 {
 		return c
 	}
-	// unmapOrClone should unmap-in-place because the existing
-	// container isn't frozen (or we'd already have returned it).
-	c = c.unmapOrClone()
 	c.flags |= flagFrozen
 	return c
 }
@@ -419,6 +416,48 @@ func (c *Container) bitmap() []uint64 {
 	return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)}))
 }
 
+// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
+// is provided. The target should be zeroed, or this becomes an implicit
+// union.
+func (c *Container) AsBitmap(target []uint64) (out []uint64) {
+	if c.typeID == containerBitmap {
+		return c.bitmap()
+	}
+	// Reminder: len(nil) == 0.
+	if len(target) < 1024 {
+		out = make([]uint64, 1024)
+	} else {
+		out = target
+		for i := range out {
+			out[i] = 0
+		}
+	}
+	if c.typeID == containerArray {
+		a := c.array()
+		for _, v := range a {
+			out[v/64] |= 1 << (v % 64)
+		}
+		return out
+	}
+	if c.typeID == containerRun {
+		runs := c.runs()
+		for _, r := range runs {
+			splatRun(out, r)
+		}
+		return out
+	}
+	// in theory this shouldn't happen?
+	return out
+}
+
+func splatRun(into []uint64, from interval16) {
+	// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
+	//note v must be int or will overflow
+	for v := int(from.start); v <= int(from.last); v++ {
+		into[v/64] |= (uint64(1) << uint(v%64))
+	}
+}
+
 // setBitmap stores a set of uint64s as data.
 func (c *Container) setBitmap(bitmap []uint64) {
 	if c == nil || c.frozen() {

roaring/containers_slice.go

@@ -212,7 +212,7 @@ func (sc *sliceContainers) Update(key uint64, fn func(*Container, bool) (*Contai
 		// don't expand the slice just to add a nil container, we
 		// could return that anyway
 		if write && nc != nil {
-			sc.insertAt(key, nc, -i-1)
+			sc.insertAt(key, nc, i)
 		}
 	}
 }

roaring/generation_debug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build generationdebug
+
+package roaring
+
+const generationDebug = true

roaring/generation_nodebug.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !generationdebug
+
+package roaring
+
+const generationDebug = false

roaring/roaring_internal_test.go

@@ -2710,6 +2710,15 @@ func unionInPlaceWrapper(a, b *Container) *Container {
 	return out.Containers.Get(0)
 }
 
+func differenceInPlaceWrapper(a, b *Container) *Container {
+	out := NewBitmap()
+	out.Containers.Put(0, a.Clone())
+	B := NewBitmap()
+	B.Containers.Put(0, b)
+	out.DifferenceInPlace(B)
+	return out.Containers.Get(0)
+}
+
 func TestContainerCombinations(t *testing.T) {
 
 	cts := setupContainerTests()
@@ -3283,6 +3292,117 @@ func TestContainerCombinations(t *testing.T) {
 		{flip, "outerBitsSet", "", "innerBitsSet"},
 		{flip, "oddBitsSet", "", "evenBitsSet"},
 		{flip, "evenBitsSet", "", "oddBitsSet"},
+
+		// differenceInPlace
+		{differenceInPlaceWrapper, "empty", "empty", "empty"},
+		{differenceInPlaceWrapper, "empty", "full", "empty"},
+		{differenceInPlaceWrapper, "empty", "firstBitSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "lastBitSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "empty", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "empty", "innerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "empty", "evenBitsSet", "empty"},
+		//
+		{differenceInPlaceWrapper, "full", "empty", "full"},
+		{differenceInPlaceWrapper, "full", "full", "empty"},
+		{differenceInPlaceWrapper, "full", "firstBitSet", "firstBitUnset"},
+		{differenceInPlaceWrapper, "full", "lastBitSet", "lastBitUnset"},
+		{differenceInPlaceWrapper, "full", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "full", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "full", "innerBitsSet", "outerBitsSet"},
+		{differenceInPlaceWrapper, "full", "outerBitsSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "full", "oddBitsSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "full", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "firstBitSet", "empty", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "full", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "firstBitSet", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "lastBitSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "innerBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "firstBitSet", "oddBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "firstBitSet", "evenBitsSet", "empty"},
+		//
+		{differenceInPlaceWrapper, "lastBitSet", "empty", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "full", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "firstBitSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "lastBitSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "innerBitsSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "lastBitSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "lastBitSet", "evenBitsSet", "lastBitSet"},
+		//
+		{differenceInPlaceWrapper, "firstBitUnset", "empty", "firstBitUnset"},
+		{differenceInPlaceWrapper, "firstBitUnset", "full", "empty"},
+		{differenceInPlaceWrapper, "firstBitUnset", "firstBitSet", "firstBitUnset"},
+		{differenceInPlaceWrapper, "firstBitUnset", "lastBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "firstBitUnset", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "innerBitsSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "firstBitUnset", "outerBitsSet", "innerBitsSet"},
+		//{differenceInPlaceWrapper, "firstBitUnset", "oddBitsSet", ""},
+		{differenceInPlaceWrapper, "firstBitUnset", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "lastBitUnset", "empty", "lastBitUnset"},
+		{differenceInPlaceWrapper, "lastBitUnset", "full", "empty"},
+		{differenceInPlaceWrapper, "lastBitUnset", "firstBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "lastBitSet", "lastBitUnset"},
+		{differenceInPlaceWrapper, "lastBitUnset", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "lastBitUnset", "innerBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "outerBitsSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "lastBitUnset", "oddBitsSet", "evenBitsSet"},
+		//{differenceInPlaceWrapper, "lastBitUnset", "evenBitsSet", ""},
+		//
+		{differenceInPlaceWrapper, "innerBitsSet", "empty", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "firstBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "lastBitSet", "innerBitsSet"},
+		{differenceInPlaceWrapper, "innerBitsSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "innerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "innerBitsSet", "outerBitsSet", "innerBitsSet"},
+		//{differenceInPlaceWrapper, "innerBitsSet", "oddBitsSet", ""},
+		//{differenceInPlaceWrapper, "innerBitsSet", "evenBitsSet", ""},
+		//
+		{differenceInPlaceWrapper, "outerBitsSet", "empty", "outerBitsSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "outerBitsSet", "firstBitSet", "lastBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "lastBitSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "innerBitsSet", "outerBitsSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "outerBitsSet", "empty"},
+		{differenceInPlaceWrapper, "outerBitsSet", "oddBitsSet", "firstBitSet"},
+		{differenceInPlaceWrapper, "outerBitsSet", "evenBitsSet", "lastBitSet"},
+		//
+		{differenceInPlaceWrapper, "oddBitsSet", "empty", "oddBitsSet"},
+		{differenceInPlaceWrapper, "oddBitsSet", "full", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "firstBitSet", "oddBitsSet"},
+		//{differenceInPlaceWrapper, "oddBitsSet", "lastBitSet", ""},
+		{differenceInPlaceWrapper, "oddBitsSet", "firstBitUnset", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "lastBitUnset", "lastBitSet"},
+		{differenceInPlaceWrapper, "oddBitsSet", "innerBitsSet", "lastBitSet"},
+		//{differenceInPlaceWrapper, "oddBitsSet", "outerBitsSet", ""},
+		{differenceInPlaceWrapper, "oddBitsSet", "oddBitsSet", "empty"},
+		{differenceInPlaceWrapper, "oddBitsSet", "evenBitsSet", "oddBitsSet"},
+		//
+		{differenceInPlaceWrapper, "evenBitsSet", "empty", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "full", "empty"},
+		//{differenceInPlaceWrapper, "evenBitsSet", "firstBitSet", ""},
+		{differenceInPlaceWrapper, "evenBitsSet", "lastBitSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "firstBitUnset", "firstBitSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "lastBitUnset", "empty"},
+		{differenceInPlaceWrapper, "evenBitsSet", "innerBitsSet", "firstBitSet"},
+		//{differenceInPlaceWrapper, "evenBitsSet", "outerBitsSet", ""},
+		{differenceInPlaceWrapper, "evenBitsSet", "oddBitsSet", "evenBitsSet"},
+		{differenceInPlaceWrapper, "evenBitsSet", "evenBitsSet", "empty"},
 	}
 	for _, testOp := range testOps {
 		for _, x := range containerTypes {
@@ -3896,3 +4016,12 @@ func TestBitmapAny(t *testing.T) {
 		t.Error("shouldn't be any left")
 	}
 }
+
+func TestDifferenceInPlace_N(t *testing.T) {
+	a := doContainer(containerRun, runFull())
+	b := doContainer(containerBitmap, bitmapFull())
+	r := differenceInPlaceWrapper(a, b)
+	if r.N() != 0 {
+		t.Error("expected difference of containers to have n=0")
+	}
+}

roaring/roaring_test.go

@@ -1739,3 +1739,57 @@ func BenchmarkUnionBulk(b *testing.B) {
 			UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
 	}
 }
+
+func TestBitmap_DifferenceInPlace(t *testing.T) {
+	// array
+	arraybm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 1024; i += 4 {
+		_, _ = arraybm.Add((1 << 16) + i)
+	}
+	arraybm.Optimize()
+
+	// bitmap
+	bitmapbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 16384; i += 2 {
+		_, _ = bitmapbm.Add((2 << 16) + i)
+	}
+	bitmapbm.Optimize()
+
+	// small run
+	smallrunbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 1024; i++ {
+		_, _ = smallrunbm.Add((3 << 16) + i)
+	}
+	smallrunbm.Optimize()
+
+	// large run
+	largerunbm := roaring.NewSliceBitmap()
+	for i := uint64(0); i < 65535; i++ {
+		_, _ = largerunbm.Add((4 << 16) + i)
+	}
+	largerunbm.Optimize()
+
+	// test the difference in place
+	bm := testBM()
+	bm.DifferenceInPlace(arraybm, bitmapbm, smallrunbm, largerunbm)
+	if bm.Count() != 0 {
+		t.Fatalf("expected bitmap count to be 0, but got: %d", bm.Count())
+	}
+
+	bm = testBM()
+	bm.DifferenceInPlace(bitmapbm, smallrunbm, largerunbm)
+	if bm.Count() != 256 {
+		t.Fatalf("expected bitmap count to be 256, but got: %d", bm.Count())
+	}
+}
+
+func BenchmarkDifferencInPlace(b *testing.B) {
+	data := getBenchData(b)
+	for n := 0; n < b.N; n++ {
+		bm := roaring.NewBitmap()
+		bm.
+			UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
+		bm.
+			DifferenceInPlace(data.a1, data.r2, data.b, data.r1)
+	}
+}

roaring/source.go

@@ -0,0 +1,98 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roaring
+
+import (
+	"strings"
+)
+
+// A Source represents the source a given bitmap gets its data from,
+// such as a memory-mapped file. When combining bitmaps, we might
+// track them together in a single combined-source of some sort.
+type Source interface {
+	ID() string
+	Dead() bool
+}
+
+// MergeSources combines sources. If you have two bitmaps, and you're
+// combining them, then the combination's source is a combination of
+// those two sources.
+func MergeSources(sources ...Source) Source {
+	sourceCount := 0
+	totalCount := 0
+	var lastSource Source
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		lastSource = s
+		if s, ok := s.(combinedSource); ok {
+			sourceCount++
+			totalCount += len(s)
+		} else {
+			sourceCount++
+			totalCount++
+		}
+	}
+	// if there's no sources (this includes all sources being
+	// empty combinedSources), we don't have a source.
+	if totalCount == 0 {
+		return nil
+	}
+	// if there's exactly one source, combined or otherwise, that's
+	// fine, we'll just return it.
+	if sourceCount == 1 {
+		return lastSource
+	}
+	// make a new combinedSource, flattening any combinedSources
+	// already present.
+	newSources := make([]Source, 0, totalCount)
+	for _, s := range sources {
+		if s == nil {
+			continue
+		}
+		if s, ok := s.(combinedSource); ok {
+			newSources = append(newSources, s...)
+		} else {
+			newSources = append(newSources, s)
+		}
+	}
+	return combinedSource(newSources)
+}
+
+// SetSource tells the bitmap what source to associate with new things it
+// creates. This is possibly logically incorrect.
+func (b *Bitmap) SetSource(s Source) {
+	b.Source = s
+}
+
+type combinedSource []Source
+
+func (c combinedSource) ID() string {
+	ids := make([]string, len(c))
+	for i := range c {
+		ids[i] = c[i].ID()
+	}
+	return strings.Join(ids, ",")
+}
+
+func (c combinedSource) Dead() bool {
+	for i := range c {
+		if c[i].Dead() {
+			return true
+		}
+	}
+	return false
+}

roaring/unmarshal_binary.go

@@ -30,7 +30,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return nil
 	}
 	statsHit("Bitmap/UnmarshalBinary")
-	b.opN = 0 // reset opN since we're reading new data.
+	// reset ops/opN since we're reading new data.
+	b.ops = 0
+	b.opN = 0
 	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
 	if fileMagic == MagicNumber { // if pilosa roaring
 		return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
@@ -86,7 +88,12 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error {
 
 		// Map byte slice directly to the container data.
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerArray:
 			c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
@@ -106,7 +113,12 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) error {
 	citer, _ := b.Containers.Iterator(0)
 	for i := 0; i < int(keyN); i++ {
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerRun:
 			runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize])
@@ -174,12 +186,17 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 
 		// Map byte slice directly to the container data.
 		citer.Next()
-		_, c := citer.Value()
+		k, c := citer.Value()
 
 		// this shouldn't happen, since we don't normally store nils.
 		if c == nil {
 			continue
 		}
+		if !c.Mapped() {
+			fmt.Printf("inexplicable: container %d (%d/%d) doesn't think it's mapped. fixing that.\n",
+				k, i, keyN)
+			c.setMapped(true)
+		}
 		switch c.typ() {
 		case containerRun:
 			runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
@@ -205,15 +222,15 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
 		// Unmarshal the op and apply it.
 		var opr op
 		if err := opr.UnmarshalBinary(buf); err != nil {
-			// FIXME(benbjohnson): return error with position so file can be trimmed.
-			return err
+			return newFileShouldBeTruncatedError(err, int64(opsOffset))
 		}
 		opr.apply(b)
 		// Increase the op count.
 		b.ops++
 		b.opN += opr.count()
+		opsOffset += opr.size()
 		// Move the buffer forward.
-		buf = buf[opr.size():]
+		buf = data[opsOffset:]
 	}
 
 	return nil

row.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"sort"
 
+	"github.com/molecula/ext"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -43,6 +44,55 @@ func NewRow(columns ...uint64) *Row {
 	return r
 }
 
+// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
+// transposes; data that was in any shard for Row 0 is now considered shard 0,
+// etcetera.
+func NewRowFromBitmap(b *roaring.Bitmap) *Row {
+	r := &Row{}
+	if b == nil {
+		return r
+	}
+	rowNum := uint64(0)
+	for col, ok := b.MinAt(rowNum * ShardWidth); ok; col, ok = b.MinAt(rowNum * ShardWidth) {
+		rowNum = col / ShardWidth
+		seg := rowSegment{
+			shard:    rowNum,
+			data:     b.OffsetRange(rowNum*ShardWidth, rowNum*ShardWidth, (rowNum+1)*ShardWidth),
+			writable: true,
+		}
+		seg.n = seg.data.Count()
+		r.segments = append(r.segments, seg)
+		rowNum++
+	}
+	return r
+}
+
+// NewRowFromRoaring parses a roaring data file as a row, dividing it into
+// bitmaps and rowSegments based on shard width.
+func NewRowFromRoaring(data []byte) *Row {
+	bitmaps, shards := roaring.RoaringToBitmaps(data, ShardWidth)
+	r := &Row{segments: make([]rowSegment, len(bitmaps))}
+	for i := range bitmaps {
+		segment := rowSegment{
+			shard:    shards[i],
+			data:     bitmaps[i],
+			writable: false,
+			n:        bitmaps[i].Count(),
+		}
+		r.segments[i] = segment
+	}
+	return r
+}
+
+// Roaring returns the row treated as a unified roaring bitmap.
+func (r *Row) Roaring() []byte {
+	bitmaps := make([]*roaring.Bitmap, len(r.segments))
+	for i := range r.segments {
+		bitmaps[i] = r.segments[i].data
+	}
+	return roaring.BitmapsToRoaring(bitmaps)
+}
+
 // IsEmpty returns true if the row doesn't contain any set bits.
 func (r *Row) IsEmpty() bool {
 	if len(r.segments) == 0 {
@@ -194,24 +244,99 @@ func (r *Row) Union(others ...*Row) *Row {
 	return &Row{segments: output}
 }
 
-// Difference returns the diff of r and other.
-func (r *Row) Difference(other *Row) *Row {
+// GenericBinaryOp returns the output of a generic op on r and other.
+func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
 	var segments []rowSegment
-
 	itr := newMergeSegmentIterator(r.segments, other.segments)
 	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s0 == nil {
-			continue
-		} else if s1 == nil {
+		if s1 == nil {
 			segments = append(segments, *s0)
 			continue
+		} else if s0 == nil {
+			segments = append(segments, *s1)
+			continue
 		}
-		segments = append(segments, *s0.Difference(s1))
+		segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
 	}
 
 	return &Row{segments: segments}
 }
 
+// GenericNaryOp returns the output of an nary op on r and others.
+func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
+	segments := make([][]rowSegment, 0, len(others)+1)
+	if len(r.segments) > 0 {
+		segments = append(segments, r.segments)
+	}
+	nextSegs := make([][]rowSegment, 0, len(others)+1)
+	toProcess := make([]*rowSegment, 0, len(others)+1)
+	var output []rowSegment
+	for _, other := range others {
+		if len(other.segments) > 0 {
+			segments = append(segments, other.segments)
+		}
+	}
+	for len(segments) > 0 {
+		shard := segments[0][0].shard
+		for _, segs := range segments {
+			if segs[0].shard < shard {
+				shard = segs[0].shard
+			}
+		}
+		nextSegs = nextSegs[:0]
+		toProcess := toProcess[:0]
+		for _, segs := range segments {
+			if segs[0].shard == shard {
+				toProcess = append(toProcess, &segs[0])
+				segs = segs[1:]
+			}
+			if len(segs) > 0 {
+				nextSegs = append(nextSegs, segs)
+			}
+		}
+		// at this point, "toProcess" is a list of all the segments
+		// sharing the lowest ID, and nextSegs is a list of all the others.
+		// Swap the segment lists (so we don't have to reallocate it)
+		segments, nextSegs = nextSegs, segments
+		output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
+	}
+	return &Row{segments: output}
+}
+
+// Difference returns the diff of r and other.
+func (r *Row) Difference(others ...*Row) *Row {
+	var output []rowSegment
+	o := make(map[uint64][]*rowSegment)
+
+	for x := range others {
+		for y := range others[x].segments {
+			segment := others[x].segments[y]
+			o[segment.shard] = append(o[segment.shard], &segment)
+		}
+	}
+	for _, segment := range r.segments {
+
+		dest, ok := o[segment.shard]
+		if ok {
+			output = append(output, *segment.Difference(dest...))
+		} else {
+			output = append(output, segment)
+		}
+	}
+	return &Row{segments: output}
+}
+
+// GenericUnary returns the results of a generic op on r.
+func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
+	work := r
+	var segments []rowSegment
+	for _, segment := range work.segments {
+		opped := segment.GenericUnaryOp(op, args)
+		segments = append(segments, *opped)
+	}
+	return &Row{segments: segments}
+}
+
 // Shift returns the bitwise shift of r by n bits.
 // Currently only positive shift values are supported.
 func (r *Row) Shift(n int64) (*Row, error) {
@@ -299,6 +424,15 @@ func (r *Row) Count() uint64 {
 	return n
 }
 
+// GenericCount applies an op to lots of things.
+func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
+	var n int64
+	for i := range r.segments {
+		n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
+	}
+	return uint64(n)
+}
+
 // MarshalJSON returns a JSON-encoded byte slice of r.
 func (r *Row) MarshalJSON() ([]byte, error) {
 	var o struct {
@@ -326,6 +460,21 @@ func (r *Row) Columns() []uint64 {
 	return a
 }
 
+// Includes returns true if the row contains the given column.
+func (r *Row) Includes(col uint64) bool {
+	// TODO: improve the efficiency of this method by
+	// performing the column filter at the bitmap level
+	// rather than iterating through the results here.
+	for i := range r.segments {
+		for _, c := range r.segments[i].Columns() {
+			if c == col {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // rowSegment holds a subset of a row.
 // This could point to a mmapped roaring bitmap or an in-memory bitmap. The
 // width of the segment will always match the shard width.
@@ -344,9 +493,20 @@ type rowSegment struct {
 }
 
 func (s *rowSegment) Freeze() {
-	s.data.Freeze()
+	s.data = s.data.Freeze()
 }
 
+/*
+// Raw returns the row segment as a byte slice.
+// It may be used by the gRPC server to deliver results
+// as a roaring bitmap instead of a stream of RowResults.
+func (s *rowSegment) Raw() (uint64, []byte) {
+	var buf bytes.Buffer
+	s.data.WriteTo(&buf)
+	return s.shard, buf.Bytes()
+}
+*/
+
 // Merge adds chunks from other to s.
 // Chunks in s are overwritten if they exist in other.
 func (s *rowSegment) Merge(other *rowSegment) {
@@ -366,7 +526,7 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
 // Intersect returns the itersection of s and other.
 func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
 	data := s.data.Intersect(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -393,9 +553,40 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
 	}
 }
 
+// GenericOp performs a generic op on s and other
+func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
+	data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
+// GenericOp performs a generic op on s and others
+func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
+	bitmaps := make([]ext.Bitmap, len(others)+1)
+	bitmaps[0] = WrapBitmap(s.data)
+	for i, seg := range others {
+		bitmaps[i+1] = WrapBitmap(seg.data)
+	}
+	data := op(bitmaps, args)
+
+	return &rowSegment{
+		data:  UnwrapBitmap(data),
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // Difference returns the diff of s and other.
-func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
-	data := s.data.Difference(other.data)
+func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
+	datas := make([]*roaring.Bitmap, len(others))
+	for i, other := range others {
+		datas[i] = other.data
+	}
+	data := s.data.Difference(datas...)
 	data.Freeze()
 
 	return &rowSegment{
@@ -409,7 +600,7 @@ func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
 // Xor returns the xor of s and other.
 func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
 	data := s.data.Xor(other.data)
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -426,7 +617,7 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "shifting roaring data")
 	}
-	data.Freeze()
+	data = data.Freeze()
 
 	return &rowSegment{
 		data:     data,
@@ -436,6 +627,18 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	}, nil
 }
 
+// GenericUnary returns s subject to op.
+func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
+	//TODO deal with overflow
+	data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
+
+	return &rowSegment{
+		data:  data,
+		shard: s.shard,
+		n:     data.Count(),
+	}
+}
+
 // SetBit sets the i-th column of the row.
 func (s *rowSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()

row_test.go

@@ -123,5 +123,76 @@ func TestRow_IsEmpty(t *testing.T) {
 	if !res.IsEmpty() {
 		t.Fatal("Result Should Be Empty\n")
 	}
-
+}
+
+func TestRow_Includes(t *testing.T) {
+	row := pilosa.NewRow(0, 2*ShardWidth)
+
+	if !row.Includes(0) {
+		t.Fatal("row should include 0")
+	}
+	if row.Includes(1) {
+		t.Fatal("row should not include 1")
+	}
+	if !row.Includes(2 * ShardWidth) {
+		t.Fatalf("row should include %d", 2*ShardWidth)
+	}
+}
+
+func TestRow_DifferenceInPlace(t *testing.T) {
+	row0 := pilosa.NewRow(0)
+	row1 := pilosa.NewRow()
+	row2 := pilosa.NewRow(0)
+	res := row0.Difference(row1, row2)
+
+	if !row0.Includes(0) {
+		t.Fatal("row should include 0")
+	}
+	if res.Count() != 0 {
+		t.Fatal("results should be empty")
+	}
+}
+func eq(a, b []uint64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+func TestRow_DifferenceInPlace2(t *testing.T) {
+	lucky := []uint64{1, 3, 7, 9, 13, 15, 21, 25, 31, 33, 37, 43, 49, 51, 63, 67, 69, 73, 75, 79, 87, 93, 99, 105, 111, 115, 127, 129, 133, 135, 141, 151, 159, 163, 169, 171, 189, 193, 195, 201, 205, 211, 219, 223, 231, 235, 237, 241, 259, 261, 267, 273, 283, 285, 289, 297}
+	src := pilosa.NewRow(lucky...)
+	row1 := pilosa.NewRow()
+	m := uint64(9)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row1.SetBit(i)
+	}
+	m = uint64(3)
+	row2 := pilosa.NewRow()
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row2.SetBit(i)
+	}
+	row3 := pilosa.NewRow()
+	m = uint64(5)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row3.SetBit(i)
+	}
+	row4 := pilosa.NewRow()
+	m = uint64(7)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row4.SetBit(i)
+	}
+	res5 := src.Difference(row1, row2, row3, row4)
+	res6 := src.Difference(row4, row3, row2, row1)
+	res7 := src.Difference(row4).Difference(row3).Difference(row2).Difference(row1)
+	if !eq(res5.Columns(), res6.Columns()) {
+		t.Fatalf("results do not match: %v, %v", res5.Columns(), res6.Columns())
+	}
+	if !eq(res6.Columns(), res7.Columns()) {
+		t.Fatalf("results do not match: %v, %v", res6.Columns(), res7.Columns())
+	}
 }

server.go

@@ -27,7 +27,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/molecula/ext"
+	// extensions pulls in some extensions depending on build tags
+	_ "github.com/pilosa/pilosa/v2/extensions"
 	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pkg/errors"
@@ -57,6 +61,7 @@ type Server struct { // nolint: maligned
 	hosts            []string
 	clusterDisabled  bool
 	serializer       Serializer
+	extensions       []*ext.ExtensionInfo
 
 	// External
 	systemInfo SystemInfo
@@ -72,6 +77,9 @@ type Server struct { // nolint: maligned
 	isCoordinator       bool
 	syncer              holderSyncer
 
+	translationSyncer      translationSyncer
+	resetTranslationSyncCh chan struct{}
+
 	defaultClient InternalClient
 	dataDir       string
 }
@@ -295,10 +303,12 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
 
 // NewServer returns a new instance of Server.
 func NewServer(opts ...ServerOption) (*Server, error) {
+	cluster := newCluster()
+
 	s := &Server{
 		closing:       make(chan struct{}),
-		cluster:       newCluster(),
-		holder:        NewHolder(),
+		cluster:       cluster,
+		holder:        NewHolder(cluster.partitionN),
 		diagnostics:   newDiagnosticsCollector(defaultDiagnosticServer),
 		systemInfo:    newNopSystemInfo(),
 		defaultClient: nopInternalClient{},
@@ -309,10 +319,16 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		metricInterval:      0,
 		diagnosticInterval:  0,
 
+		resetTranslationSyncCh: make(chan struct{}),
+
 		logger: logger.NopLogger,
 	}
 	s.cluster.InternalClient = s.defaultClient
 
+	s.translationSyncer = newActiveTranslationSyncer(s.resetTranslationSyncCh)
+	s.holder.translationSyncer = s.translationSyncer
+	s.cluster.translationSyncer = s.translationSyncer
+
 	s.diagnostics.server = s
 
 	for _, opt := range opts {
@@ -335,7 +351,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	s.holder.Path = path
 	// s.holder.translateFile.Path = filepath.Join(path, ".keys")
 	s.holder.Logger = s.logger
@@ -376,6 +391,30 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	s.cluster.broadcaster = s
 	s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
 	s.holder.broadcaster = s
+	err = s.loadAllExtensions()
+	if err != nil {
+		s.logger.Printf("not all plugins loaded successfully")
+	}
+	if len(s.extensions) > 0 {
+		s.logger.Printf("loaded extensions:")
+		for _, ext := range s.extensions {
+			if ext == nil {
+				s.logger.Printf("  inexplicably, a nil extension?!?")
+				continue
+			}
+			s.logger.Printf("  %s %s: %s", ext.Name, ext.Version, ext.Description)
+			if ext.License != "" {
+				s.logger.Printf("    License: %s", ext.License)
+			}
+			if len(ext.BitmapOps) > 0 {
+				opList := make([]string, len(ext.BitmapOps))
+				for i := range ext.BitmapOps {
+					opList[i] = ext.BitmapOps[i].Name
+				}
+				s.logger.Printf("    Ops: %s", strings.Join(opList, ", "))
+			}
+		}
+	}
 
 	err = s.cluster.setup()
 	if err != nil {
@@ -389,6 +428,56 @@ func (s *Server) InternalClient() InternalClient {
 	return s.defaultClient
 }
 
+// loadNewExtensions loads extensions that have been
+// registered since the last call to loadNewExtensions.
+func (s *Server) loadNewExtensions() error { //nolint:unused
+	return s.loadExtensions(ext.NewExtensions())
+}
+
+// loadAllExtensions loads all extensions.
+func (s *Server) loadAllExtensions() error {
+	return s.loadExtensions(ext.AllExtensions())
+}
+
+func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error {
+	var lastError error
+	for _, extension := range exts {
+		if err := s.loadExtension(extension); err != nil {
+			lastError = err
+		}
+	}
+	return lastError
+}
+
+func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
+	if extInfo.ExtensionAPI != "v0" {
+		return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI)
+	}
+	s.extensions = append(s.extensions, extInfo)
+	bitmapOps := extInfo.BitmapOps
+	bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
+	for i := range bitmapOps {
+		typ := bitmapOps[i].Func.BitmapOpType()
+		switch {
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
+			countOps++
+		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
+			bmOps++
+		case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
+			fieldOps++
+		default:
+			unknownOps++
+		}
+	}
+	err := s.executor.registerOps(bitmapOps)
+	if err != nil {
+		s.logger.Printf("warning: extension registration failed: %v", err)
+	} else {
+		pql.RegisterPluginFuncs(bitmapOps)
+	}
+	return nil
+}
+
 // UpAndDown brings the server up minimally and shuts it down
 // again; basically, it exists for testing holder open and close.
 func (s *Server) UpAndDown() error {
@@ -423,6 +512,18 @@ func (s *Server) Open() error {
 		log.Println(errors.Wrap(err, "logging startup"))
 	}
 
+	// Set up the holderSyncer.
+	s.syncer.Holder = s.holder
+	s.syncer.Node = s.cluster.Node
+	s.syncer.Cluster = s.cluster
+	s.syncer.Closing = s.closing
+	s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer")
+
+	// Start background process listening for translation
+	// sync resets.
+	s.wg.Add(1)
+	go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }()
+
 	// Open Cluster management.
 	if err := s.cluster.waitForStarted(); err != nil {
 		return errors.Wrap(err, "opening Cluster")
@@ -443,12 +544,6 @@ func (s *Server) Open() error {
 	// buffered channel.
 	s.cluster.listenForJoins()
 
-	s.syncer.Holder = s.holder
-	s.syncer.Node = s.cluster.Node
-	s.syncer.Cluster = s.cluster
-	s.syncer.Closing = s.closing
-	s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer")
-
 	// Start background monitoring.
 	s.wg.Add(3)
 	go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
@@ -511,6 +606,34 @@ func (s *Server) SyncData() error {
 	return errors.Wrap(s.syncer.SyncHolder(), "syncing holder")
 }
 
+// monitorResetTranslationSync is a background process which
+// listens for events indicating the need to reset the translation
+// sync processes.
+func (s *Server) monitorResetTranslationSync() {
+	s.logger.Printf("holder translation sync monitor initializing")
+	for {
+		// Wait for a reset or a close.
+		select {
+		case <-s.closing:
+			return
+		case <-s.resetTranslationSyncCh:
+			s.logger.Printf("holder translation sync beginning")
+			s.wg.Add(1)
+			go func() {
+				// Obtaining this lock ensures that there is only
+				// once instance of resetTranslationSync() running
+				// at once.
+				s.syncer.mu.Lock()
+				defer s.syncer.mu.Unlock()
+				defer s.wg.Done()
+				if err := s.syncer.resetTranslationSync(); err != nil {
+					s.logger.Printf("holder translation sync error: err=%s", err)
+				}
+			}()
+		}
+	}
+}
+
 func (s *Server) monitorAntiEntropy() {
 	if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 {
 		return // anti entropy disabled
@@ -592,7 +715,7 @@ func (s *Server) receiveMessage(m Message) error {
 			return fmt.Errorf("local index not found: %s", obj.Index)
 		}
 		opt := obj.Meta
-		_, err := idx.createFieldIfNotExists(obj.Field, *opt)
+		_, err := idx.createFieldIfNotExists(obj.Field, opt)
 		if err != nil {
 			return err
 		}

server/config.go

@@ -53,6 +53,9 @@ type Config struct {
 	// Bind is the host:port on which Pilosa will listen.
 	Bind string `toml:"bind"`
 
+	// BindGRPC is the host:port on which Pilosa will bind for gRPC.
+	BindGRPC string `toml:"bind-grpc"`
+
 	// Advertise is the address advertised by the server to other nodes
 	// in the cluster. It should be reachable by all other nodes and should
 	// route to an interface that Bind is listening on.
@@ -161,6 +164,7 @@ func NewConfig() *Config {
 	c := &Config{
 		DataDir:             "~/.pilosa",
 		Bind:                ":10101",
+		BindGRPC:            ":20101",
 		MaxWritesPerRequest: 5000,
 
 		// We default these Max File/Map counts very high. This is basically a
@@ -233,6 +237,13 @@ func (cfg *Config) validateAddrs(ctx context.Context) error {
 	}
 	cfg.Bind = schemeHostPortString(listenScheme, listenHost, listenPort)
 
+	// Validate the gRPC listen address.
+	grpcListenScheme, grpcListenHost, grpcListenPort, err := validateListenAddr(ctx, cfg.BindGRPC)
+	if err != nil {
+		return errors.Wrap(err, "validating grpc listen address")
+	}
+	cfg.BindGRPC = schemeHostPortString(grpcListenScheme, grpcListenHost, grpcListenPort)
+
 	return nil
 }
 

server/grpc.go

@@ -0,0 +1,908 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/logger"
+	pb "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/reflection"
+	"google.golang.org/grpc/status"
+)
+
+// grpcHandler contains methods which handle the various gRPC requests.
+type grpcHandler struct {
+	api *pilosa.API
+
+	logger logger.Logger
+}
+
+// errorToStatusError appends an appropriate grpc status code
+// to the error (returning it as a status.Error). It is
+// assumed that the input err is non-nil.
+func errToStatusError(err error) error {
+	// Check error string.
+	switch errors.Cause(err) {
+	case pilosa.ErrIndexNotFound, pilosa.ErrFieldNotFound:
+		return status.Error(codes.NotFound, err.Error())
+	}
+	// Check error type.
+	switch errors.Cause(err).(type) {
+	case pilosa.NotFoundError:
+		return status.Error(codes.NotFound, err.Error())
+	}
+	return status.Error(codes.Unknown, err.Error())
+}
+
+// QueryPQL handles the PQL request and sends RowResponses to the stream.
+func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQLServer) error {
+	query := pilosa.QueryRequest{
+		Index: req.Index,
+		Query: req.Pql,
+	}
+	resp, err := h.api.Query(context.Background(), &query)
+	if err != nil {
+		return errToStatusError(err)
+	}
+	for row := range makeRows(resp, h.logger) {
+		err = stream.Send(row)
+		if err != nil {
+			return errToStatusError(err)
+		}
+	}
+
+	return nil
+}
+
+// fieldDataType returns a useful data type (string,
+// uint64, bool, etc.) based on the Pilosa field type.
+func fieldDataType(f *pilosa.Field) string {
+	switch f.Type() {
+	case "set":
+		if f.Keys() {
+			return "[]string"
+		}
+		return "[]uint64"
+	case "mutex":
+		if f.Keys() {
+			return "string"
+		}
+		return "uint64"
+	case "int":
+		if f.Keys() {
+			return "string"
+		}
+		return "int64"
+	case "decimal":
+		return "float64"
+	case "bool":
+		return "bool"
+	case "time":
+		return "int64" // TODO: this is a placeholder
+	default:
+		panic(fmt.Sprintf("unimplemented fieldDataType: %s", f.Type()))
+	}
+}
+
+// Inspect handles the inspect request and sends an InspectResponse to the stream.
+func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error {
+	const defaultLimit = 100000
+
+	index, err := h.api.Index(context.Background(), req.Index)
+	if err != nil {
+		return errToStatusError(err)
+	}
+
+	var fields []*pilosa.Field
+	for _, field := range index.Fields() {
+		// exclude internal fields (starting with "_")
+		if strings.HasPrefix(field.Name(), "_") {
+			continue
+		}
+		if len(req.FilterFields) > 0 {
+			for _, filter := range req.FilterFields {
+				if filter == field.Name() {
+					fields = append(fields, field)
+					break
+				}
+
+			}
+		} else {
+			fields = append(fields, field)
+		}
+	}
+
+	limit := req.Limit
+	if limit == 0 {
+		limit = defaultLimit
+	}
+	offset := req.Offset
+
+	if !index.Keys() {
+		ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids)
+		if !ok {
+			return errors.New("invalid int columns")
+		}
+		ci := []*pb.ColumnInfo{
+			{Name: "_id", Datatype: "uint64"},
+		}
+		for _, field := range fields {
+			ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)})
+		}
+
+		// If Columns is empty, then get the _exists list (via All()),
+		// from the index and loop over that instead.
+		cols := ints.Ids.Vals
+		if len(cols) > 0 {
+			// Apply limit/offset to the provided columns.
+			if int(offset) >= len(cols) {
+				return nil
+			}
+			end := limit + offset
+			if int(end) > len(cols) {
+				end = uint64(len(cols))
+			}
+			cols = cols[offset:end]
+		} else {
+			// Prevent getting too many records by forcing a limit.
+			pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset)
+			query := pilosa.QueryRequest{
+				Index: req.Index,
+				Query: pql,
+			}
+			resp, err := h.api.Query(context.Background(), &query)
+			if err != nil {
+				return errors.Wrapf(err, "querying for all: %s", pql)
+			}
+
+			ids, ok := resp.Results[0].(*pilosa.Row)
+			if !ok {
+				return errors.Wrap(err, "getting results as a row")
+			}
+
+			limitedCols := ids.Columns()
+			if len(limitedCols) == 0 {
+				// If cols is still empty after the limit/offset, then
+				// return with no results.
+				return nil
+			}
+			cols = limitedCols
+		}
+
+		for _, col := range cols {
+			rowResp := &pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: col}},
+				},
+			}
+			ci = nil // only include headers with the first row
+
+			for _, field := range fields {
+				// TODO: handle `time` fields
+				switch field.Type() {
+				case "set":
+					pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrapf(err, "querying rows for set: %s", pql)
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Keys) > 0 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringArrayVal{StringArrayVal: &pb.StringArray{Vals: ids.Keys}}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}})
+					}
+
+				case "mutex":
+					pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrap(err, "querying rows for mutex")
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Keys) == 1 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: ids.Keys[0]}})
+					} else if len(ids.Rows) == 1 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: ids.Rows[0]}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "int":
+					if field.Keys() {
+						var value string
+						var exists bool
+						var err error
+						if fi := field.ForeignIndex(); fi != "" {
+							// Get the value from the int field.
+							intVal, ok, err := field.Value(col)
+							if err != nil {
+								return errors.Wrap(err, "getting int value")
+							} else if ok {
+								vals, err := h.api.TranslateIndexIDs(context.Background(), fi, []uint64{uint64(intVal)})
+								if err != nil {
+									return errors.Wrap(err, "getting keys for ids")
+								}
+								if len(vals) > 0 && vals[0] != "" {
+									value = vals[0]
+									exists = true
+								}
+							}
+						} else {
+							value, exists, err = field.StringValue(col)
+							if err != nil {
+								return errors.Wrap(err, "getting string field value for column")
+							}
+						}
+						if exists {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}})
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
+						}
+					} else {
+						value, exists, err := field.Value(col)
+						if err != nil {
+							return errors.Wrap(err, "getting int field value for column")
+						} else if exists {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
+						}
+					}
+
+				case "decimal":
+					value, exists, err := field.FloatValue(col)
+					if err != nil {
+						return errors.Wrap(err, "getting decimal field value for column")
+					} else if exists {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Float64Val{Float64Val: value}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "bool":
+					pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrap(err, "querying rows for bool")
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Rows) == 1 {
+						var bval bool
+						if ids.Rows[0] == 1 {
+							bval = true
+						}
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bval}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "time":
+					rowResp.Columns = append(rowResp.Columns,
+						&pb.ColumnResponse{ColumnVal: nil})
+				}
+			}
+
+			if err := stream.Send(rowResp); err != nil {
+				return errors.Wrap(err, "sending response to stream")
+			}
+		}
+
+	} else {
+		var cols []string
+
+		switch keys := req.Columns.Type.(type) {
+		case *pb.IdsOrKeys_Ids:
+			// The default behavior (in api/client/grpc.go) is to
+			// send an empty set of Ids even if the index supports
+			// keys, so in that case we just need to ignore it.
+		case *pb.IdsOrKeys_Keys:
+			cols = keys.Keys.Vals
+		default:
+			return errToStatusError(errors.New("invalid key columns"))
+		}
+
+		ci := []*pb.ColumnInfo{
+			{Name: "_id", Datatype: "string"},
+		}
+		for _, field := range fields {
+			ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)})
+		}
+
+		// If Columns is empty, then get the _exists list (via All()),
+		// from the index and loop over that instead.
+		if len(cols) > 0 {
+			// Apply limit/offset to the provided columns.
+			if int(offset) >= len(cols) {
+				return nil
+			}
+			end := limit + offset
+			if int(end) > len(cols) {
+				end = uint64(len(cols))
+			}
+			cols = cols[offset:end]
+		} else {
+			// Prevent getting too many records by forcing a limit.
+			pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset)
+			query := pilosa.QueryRequest{
+				Index: req.Index,
+				Query: pql,
+			}
+			resp, err := h.api.Query(context.Background(), &query)
+			if err != nil {
+				return errors.Wrapf(err, "querying for all: %s", pql)
+			}
+
+			ids, ok := resp.Results[0].(*pilosa.Row)
+			if !ok {
+				return errors.Wrap(err, "getting results as a row")
+			}
+
+			limitedCols := ids.Keys
+			if len(limitedCols) == 0 {
+				// If cols is still empty after the limit/offset, then
+				// return with no results.
+				return nil
+			}
+			cols = limitedCols
+		}
+
+		for _, col := range cols {
+			rowResp := &pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: col}},
+				},
+			}
+			ci = nil // only include headers with the first row
+
+			for _, field := range fields {
+				// TODO: handle `time` fields
+				switch field.Type() {
+				case "set":
+					pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrap(err, "querying set rows(keys)")
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Keys) > 0 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringArrayVal{StringArrayVal: &pb.StringArray{Vals: ids.Keys}}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}})
+					}
+
+				case "mutex":
+					pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrap(err, "querying mutex rows(keys)")
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Keys) == 1 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: ids.Keys[0]}})
+					} else if len(ids.Rows) == 1 {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: ids.Rows[0]}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "int":
+					// Translate column key.
+					id, err := h.api.TranslateIndexKey(context.Background(), index.Name(), col)
+					if err != nil {
+						return errors.Wrap(err, "translating column key")
+					}
+
+					if field.Keys() {
+						var value string
+						var exists bool
+						var err error
+						if fi := field.ForeignIndex(); fi != "" {
+							// Get the value from the int field.
+							intVal, ok, err := field.Value(id)
+							if err != nil {
+								return errors.Wrap(err, "getting int value")
+							} else if ok {
+								vals, err := h.api.TranslateIndexIDs(context.Background(), fi, []uint64{uint64(intVal)})
+								if err != nil {
+									return errors.Wrap(err, "getting keys for ids")
+								}
+								if len(vals) > 0 && vals[0] != "" {
+									value = vals[0]
+									exists = true
+								}
+							}
+						} else {
+							value, exists, err = field.StringValue(id)
+							if err != nil {
+								return errors.Wrap(err, "getting string field value for column")
+							}
+						}
+						if exists {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}})
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
+						}
+					} else {
+						value, exists, err := field.Value(id)
+						if err != nil {
+							return errors.Wrap(err, "getting int field value for column")
+						} else if exists {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: value}})
+						} else {
+							rowResp.Columns = append(rowResp.Columns,
+								&pb.ColumnResponse{ColumnVal: nil})
+						}
+					}
+
+				case "decimal":
+					// Translate column key.
+					id, err := h.api.TranslateIndexKey(context.Background(), index.Name(), col)
+					if err != nil {
+						return errors.Wrap(err, "translating column key")
+					}
+
+					value, exists, err := field.FloatValue(id)
+					if err != nil {
+						return errors.Wrap(err, "getting decimal field value for column")
+					} else if exists {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Float64Val{Float64Val: value}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "bool":
+					pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col)
+					query := pilosa.QueryRequest{
+						Index: req.Index,
+						Query: pql,
+					}
+					resp, err := h.api.Query(context.Background(), &query)
+					if err != nil {
+						return errors.Wrap(err, "querying bool rows(keys)")
+					}
+
+					ids, ok := resp.Results[0].(pilosa.RowIdentifiers)
+					if !ok {
+						return errors.Wrap(err, "getting row identifiers")
+					}
+
+					if len(ids.Rows) == 1 {
+						var bval bool
+						if ids.Rows[0] == 1 {
+							bval = true
+						}
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bval}})
+					} else {
+						rowResp.Columns = append(rowResp.Columns,
+							&pb.ColumnResponse{ColumnVal: nil})
+					}
+
+				case "time":
+					rowResp.Columns = append(rowResp.Columns,
+						&pb.ColumnResponse{ColumnVal: nil})
+				}
+			}
+
+			if err := stream.Send(rowResp); err != nil {
+				return errors.Wrap(err, "sending response to stream")
+			}
+		}
+
+	}
+	return nil
+}
+
+// I think ideally this would be plugged in the executor somewhere
+// in order to get some concurrency benefit but we can
+// start with the combined response
+func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowResponse {
+	results := make(chan *pb.RowResponse)
+	go func() {
+		var breakLoop bool // Support the "break" inside the switch.
+		for _, result := range resp.Results {
+			if breakLoop {
+				break
+			}
+			switch r := result.(type) {
+			case *pilosa.Row:
+				if len(r.Keys) > 0 {
+					// Column keys
+					ci := []*pb.ColumnInfo{
+						{Name: "_id", Datatype: "string"},
+					}
+					for _, x := range r.Keys {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}},
+							}}
+						ci = nil //only send on the first
+					}
+				} else {
+					// Column IDs
+					ci := []*pb.ColumnInfo{
+						{Name: "_id", Datatype: "uint64"},
+					}
+					for _, x := range r.Columns() {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}},
+							}}
+						ci = nil //only send on the first
+					}
+
+					// The following will return roaring segments.
+					// This is commented out for now until we decide how we want to use this.
+					/*
+						// Roaring segments
+						ci := []*pb.ColumnInfo{
+							// TODO:
+							{Name: "shard", Datatype: "uint64"},
+							{Name: "segment", Datatype: "roaring"},
+						}
+						for _, x := range r.Segments() {
+							shard, b := x.Raw()
+							results <- &pb.RowResponse{
+								Headers: ci,
+								Columns: []*pb.ColumnResponse{
+									&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_IntVal{int64(shard)}},
+									&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BlobVal{b}},
+								}}
+							ci = nil //only send on the first
+						}
+					*/
+				}
+			case pilosa.PairField:
+				if r.Pair.Key != "" {
+					results <- &pb.RowResponse{
+						Headers: []*pb.ColumnInfo{
+							{Name: r.Field, Datatype: "string"},
+							{Name: "count", Datatype: "uint64"},
+						},
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Pair.Key}},
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}},
+						},
+					}
+				} else {
+					results <- &pb.RowResponse{
+						Headers: []*pb.ColumnInfo{
+							{Name: r.Field, Datatype: "uint64"},
+							{Name: "count", Datatype: "uint64"},
+						},
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.ID}},
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}},
+						},
+					}
+				}
+			case *pilosa.PairsField:
+				// Determine if the ID has string keys.
+				var stringKeys bool
+				if len(r.Pairs) > 0 {
+					if r.Pairs[0].Key != "" {
+						stringKeys = true
+					}
+				}
+
+				dtype := "uint64"
+				if stringKeys {
+					dtype = "string"
+				}
+				ci := []*pb.ColumnInfo{
+					{Name: r.Field, Datatype: dtype},
+					{Name: "count", Datatype: "uint64"},
+				}
+				for _, pair := range r.Pairs {
+					if stringKeys {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+							},
+						}
+					} else {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+							},
+						}
+					}
+					ci = nil //only send on the first
+				}
+			case []pilosa.GroupCount:
+				for i, gc := range r {
+					var ci []*pb.ColumnInfo
+					if i == 0 {
+						for _, fieldRow := range gc.Group {
+							if fieldRow.RowKey != "" {
+								ci = append(ci, &pb.ColumnInfo{Name: fieldRow.Field, Datatype: "string"})
+							} else {
+								ci = append(ci, &pb.ColumnInfo{Name: fieldRow.Field, Datatype: "uint64"})
+							}
+						}
+						ci = append(ci, &pb.ColumnInfo{Name: "count", Datatype: "uint64"})
+						ci = append(ci, &pb.ColumnInfo{Name: "sum", Datatype: "int64"})
+					}
+					rowResp := &pb.RowResponse{
+						Headers: ci,
+						Columns: []*pb.ColumnResponse{},
+					}
+
+					for _, fieldRow := range gc.Group {
+						if fieldRow.RowKey != "" {
+							rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: fieldRow.RowKey}})
+						} else {
+							rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(fieldRow.RowID)}})
+						}
+					}
+					rowResp.Columns = append(rowResp.Columns,
+						&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}},
+						&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Sum}},
+					)
+					results <- rowResp
+				}
+			case pilosa.RowIdentifiers:
+				if len(r.Keys) > 0 {
+					ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "string"}}
+					for _, key := range r.Keys {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: key}},
+							}}
+						ci = nil
+					}
+				} else {
+					ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "uint64"}}
+					for _, id := range r.Rows {
+						results <- &pb.RowResponse{
+							Headers: ci,
+							Columns: []*pb.ColumnResponse{
+								&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(id)}},
+							}}
+						ci = nil
+					}
+				}
+			case uint64:
+				ci := []*pb.ColumnInfo{{Name: "count", Datatype: "uint64"}}
+				results <- &pb.RowResponse{
+					Headers: ci,
+					Columns: []*pb.ColumnResponse{
+						&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}},
+					}}
+			case bool:
+				ci := []*pb.ColumnInfo{{Name: "result", Datatype: "bool"}}
+				results <- &pb.RowResponse{
+					Headers: ci,
+					Columns: []*pb.ColumnResponse{
+						&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: r}},
+					}}
+			case pilosa.ValCount:
+				var ci []*pb.ColumnInfo
+				// ValCount can have a float or integeger value, but
+				// not both (as of this writing).
+				if r.FloatVal != 0 {
+					ci = []*pb.ColumnInfo{
+						{Name: "value", Datatype: "float64"},
+						{Name: "count", Datatype: "int64"},
+					}
+					results <- &pb.RowResponse{
+						Headers: ci,
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Float64Val{Float64Val: r.FloatVal}},
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Count}},
+						}}
+				} else {
+					ci = []*pb.ColumnInfo{
+						{Name: "value", Datatype: "int64"},
+						{Name: "count", Datatype: "int64"},
+					}
+					results <- &pb.RowResponse{
+						Headers: ci,
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Val}},
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Count}},
+						}}
+				}
+			case pilosa.SignedRow:
+				// TODO: address the overflow issue with values outside the int64 range
+				ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "int64"}}
+				negs := r.Neg.Columns()
+				for i := len(negs) - 1; i >= 0; i-- {
+					results <- &pb.RowResponse{
+						Headers: ci,
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: -1 * int64(negs[i])}},
+						}}
+					ci = nil
+				}
+				for _, id := range r.Pos.Columns() {
+					results <- &pb.RowResponse{
+						Headers: ci,
+						Columns: []*pb.ColumnResponse{
+							&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: int64(id)}},
+						}}
+					ci = nil
+				}
+
+			default:
+				logger.Printf("unhandled %T\n", r)
+				breakLoop = true
+			}
+		}
+		close(results)
+	}()
+	return results
+}
+
+type grpcServer struct {
+	api        *pilosa.API
+	grpcServer *grpc.Server
+	hostPort   string
+
+	logger logger.Logger
+}
+
+type grpcServerOption func(s *grpcServer) error
+
+func OptGRPCServerAPI(api *pilosa.API) grpcServerOption {
+	return func(s *grpcServer) error {
+		s.api = api
+		return nil
+	}
+}
+
+func OptGRPCServerURI(uri *pilosa.URI) grpcServerOption {
+	hostport := fmt.Sprintf("%s:%d", uri.Host, uri.Port)
+	return func(s *grpcServer) error {
+		s.hostPort = hostport
+		return nil
+	}
+}
+
+func OptGRPCServerLogger(logger logger.Logger) grpcServerOption {
+	return func(s *grpcServer) error {
+		s.logger = logger
+		return nil
+	}
+}
+
+func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
+	// create listener
+	lis, err := net.Listen("tcp", s.hostPort)
+	if err != nil {
+		return errors.Wrap(err, "creating listener")
+	}
+	s.logger.Printf("enabled grpc listening on %s", lis.Addr())
+
+	opts := make([]grpc.ServerOption, 0)
+	if tlsConfig != nil {
+		creds := credentials.NewTLS(tlsConfig)
+		opts = append(opts, grpc.Creds(creds))
+	}
+
+	// create grpc server
+	s.grpcServer = grpc.NewServer(opts...)
+	pb.RegisterPilosaServer(s.grpcServer, grpcHandler{api: s.api, logger: s.logger})
+
+	// register the server so its services are available to grpc_cli and others
+	reflection.Register(s.grpcServer)
+
+	// and start...
+	if err := s.grpcServer.Serve(lis); err != nil {
+		return errors.Wrap(err, "starting grpc server")
+	}
+	return nil
+}
+
+func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
+	server := &grpcServer{
+		logger: logger.NopLogger,
+	}
+	for _, opt := range opts {
+		err := opt(server)
+		if err != nil {
+			return nil, errors.Wrap(err, "applying option")
+		}
+	}
+	return server, nil
+}

server/grpc_internal_test.go

@@ -0,0 +1,293 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/logger"
+)
+
+func TestGRPC(t *testing.T) {
+	t.Run("makeRows", func(t *testing.T) {
+		type expHeader struct {
+			name     string
+			dataType string
+		}
+
+		type expColumn interface{}
+
+		tests := []struct {
+			result     interface{}
+			expHeaders []expHeader
+			expColumns [][]expColumn
+		}{
+			{
+				pilosa.ValCount{Val: 1, Count: 1},
+				[]expHeader{{"value", "int64"}, {"count", "int64"}},
+				[][]expColumn{{int64(1), int64(1)}},
+			},
+			{
+				pilosa.ValCount{FloatVal: 1.24, Count: 1},
+				[]expHeader{{"value", "float64"}, {"count", "int64"}},
+				[][]expColumn{{float64(1.24), int64(1)}},
+			},
+			// Row (uint64)
+			{
+				pilosa.NewRow(10, 11, 12),
+				[]expHeader{
+					{"_id", "uint64"},
+				},
+				[][]expColumn{
+					{uint64(10)},
+					{uint64(11)},
+					{uint64(12)},
+				},
+			},
+			// Row (string)
+			{
+				&pilosa.Row{Keys: []string{"ten", "eleven", "twelve"}},
+				[]expHeader{
+					{"_id", "string"},
+				},
+				[][]expColumn{
+					{"ten"},
+					{"eleven"},
+					{"twelve"},
+				},
+			},
+			// Pair (uint64)
+			{
+				pilosa.Pair{ID: 10, Count: 123},
+				[]expHeader{
+					{"_id", "uint64"},
+					{"count", "uint64"},
+				},
+				[][]expColumn{
+					{uint64(10), uint64(123)},
+				},
+			},
+			// Pair (string)
+			{
+				pilosa.Pair{Key: "ten", Count: 123},
+				[]expHeader{
+					{"_id", "string"},
+					{"count", "uint64"},
+				},
+				[][]expColumn{
+					{string("ten"), uint64(123)},
+				},
+			},
+			// []Pair (uint64)
+			{
+				[]pilosa.Pair{
+					{ID: 10, Count: 123},
+					{ID: 11, Count: 456},
+				},
+				[]expHeader{
+					{"_id", "uint64"},
+					{"count", "uint64"},
+				},
+				[][]expColumn{
+					{uint64(10), uint64(123)},
+					{uint64(11), uint64(456)},
+				},
+			},
+			// []Pair (string)
+			{
+				[]pilosa.Pair{
+					{Key: "ten", Count: 123},
+					{Key: "eleven", Count: 456},
+				},
+				[]expHeader{
+					{"_id", "string"},
+					{"count", "uint64"},
+				},
+				[][]expColumn{
+					{"ten", uint64(123)},
+					{"eleven", uint64(456)},
+				},
+			},
+			// []GroupCount (uint64)
+			{
+				[]pilosa.GroupCount{
+					pilosa.GroupCount{
+						Group: []pilosa.FieldRow{
+							{Field: "a", RowID: 10},
+							{Field: "b", RowID: 11},
+						},
+						Count: 123,
+					},
+					pilosa.GroupCount{
+						Group: []pilosa.FieldRow{
+							{Field: "a", RowID: 10},
+							{Field: "b", RowID: 12},
+						},
+						Count: 456,
+					},
+				},
+				[]expHeader{
+					{"a", "uint64"},
+					{"b", "uint64"},
+					{"count", "uint64"},
+					{"sum", "int64"},
+				},
+				[][]expColumn{
+					{uint64(10), uint64(11), uint64(123), int64(0)},
+					{uint64(10), uint64(12), uint64(456), int64(0)},
+				},
+			},
+			// []GroupCount (string)
+			{
+				[]pilosa.GroupCount{
+					pilosa.GroupCount{
+						Group: []pilosa.FieldRow{
+							{Field: "a", RowKey: "ten"},
+							{Field: "b", RowKey: "eleven"},
+						},
+						Count: 123,
+					},
+					{
+						Group: []pilosa.FieldRow{
+							{Field: "a", RowKey: "ten"},
+							{Field: "b", RowKey: "twelve"},
+						},
+						Count: 456,
+					},
+				},
+				[]expHeader{
+					{"a", "string"},
+					{"b", "string"},
+					{"count", "uint64"},
+					{"sum", "int64"},
+				},
+				[][]expColumn{
+					{"ten", "eleven", uint64(123), int64(0)},
+					{"ten", "twelve", uint64(456), int64(0)},
+				},
+			},
+			// RowIdentifiers (uint64)
+			{
+				pilosa.RowIdentifiers{
+					Rows: []uint64{10, 11, 12},
+				},
+				[]expHeader{
+					{"", "uint64"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests.
+				},
+				[][]expColumn{
+					{uint64(10)},
+					{uint64(11)},
+					{uint64(12)},
+				},
+			},
+			// RowIdentifiers (string)
+			{
+				pilosa.RowIdentifiers{
+					Keys: []string{"ten", "eleven", "twelve"},
+				},
+				[]expHeader{
+					{"", "string"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests.
+				},
+				[][]expColumn{
+					{"ten"},
+					{"eleven"},
+					{"twelve"},
+				},
+			},
+			// uint64
+			{
+				uint64(123),
+				[]expHeader{
+					{"count", "uint64"},
+				},
+				[][]expColumn{
+					{uint64(123)},
+				},
+			},
+			// bool
+			{
+				true,
+				[]expHeader{
+					{"result", "bool"},
+				},
+				[][]expColumn{
+					{true},
+				},
+			},
+		}
+
+		logger := logger.NopLogger
+		for ti, test := range tests {
+			results := make([]interface{}, 0)
+			results = append(results, test.result)
+
+			qr := pilosa.QueryResponse{}
+			qr.Results = results
+
+			ch := makeRows(qr, logger)
+
+			cnt := 0
+			for row := range ch {
+				// Ensure headers match (on the first row).
+				if cnt == 0 {
+					for i, header := range row.GetHeaders() {
+						if header.Name != test.expHeaders[i].name {
+							t.Fatalf("test %d expected header name: %s, but got: %s", ti, test.expHeaders[i].name, header.Name)
+						}
+						if header.Datatype != test.expHeaders[i].dataType {
+							t.Fatalf("test %d expected header data type: %s, but got: %s", ti, test.expHeaders[i].dataType, header.Datatype)
+						}
+					}
+				}
+
+				// Ensure column data matches.
+				for i, column := range row.GetColumns() {
+					switch v := test.expColumns[cnt][i].(type) {
+					case string:
+						val := column.GetStringVal()
+						if val != v {
+							t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val)
+						}
+					case uint64:
+						val := column.GetUint64Val()
+						if val != v {
+							t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val)
+						}
+					case bool:
+						val := column.GetBoolVal()
+						if val != v {
+							t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val)
+						}
+					case int64:
+						val := column.GetInt64Val()
+						if val != v {
+							t.Fatalf("test %d expected column val: %v but got: %v", ti, v, val)
+						}
+					case float64:
+						val := column.GetFloat64Val()
+						if val != v {
+							t.Fatalf("test %d expected column val: %v but got: %v", ti, v, val)
+						}
+					default:
+						t.Fatalf("test %d has unhandled data type: %T", ti, v)
+					}
+				}
+
+				cnt++
+			}
+		}
+	})
+}

server/handler_test.go

@@ -224,7 +224,12 @@ func TestHandler_Endpoints(t *testing.T) {
 		if err != nil {
 			t.Fatalf("querying: %v", err)
 		}
-		if !reflect.DeepEqual(resp.Results[0], []pilosa.Pair{{Count: 12, ID: 0}}) {
+		if !reflect.DeepEqual(resp.Results[0], &pilosa.PairsField{
+			Pairs: []pilosa.Pair{
+				{Count: 12, ID: 0},
+			},
+			Field: "f1",
+		}) {
 			t.Fatalf("Unexpected result %v", resp.Results[0])
 		}
 
@@ -487,7 +492,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)))
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
-		} else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" {
+		} else if body := w.Body.String(); body != `{"results":[[{"id":30,"key":"","count":3},{"id":31,"key":"","count":1}]]}`+"\n" {
 			t.Fatalf("unexpected body: %q", body)
 		}
 	})
@@ -504,8 +509,8 @@ func TestHandler_Endpoints(t *testing.T) {
 		var resp pilosa.QueryResponse
 		if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
 			t.Fatal(err)
-		} else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 {
-			t.Fatalf("unexpected pair length: %d", len(a))
+		} else if a := resp.Results[0].(*pilosa.PairsField); len(a.Pairs) != 2 {
+			t.Fatalf("unexpected pair length: %d", len(a.Pairs))
 		}
 	})
 
@@ -636,7 +641,6 @@ func TestHandler_Endpoints(t *testing.T) {
 		fieldName := "f-int-ubound-err"
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName),
 			strings.NewReader(`{"options":{"type":"int", "min": 10, "max": -10}}`)))
-		fmt.Println("body", w.Body.String())
 		if w.Code != gohttp.StatusBadRequest {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
@@ -986,7 +990,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
-		target := []uint64{1, 2, 3}
+		target := []uint64{162529281, 159383553, 160432129}
 		resp := pilosa.TranslateKeysResponse{}
 		err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp)
 		if err != nil {
@@ -1026,13 +1030,34 @@ func TestHandler_Endpoints(t *testing.T) {
 	})
 }
 
-func TestClusterTranslator(t *testing.T) {
-	cluster := make(test.Cluster, 2)
-	cluster[0] = test.NewCommandNode(true)
+func TestCluster_TranslateStore(t *testing.T) {
+	cluster := make(test.Cluster, 1)
+	cluster[0] = test.NewCommandNode(true,
+		server.OptCommandServerOptions(
+			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+		),
+	)
 	cluster[0].Config.Gossip.Port = "0"
 	err := cluster[0].Start()
 	if err != nil {
-		t.Fatalf("starting cluster 1: %v", err)
+		t.Fatalf("starting cluster 0: %v", err)
+	}
+
+	test.MustDo("POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+}
+
+func TestClusterTranslator(t *testing.T) {
+	cluster := make(test.Cluster, 2)
+	cluster[0] = test.NewCommandNode(true,
+		server.OptCommandServerOptions(
+			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+		),
+	)
+	cluster[0].Config.Gossip.Port = "0"
+	err := cluster[0].Start()
+	if err != nil {
+		t.Fatalf("starting cluster 0: %v", err)
 	}
 	cluster[1] = test.NewCommandNode(false,
 		server.OptCommandServerOptions(

server/server.go

@@ -80,9 +80,11 @@ type Command struct {
 	logger    loggerLogger
 
 	Handler      pilosa.Handler
+	grpcServer   *grpcServer
 	API          *pilosa.API
 	ln           net.Listener
 	listenURI    *pilosa.URI
+	tlsConfig    *tls.Config
 	closeTimeout time.Duration
 
 	serverOptions []pilosa.ServerOption
@@ -135,7 +137,6 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption
 
 // Start starts the pilosa server - it returns once the server is running.
 func (m *Command) Start() (err error) {
-
 	// Seed random number generator
 	rand.Seed(time.Now().UTC().UnixNano())
 
@@ -163,6 +164,11 @@ func (m *Command) Start() (err error) {
 	}
 
 	m.logger.Printf("listening as %s\n", m.listenURI)
+	go func() {
+		if err := m.grpcServer.Serve(m.tlsConfig); err != nil {
+			m.logger.Printf("grpc server error: %v", err)
+		}
+	}()
 
 	close(m.Started)
 	return nil
@@ -223,8 +229,8 @@ func (m *Command) SetupServer() error {
 	runtime.SetBlockProfileRate(m.Config.Profile.BlockRate)
 	runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction)
 
-	syswrap.SetMaxMapCount(m.Config.MaxMapCount)
-	syswrap.SetMaxFileCount(m.Config.MaxFileCount)
+	_ = syswrap.SetMaxMapCount(m.Config.MaxMapCount)
+	_ = syswrap.SetMaxFileCount(m.Config.MaxFileCount)
 
 	err := m.setupLogger()
 	if err != nil {
@@ -251,10 +257,14 @@ func (m *Command) SetupServer() error {
 		return errors.Wrap(err, "processing bind address")
 	}
 
+	grpcURI, err := pilosa.NewURIFromAddress(m.Config.BindGRPC)
+	if err != nil {
+		return errors.Wrap(err, "processing bind grpc address")
+	}
+
 	// Setup TLS
-	var TLSConfig *tls.Config
 	if uri.Scheme == "https" {
-		TLSConfig, err = GetTLSConfig(&m.Config.TLS, m.logger.Logger())
+		m.tlsConfig, err = GetTLSConfig(&m.Config.TLS, m.logger.Logger())
 		if err != nil {
 			return errors.Wrap(err, "get tls config")
 		}
@@ -270,7 +280,7 @@ func (m *Command) SetupServer() error {
 		return errors.Wrap(err, "new stats client")
 	}
 
-	m.ln, err = getListener(*uri, TLSConfig)
+	m.ln, err = getListener(*uri, m.tlsConfig)
 	if err != nil {
 		return errors.Wrap(err, "getting listener")
 	}
@@ -283,7 +293,7 @@ func (m *Command) SetupServer() error {
 	// Save listenURI for later reference.
 	m.listenURI = uri
 
-	c := http.GetHTTPClient(TLSConfig)
+	c := http.GetHTTPClient(m.tlsConfig)
 
 	// Get advertise address as uri.
 	advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise)
@@ -351,7 +361,16 @@ func (m *Command) SetupServer() error {
 		http.OptHandlerListener(m.ln),
 		http.OptHandlerCloseTimeout(m.closeTimeout),
 	)
-	return errors.Wrap(err, "new handler")
+	if err != nil {
+		return errors.Wrap(err, "new handler")
+	}
+
+	m.grpcServer, err = NewGRPCServer(
+		OptGRPCServerAPI(m.API),
+		OptGRPCServerURI(grpcURI),
+		OptGRPCServerLogger(m.logger),
+	)
+	return errors.Wrap(err, "new grpc server")
 }
 
 // setupNetworking sets up internode communication based on the configuration.

server/server_test.go

@@ -27,7 +27,6 @@ import (
 	"strconv"
 	"strings"
 	"testing"
-	"testing/quick"
 	"time"
 
 	"github.com/pelletier/go-toml"
@@ -51,78 +50,77 @@ func TestMain_Set_Quick(t *testing.T) {
 		t.Skip("short")
 	}
 
-	if err := quick.Check(func(cmds []SetCommand) bool {
-		m := test.MustRunCommand()
-		defer m.Close()
+	for i := 0; i < 100; i++ {
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			t.Parallel()
 
-		// Create client.
-		client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
-		if err != nil {
-			t.Fatal(err)
-		}
+			rand := rand.New(rand.NewSource(int64(i)))
+			cmds := GenerateSetCommands(1000, rand)
 
-		// Execute Set() commands.
-		for _, cmd := range cmds {
-			if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+			m := test.MustRunCommand()
+			defer m.Close()
+
+			// Create client.
+			client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
+			if err != nil {
 				t.Fatal(err)
 			}
-			if err := client.CreateField(context.Background(), "i", cmd.Field); err != nil && err != pilosa.ErrFieldExists {
-				t.Fatal(err)
-			}
-			if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil {
-				t.Fatal(err)
-			}
-		}
 
-		// Validate data.
-		for field, fieldSet := range SetCommands(cmds).Fields() {
-			for id, columnIDs := range fieldSet {
-				exp := MustMarshalJSON(map[string]interface{}{
-					"results": []interface{}{
-						map[string]interface{}{
-							"columns": columnIDs,
-							"attrs":   map[string]interface{}{},
-						},
-					},
-				}) + "\n"
-				if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
+			// Execute Set() commands.
+			for _, cmd := range cmds {
+				if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+					t.Fatal(err)
+				}
+				if err := client.CreateField(context.Background(), "i", cmd.Field); err != nil && err != pilosa.ErrFieldExists {
+					t.Fatal(err)
+				}
+				if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil {
 					t.Fatal(err)
-				} else if res != exp {
-					t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
 				}
 			}
-		}
 
-		if err := m.Reopen(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Validate data after reopening.
-		for field, fieldSet := range SetCommands(cmds).Fields() {
-			for id, columnIDs := range fieldSet {
-				exp := MustMarshalJSON(map[string]interface{}{
-					"results": []interface{}{
-						map[string]interface{}{
-							"columns": columnIDs,
-							"attrs":   map[string]interface{}{},
+			// Validate data.
+			for field, fieldSet := range SetCommands(cmds).Fields() {
+				for id, columnIDs := range fieldSet {
+					exp := MustMarshalJSON(map[string]interface{}{
+						"results": []interface{}{
+							map[string]interface{}{
+								"columns": columnIDs,
+								"attrs":   map[string]interface{}{},
+							},
 						},
-					},
-				}) + "\n"
-				if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
-					t.Fatal(err)
-				} else if res != exp {
-					t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
+					}) + "\n"
+					if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
+						t.Fatal(err)
+					} else if res != exp {
+						t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
+					}
 				}
 			}
-		}
 
-		return true
-	}, &quick.Config{
-		Values: func(values []reflect.Value, rand *rand.Rand) {
-			values[0] = reflect.ValueOf(GenerateSetCommands(1000, rand))
-		},
-	}); err != nil {
-		t.Fatal(err)
+			if err := m.Reopen(); err != nil {
+				t.Fatal(err)
+			}
+
+			// Validate data after reopening.
+			for field, fieldSet := range SetCommands(cmds).Fields() {
+				for id, columnIDs := range fieldSet {
+					exp := MustMarshalJSON(map[string]interface{}{
+						"results": []interface{}{
+							map[string]interface{}{
+								"columns": columnIDs,
+								"attrs":   map[string]interface{}{},
+							},
+						},
+					}) + "\n"
+					if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
+						t.Fatal(err)
+					} else if res != exp {
+						t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
+					}
+				}
+			}
+		})
 	}
 }
 
@@ -300,6 +298,41 @@ func TestMain_GroupBy(t *testing.T) {
 	}
 }
 
+func TestMain_MinMaxFloat(t *testing.T) {
+	m := test.MustRunCommand()
+	defer m.Close()
+
+	// Create fields.
+	client := m.Client()
+	if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+		t.Fatal(err)
+	}
+	if err := client.CreateFieldWithOptions(context.Background(), "i", "dec", pilosa.FieldOptions{Type: pilosa.FieldTypeDecimal, Scale: 3, Max: 100000}); err != nil {
+		t.Fatal(err)
+	}
+
+	query := `
+		Set(0, dec=1.32)
+		Set(1, dec=4.44)
+	`
+
+	// Set columns on row.
+	if _, err := m.Query("i", "", query); err != nil {
+		t.Fatal(err)
+	}
+
+	// Query row.
+	exp0 := pilosa.ValCount{FloatVal: 4.44, Count: 1}
+	exp1 := pilosa.ValCount{FloatVal: 1.32, Count: 1}
+	if res, err := m.QueryProtobuf("i", `Max(field=dec) Min(field=dec)`); err != nil {
+		t.Fatal(err)
+	} else if res.Results[0] != exp0 ||
+		res.Results[1] != exp1 {
+		t.Fatalf("unexpected results: %+v", res.Results)
+
+	}
+}
+
 // Ensure the host can be parsed.
 func TestConfig_Parse_Host(t *testing.T) {
 	if c, err := ParseConfig(`bind = "local"`); err != nil {
@@ -373,7 +406,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
 		t.Fatalf("recalculating caches: %v", err)
 	}
 
-	target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}`
+	target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}`
 
 	// Run a TopN query on all nodes. The result should be the same as the target.
 	for _, m := range cluster {
@@ -497,7 +530,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr)
+	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	cluster[2].Command.Config = config
 
 	// Run new program.
@@ -571,7 +604,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr)
+	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	cluster[2].Command.Config = config
 
 	// Run new program.
@@ -591,7 +624,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port))
 
-	cluster[1].Command = server.NewCommand(cluster[1].Stdin, cluster[1].Stdout, cluster[1].Stderr)
+	cluster[1].Command = server.NewCommand(cluster[1].Stdin, cluster[1].Stdout, cluster[1].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	cluster[1].Command.Config = config
 
 	// Run new program.
@@ -864,7 +897,7 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port))
-	cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr)
+	cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	cmd1.Command.Config = config
 	err = cmd1.Start()
 	if err != nil {
@@ -885,6 +918,7 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 	if results.Results[0].(uint64) != 100 {
 		t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
 	}
+
 }
 
 // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
@@ -985,3 +1019,45 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 		t.Fatalf("setting lots of shards: %v", err)
 	}
 }
+
+func TestClusterMinMaxSumDecimal(t *testing.T) {
+	cluster := test.MustRunCluster(t, 3)
+	defer cluster.Close()
+	cmd := cluster[0]
+
+	cmd.MustCreateIndex(t, "testdec", pilosa.IndexOptions{Keys: true, TrackExistence: true})
+	cmd.MustCreateField(t, "testdec", "adec", pilosa.OptFieldTypeDecimal(2))
+
+	test.MustDo("POST", cluster[0].URL()+"/index/testdec/query", `
+Set("a", adec=42.2)
+Set("b", adec=11.12)
+Set("c", adec=13.41)
+Set("d", adec=99.87)
+Set("e", adec=11.13)
+Set("f", adec=12.12)
+Set("g", adec=15.52)
+Set("h", adec=100.22)
+`)
+
+	result := test.MustDo("POST", cluster[0].URL()+"/index/testdec/query", "Sum(field=adec)")
+	if !strings.Contains(result.Body, `"floatValue":305.59`) {
+		t.Fatalf("expected float sum of 305.59, but got: '%s'", result.Body)
+	} else if !strings.Contains(result.Body, `"count":8`) {
+		t.Fatalf("expected count 8, but got: '%s'", result.Body)
+	}
+
+	result = test.MustDo("POST", cluster[0].URL()+"/index/testdec/query", "Max(field=adec)")
+	if !strings.Contains(result.Body, `"floatValue":100.22`) {
+		t.Fatalf("expected float max of 100.22, but got: '%s'", result.Body)
+	} else if !strings.Contains(result.Body, `"count":1`) {
+		t.Fatalf("expected count 1, but got: '%s'", result.Body)
+	}
+
+	result = test.MustDo("POST", cluster[0].URL()+"/index/testdec/query", "Min(field=adec)")
+	if !strings.Contains(result.Body, `"floatValue":11.12`) {
+		t.Fatalf("expected float min of 11.12, but got: '%s'", result.Body)
+	} else if !strings.Contains(result.Body, `"count":1`) {
+		t.Fatalf("expected count 1, but got: '%s'", result.Body)
+	}
+
+}

snapshotqueue.go

@@ -0,0 +1,406 @@
+// Copyright 2019 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pkg/errors"
+)
+
+// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot
+// queue distinguishes between high-priority requests, which get satisfied
+// by the next available worker, and regular requests, which get enqueued
+// if there's space in the queue, and otherwise dropped. There's also a
+// separate background task to scan a holder for fragments which may need
+// snapshots, but which is processed only when the queue is empty, and only
+// slowly. "Await" awaits an existing snapshot if one is already enqueued.
+// "Immediate" tries to do one right away. (If one's already enqueued, this
+// can leave it in the queue, which will ignore anything that shows up with
+// the request flag cleared.)
+//
+// Await, Enqueue, and Immediate should be called only with the fragment lock
+// held.
+//
+// ScanHolder spawns a new goroutine. You don't need to use `go` on it.
+type snapshotQueue interface {
+	Immediate(*fragment) error
+	Enqueue(*fragment)
+	Await(*fragment) error
+	ScanHolder(*Holder)
+	Stop()
+}
+
+// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the
+// interface.
+type queuelessSnapshotQueue struct{}
+
+func (q *queuelessSnapshotQueue) Enqueue(f *fragment) {
+	_ = f.snapshot()
+}
+
+func (q *queuelessSnapshotQueue) Await(f *fragment) error {
+	return nil
+}
+
+func (q *queuelessSnapshotQueue) Immediate(f *fragment) error {
+	return f.snapshot()
+}
+
+func (q *queuelessSnapshotQueue) ScanHolder(h *Holder) {
+}
+
+func (q *queuelessSnapshotQueue) Stop() {
+}
+
+// defaultSnapshotQueue is the fallback to use if none is available,
+// and currently uses queueless -- it runs all snapshots immediately.
+var defaultSnapshotQueue *queuelessSnapshotQueue
+
+// newSnapshotQueue makes a new snapshot queue, of depth N, with
+// w worker threads.
+func newSnapshotQueue(n int, w int, l logger.Logger) snapshotQueue {
+	sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), logger: l}
+	if sq.logger == nil {
+		sq.logger = logger.NewStandardLogger(os.Stderr)
+	}
+	sq.spawnWorkers(w)
+	return &sq
+}
+
+type snapshotRequest struct {
+	frag *fragment
+	when time.Time
+}
+
+// prioritySnapshotQueue gives preference to "immediate" requests, and
+// dispreference to "background" requests from ScanHolder. It timestamps
+// requests, so it can discard a request if the most recent snapshot is
+// newer than the request. The snapshotPending flag in the fragment is
+// used to track that a given fragment thinks it has been successfully
+// enqueued. Background requests are not considered enqueued, since
+// they'll never get processed if there's anything else. In normal workloads,
+// immediate/urgent snapshots should be rare, but we'll happily drop
+// most requests on the floor; the scanner should pick them up once things
+// are quiet.
+type prioritySnapshotQueue struct {
+	logger           logger.Logger
+	urgent           chan snapshotRequest
+	normal           chan snapshotRequest
+	background       chan snapshotRequest
+	done             chan struct{}
+	mu               sync.RWMutex
+	scanWG, workerWG sync.WaitGroup
+	stats            struct {
+		enqueued uint64
+		skipped  uint64
+	}
+}
+
+func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
+	sq.mu.Lock()
+	defer sq.mu.Unlock()
+	if sq.done == nil {
+		sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?")
+		return
+	}
+	sq.workerWG.Add(w)
+	for i := 0; i < w; i++ {
+		go sq.worker(sq.urgent, sq.normal, sq.background, sq.done)
+	}
+}
+
+func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) {
+	// We don't want a race condition on these. If they're non-nil when
+	// we get them, they should get closed at some point. If done is
+	// already nil, we shouldn't do anything.
+	defer sq.workerWG.Done()
+	ok := true
+	var req snapshotRequest
+	for ok {
+		req.frag = nil
+
+		select {
+		case req, ok = <-urgent:
+		default:
+			select {
+			case req, ok = <-urgent:
+			case req, ok = <-normal:
+			default:
+				select {
+				case req, ok = <-urgent:
+				case req, ok = <-normal:
+				case req, ok = <-background:
+				case _, ok = <-done:
+				}
+			}
+		}
+		if req.frag != nil {
+			sq.process(req)
+		}
+	}
+}
+
+// process actually runs a fragment. it will do this if either the fragment
+// has a pending snapshot, or the force flag is set.
+func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
+	f := req.frag
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.snapshotStamp.Before(req.when) {
+		f.snapshotErr = f.snapshot()
+		if f.snapshotErr != nil {
+			fmt.Printf("snapshot error: %v\n", f.snapshotErr)
+			sq.logger.Printf("snapshot error: %v", f.snapshotErr)
+		}
+		f.snapshotPending = false
+		f.snapshotCond.Broadcast()
+	}
+}
+
+// Stop shuts down the snapshot queue. It first marks it as done, causing
+// the background scanner(s), if any, to shut down, then waits for them, then
+// closes and nils the queues. The background scanner has to get stopped
+// because otherwise it might try to write to those closed queues.
+func (sq *prioritySnapshotQueue) Stop() {
+	sq.mu.Lock()
+	defer sq.mu.Unlock()
+	close(sq.done)
+	// scanners need to be done before we close the other channels.
+	sq.scanWG.Wait()
+	sq.done = nil
+	close(sq.normal)
+	sq.normal = nil
+	close(sq.urgent)
+	sq.urgent = nil
+	close(sq.background)
+	sq.background = nil
+	sq.logger.Printf("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped)
+}
+
+// Enqueue tries to add a fragment to the queue, if the fragment is not already
+// enqueued. You should hold a lock on the fragment when calling this.
+func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
+	if f.snapshotPending {
+		return
+	}
+	sq.mu.RLock()
+	defer sq.mu.RUnlock()
+	if sq.normal == nil {
+		sq.logger.Printf("requested snapshot after snapshot queue was closed")
+		return
+	}
+	// we have to set this before enqueing, because it's
+	// otherwise possible that we're at the head of the queue,
+	// and the recipient gets the fragment before we execute the
+	// line after the send.
+	f.snapshotPending = true
+	// try to enqueue snapshot
+	select {
+	case sq.normal <- snapshotRequest{frag: f, when: time.Now()}:
+		atomic.AddUint64(&sq.stats.enqueued, 1)
+		return
+	default:
+		atomic.AddUint64(&sq.stats.skipped, 1)
+		f.snapshotPending = false
+		return
+	}
+}
+
+// Await returns when f is not pending a snapshot. Call with the fragment lock
+// held. Await waits on a condition variable inside f, associated with the
+// fragment's lock, so this does not conflict with the lock being used for
+// snapshots.
+func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) {
+	for f.snapshotPending {
+		f.snapshotCond.Wait()
+	}
+	err, f.snapshotErr = f.snapshotErr, nil
+	return err
+}
+
+// Immediate forces an immediate snapshot of the given fragment. Call with
+// the fragment locked. If the queue is already closing, the fragment does
+// not get snapshotted.
+func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
+	sq.mu.RLock()
+	// no deferred unlock, because we want to unlock this before calling Await.
+	// Not because that needs this lock, but because once we're that far, we
+	// *don't* need this lock anymore so someone else should have it.
+	if sq.urgent == nil {
+		sq.mu.RUnlock()
+		sq.logger.Printf("requested immediate snapshot after snapshot queue was closed")
+		return errors.New("requested immediate snapshot after snapshot queue was closed")
+	}
+	f.snapshotPending = true
+	req := snapshotRequest{frag: f, when: time.Now()}
+	// if the fragment was already in the work queue, it's *possible*
+	// that the only available worker just picked it off the queue, and
+	// is now waiting on getting the fragment's lock, so it can run
+	// a snapshot. So we let go of the lock on the fragment, send the
+	// request, then request the fragment lock again, because Await will
+	// be sleeping on the condition variable associated with the lock,
+	// which means it needs to hold the lock so it can let it go during
+	// the wait... No, really, this made sense.
+	f.mu.Unlock()
+	sq.urgent <- req
+	sq.mu.RUnlock()
+	f.mu.Lock()
+	return sq.Await(f)
+}
+
+// needsSnapshot determines whether a fragment probably wants snapshotting.
+// Specifically, it looks for fragments not already marked to receive
+// snapshots, but which have a high enough opN to justify a snapshot. This
+// is only used from the background scan.
+func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool {
+	if f == nil {
+		return false
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.snapshotPending {
+		return false
+	}
+	if f.opN > f.MaxOpN {
+		return true
+	}
+	return false
+}
+
+// ScanHolder spawns a goroutine which iterates through the holder's
+// indexes/fields/views/fragments, looking for fragments which have OpN
+// high enough to justify a snapshot but don't seem to have one pending.
+// It then dumps these in the low priority background queue.
+func (sq *prioritySnapshotQueue) ScanHolder(h *Holder) {
+	sq.mu.Lock()
+	sq.scanWG.Add(1)
+	go sq.scanHolderWorker(h, sq.background, sq.done)
+	sq.mu.Unlock()
+}
+
+// scanHolderWorker is a background task that scans a holder looking for
+// fragments which need snapshots taken. It's the cleanup task for snapshots
+// that would have been requested by Enqueue, but the queue was full.
+func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) {
+	defer sq.scanWG.Done()
+	var indexNames, fieldNames, viewNames []string
+	var fragNums []uint64
+	for {
+		// To avoid abusing things, cap activity rate; every time we finish
+		// the holder, or every couple hundred fragments considered, we
+		// pause for a bit.
+		counter := 0
+		hits := 0
+		h.mu.Lock()
+		indexNames = indexNames[:0]
+		for indexName := range h.indexes {
+			indexNames = append(indexNames, indexName)
+		}
+		h.mu.Unlock()
+		for _, indexName := range indexNames {
+			h.mu.Lock()
+			index := h.indexes[indexName]
+			h.mu.Unlock()
+			if index == nil {
+				continue
+			}
+			fieldNames = fieldNames[:0]
+			index.mu.Lock()
+			for fieldName := range index.fields {
+				fieldNames = append(fieldNames, fieldName)
+			}
+			index.mu.Unlock()
+			for _, fieldName := range fieldNames {
+				index.mu.Lock()
+				field := index.fields[fieldName]
+				index.mu.Unlock()
+				if field == nil {
+					continue
+				}
+				viewNames = viewNames[:0]
+				field.mu.Lock()
+				for viewName := range field.viewMap {
+					viewNames = append(viewNames, viewName)
+				}
+				field.mu.Unlock()
+				for _, viewName := range viewNames {
+					field.mu.Lock()
+					view := field.viewMap[viewName]
+					field.mu.Unlock()
+					if view == nil {
+						continue
+					}
+					fragNums := fragNums[:0]
+					view.mu.Lock()
+					for fragNum := range view.fragments {
+						fragNums = append(fragNums, fragNum)
+					}
+					view.mu.Unlock()
+					for _, fragNum := range fragNums {
+						view.mu.Lock()
+						frag := view.fragments[fragNum]
+						view.mu.Unlock()
+						if sq.needsSnapshot(frag) {
+							hits++
+							select {
+							case background <- snapshotRequest{frag: frag, when: time.Now()}:
+								sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path)
+							case <-done:
+								return
+							}
+						} else {
+							// Count fragments examined *without* finding anything that
+							// needed a snapshot. When we find things that need snapshots,
+							// the time it takes the workers to respond to us is enough
+							// of a delay to keep us from eating every CPU. So, if a lot
+							// of things need snapshots, and the workers aren't doing
+							// anything else, ScanHolder will mostly keep them saturated.
+							// If they're busy, we'll block forever in the write to the
+							// background queue. If there's nothing that needs snapshots,
+							// we pause frequently for a second or so at a time.
+							counter++
+							if counter == 100 {
+								select {
+								case <-time.After(1 * time.Second):
+								case <-done:
+									return
+								}
+								counter = 0
+							}
+						}
+					}
+				}
+			}
+		}
+		if hits > 0 {
+			sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits)
+			hits = 0
+		} else {
+			sq.logger.Debugf("background scan: no fragments needed snapshots, waiting\n")
+			// No reason to be active if we're not finding anything.
+			select {
+			case <-time.After(60 * time.Second):
+			case <-done:
+				return
+			}
+		}
+	}
+}

stats/stats.go

@@ -25,7 +25,7 @@ import (
 )
 
 // Expvar global expvar map.
-var Expvar = expvar.NewMap("index")
+var Expvar *expvar.Map
 
 // StatsClient represents a client to a stats server.
 type StatsClient interface {
@@ -90,6 +90,9 @@ type expvarStatsClient struct {
 // NewExpvarStatsClient returns a new instance of ExpvarStatsClient.
 // This client points at the root of the expvar index map.
 func NewExpvarStatsClient() *expvarStatsClient {
+	if Expvar == nil {
+		Expvar = expvar.NewMap("index")
+	}
 	return &expvarStatsClient{
 		m: Expvar,
 	}

syswrap/mmap.go

@@ -34,10 +34,13 @@ var ErrMaxMapCountReached = errors.New("maximum map count reached")
 var maxMapCount uint64 = 60000
 var mu sync.RWMutex
 
-func SetMaxMapCount(max uint64) {
+// SetMaxMapCount sets the maximum map count, and returns the previous maximum.
+func SetMaxMapCount(max uint64) uint64 {
 	mu.Lock()
+	prev := maxMapCount
 	maxMapCount = max
 	mu.Unlock()
+	return prev
 }
 
 // Mmap increments the global map count, and then calls syscall.Mmap. It

syswrap/os.go

@@ -27,10 +27,12 @@ var fileCount uint64
 var maxFileCount uint64 = 500000
 var fileMu sync.RWMutex
 
-func SetMaxFileCount(max uint64) {
+func SetMaxFileCount(max uint64) uint64 {
 	fileMu.Lock()
+	prev := maxFileCount
 	maxFileCount = max
 	fileMu.Unlock()
+	return prev
 }
 
 // OpenFile passes the arguments along to os.OpenFile while incrementing a

test/holder.go

@@ -16,6 +16,7 @@ package test
 
 import (
 	"io/ioutil"
+	"math"
 	"os"
 	"time"
 
@@ -35,7 +36,7 @@ func NewHolder() *Holder {
 		panic(err)
 	}
 
-	h := &Holder{Holder: pilosa.NewHolder()}
+	h := &Holder{Holder: pilosa.NewHolder(pilosa.DefaultPartitionN)}
 	h.Path = path
 	h.Holder.NewAttrStore = boltdb.NewAttrStore
 
@@ -61,7 +62,7 @@ func (h *Holder) Close() error {
 // Note that the holder must be Closed first.
 func (h *Holder) Reopen() error {
 	path, logger := h.Path, h.Holder.Logger
-	h.Holder = pilosa.NewHolder()
+	h.Holder = pilosa.NewHolder(pilosa.DefaultPartitionN)
 	h.Holder.Path = path
 	h.Holder.Logger = logger
 	h.Holder.NewAttrStore = boltdb.NewAttrStore
@@ -165,3 +166,16 @@ func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uin
 		h.SetBit(index, field, rowID, columnID)
 	}
 }
+
+// SetValue sets an value on the given field.
+func (h *Holder) SetValue(index, field string, columnID uint64, value int64) {
+	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
+	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+	if err != nil {
+		panic(err)
+	}
+	_, err = f.SetValue(columnID, value)
+	if err != nil {
+		panic(err)
+	}
+}

test/index.go

@@ -32,7 +32,7 @@ func newIndex() *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(path, "i")
+	index, err := pilosa.NewIndex(path, "i", pilosa.DefaultPartitionN)
 	if err != nil {
 		panic(err)
 	}
@@ -62,7 +62,7 @@ func (i *Index) Reopen() error {
 	}
 
 	path, name := i.Path(), i.Name()
-	i.Index, err = pilosa.NewIndex(path, name)
+	i.Index, err = pilosa.NewIndex(path, name, pilosa.DefaultPartitionN)
 	if err != nil {
 		return err
 	}

test/logger.go

@@ -18,12 +18,14 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"sync"
 )
 
 // bufferLogger represents a test Logger that holds log messages
 // in a buffer for review.
 type bufferLogger struct {
 	buf *bytes.Buffer
+	mu  sync.Mutex
 }
 
 // NewBufferLogger returns a new instance of BufferLogger.
@@ -34,6 +36,8 @@ func NewBufferLogger() *bufferLogger {
 }
 
 func (b *bufferLogger) Printf(format string, v ...interface{}) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
 	s := fmt.Sprintf(format, v...)
 	_, err := b.buf.WriteString(s)
 	if err != nil {
@@ -44,5 +48,7 @@ func (b *bufferLogger) Printf(format string, v ...interface{}) {
 func (b *bufferLogger) Debugf(format string, v ...interface{}) {}
 
 func (b *bufferLogger) ReadAll() ([]byte, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
 	return ioutil.ReadAll(b.buf)
 }

test/pilosa.go

@@ -73,6 +73,9 @@ func newCommand(opts ...server.CommandOption) *Command {
 	if m.Config.Bind == defaultConf.Bind {
 		m.Config.Bind = "http://localhost:0"
 	}
+	if m.Config.BindGRPC == defaultConf.BindGRPC {
+		m.Config.BindGRPC = "http://localhost:0"
+	}
 	m.Config.Translation.MapSize = 140000
 	m.Config.WorkerPoolSize = 2
 
@@ -86,6 +89,10 @@ func newCommand(opts ...server.CommandOption) *Command {
 
 // NewCommandNode returns a new instance of Command with clustering enabled.
 func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
+	// We want tests to default to using the in-memory translate store, so we
+	// prepend opts with that functional option. If a different translate store
+	// has been specified, it will override this one.
+	opts = prependWithMemStore(opts)
 	m := newCommand(opts...)
 	m.Config.Cluster.Disabled = false
 	m.Config.Cluster.Coordinator = isCoordinator
@@ -94,7 +101,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
 
 // MustRunCommand returns a new, running Main. Panic on error.
 func MustRunCommand() *Command {
-	m := newCommand()
+	m := newCommand(server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
 	m.Config.Gossip.Port = "0"
 	if err := m.Start(); err != nil {
@@ -131,9 +138,24 @@ func (m *Command) Reopen() error {
 	return m.Start()
 }
 
+// SoftOpen is like Reopen, but doesn't close the program first.
+// This is useful in the case where a test needs to decouple the
+// close from the re-open (for example, there may need to be
+// actions which take place between those two steps).
+func (m *Command) SoftOpen() error {
+	// Create new main with the same config.
+	config := m.Command.Config
+	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...)
+	m.Command.Config = config
+
+	// Run new program.
+	return m.Start()
+}
+
 // MustCreateIndex uses this command's API to create an index and fails the test
 // if there is an error.
 func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexOptions) *pilosa.Index {
+	tb.Helper()
 	idx, err := m.API.CreateIndex(context.Background(), name, opts)
 	if err != nil {
 		tb.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err)
@@ -144,6 +166,7 @@ func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexO
 // MustCreateField uses this command's API to create the field. The index must
 // already exist - it fails the test if there is an error.
 func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pilosa.FieldOption) *pilosa.Field {
+	tb.Helper()
 	f, err := m.API.CreateField(context.Background(), index, field, opts...)
 	if err != nil {
 		tb.Fatalf("creating field: %s in index: %s err: %v", field, index, err)
@@ -154,6 +177,7 @@ func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pi
 // MustQuery uses this command's API to execute the given query request, failing
 // if Query returns a non-nil error, otherwise returning the QueryResponse.
 func (m *Command) MustQuery(tb testing.TB, req *pilosa.QueryRequest) pilosa.QueryResponse {
+	tb.Helper()
 	resp, err := m.API.Query(context.Background(), req)
 	if err != nil {
 		tb.Fatalf("making query: %v, err: %v", req, err)
@@ -245,6 +269,7 @@ type Cluster []*Command
 // Query executes an API.Query through one of the cluster's node's API. It fails
 // the test if there is an error.
 func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
+	t.Helper()
 	if len(c) == 0 {
 		t.Fatal("must have at least one node in cluster to query")
 	}
@@ -253,6 +278,7 @@ func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
 }
 
 func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
+	t.Helper()
 	byShard := make(map[uint64][][2]uint64)
 	for _, rowcol := range rowcols {
 		shard := rowcol[1] / pilosa.ShardWidth
@@ -293,6 +319,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 
 // CreateField creates the index (if necessary) and field specified.
 func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
+	t.Helper()
 	idx, err := c[0].API.CreateIndex(context.Background(), index, iopts)
 	if err != nil && !strings.Contains(err.Error(), "index already exists") {
 		t.Fatalf("creating index: %v", err)
@@ -341,6 +368,7 @@ func (c Cluster) Close() error {
 
 // MustNewCluster creates a new cluster
 func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+	tb.Helper()
 	c, err := newCluster(size, opts...)
 	if err != nil {
 		tb.Fatalf("new cluster: %v", err)
@@ -388,6 +416,12 @@ func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 
 // MustRunCluster creates and starts a new cluster
 func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+	// We want tests to default to using the in-memory translate store, so we
+	// prepend opts with that functional option. If a different translate store
+	// has been specified, it will override this one.
+	opts = prependOpts(opts)
+
+	tb.Helper()
 	c, err := runCluster(size, opts...)
 	if err != nil {
 		tb.Fatalf("run cluster: %v", err)
@@ -395,6 +429,29 @@ func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
 	return c
 }
 
+// prependOpts applies prependWithMemStore to each of the ops (one per
+// node, or one for the entire cluser).
+func prependOpts(opts [][]server.CommandOption) [][]server.CommandOption {
+	if len(opts) == 0 {
+		opts = [][]server.CommandOption{
+			prependWithMemStore([]server.CommandOption{}),
+		}
+	} else {
+		for i := range opts {
+			opts[i] = prependWithMemStore(opts[i])
+		}
+	}
+	return opts
+}
+
+// prependWithMemStore prepends opts with the OpenInMemTranslateStore.
+func prependWithMemStore(opts []server.CommandOption) []server.CommandOption {
+	defaultOpts := []server.CommandOption{
+		server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)),
+	}
+	return append(defaultOpts, opts...)
+}
+
 ////////////////////////////////////////////////////////////////////////////////////
 
 // MustDo executes http.Do() with an http.NewRequest(). Panic on error.
@@ -426,6 +483,7 @@ func MustDo(method, urlStr string, body string) *httpResponse {
 }
 
 func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) {
+	t.Helper()
 	if len(results) != len(expected) {
 		t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected)
 	}