fix tag check to check against null

empty string doesn't work because gitlab doesn't set the variable at
all. How do I know that "null" is correct? Because Fletcher told
me... apparently it's a ruby-ism

.circleci/config.yml

@@ -1,195 +1,305 @@
-version: 2
-defaults: &defaults
-  working_directory: /go/src/github.com/pilosa/pilosa
-  docker:
-    - image: circleci/golang:1.13
-  environment:
-    GO111MODULE: "on"
-fast-checkout: &fast-checkout
-  attach_workspace:
-    at: .
-jobs:
-  setup:
-    <<: *defaults
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - mod-cache-{{ checksum "go.sum" }}
-      - run: "go mod download"
-      - save_cache:
-          key: mod-cache-{{ checksum "go.sum" }}
-          paths:
-            - /go/pkg/mod/
-      - persist_to_workspace:
-          root: .
-          paths: "*"
-  check-license-headers:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: make check-license-headers
-  linter:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
-      - run: sudo cp bin/golangci-lint /usr/local/bin/
-      - run: make golangci-lint
-  test-build-arm:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: make build GOOS=linux GOARCH=arm GOARM=5
-      - run: make build GOOS=linux GOARCH=arm GOARM=6
-      - run: make build GOOS=linux GOARCH=arm GOARM=7
-      - run: make build GOOS=linux GOARCH=arm64
-  test-golang-1.13: &base-test
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.13-shard22:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test SHARD_WIDTH=22
-  test-golang-1.13-race:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run:
-          command: make test TESTFLAGS="-race -v -timeout=30m"
-          no_output_timeout: 30m
-  test-golang-1.13-386:
-    <<: *base-test
-    environment:
-      GO111MODULE: "on"
-      GOARCH: 386
-  test-golang-1.13-enterprise:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test ENTERPRISE=1
-  test-golang-1.12:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.12
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  test-golang-1.11:
-    <<: *defaults
-    docker:
-      - image: circleci/golang:1.11
-    steps:
-      - *fast-checkout
-      - run: sudo apt-get install lsof
-      - run: make test
-  cluster-tests:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - setup_remote_docker
-      - run: make clustertests-build
-  prerelease:
-    <<: *base-test
-    steps:
-      - *fast-checkout
-      - run: make prerelease
-      - store_artifacts:
-          path: build
-      - persist_to_workspace:
-          root: .
-          paths: build
-  release:
-    <<: *defaults
-    steps:
-      - *fast-checkout
-      - run: make release
-      - store_artifacts:
-          path: build
-      - persist_to_workspace:
-          root: .
-          paths: build
-  prerelease-upload:
-    docker:
-      - image: circleci/python:2.7-jessie
-    steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
-      - run: sudo pip install awscli
-      - run: make prerelease-upload
-  dockerhub-upload:
-    <<: *defaults
-    steps:
-      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
-      - *fast-checkout
-      - setup_remote_docker
-      - run: make docker
-      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
-      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker push pilosa/pilosa:master
-workflows:
-  version: 2
-  test:
-    jobs:
-      - setup
-      - linter:
-          requires:
-            - setup
-      - check-license-headers:
-          requires:
-            - setup
-      - test-build-arm:
-          requires:
-            - setup
-      - test-golang-1.13-enterprise:
-          requires:
-            - setup
-      - test-golang-1.13-race:
-          requires:
-            - setup
-      - test-golang-1.13-386:
-          requires:
-            - setup
-      - test-golang-1.13:
-          requires:
-            - setup
-      - test-golang-1.12:
-          requires:
-            - setup
-      - test-golang-1.11:
-          requires:
-            - setup
-      - cluster-tests:
-          requires:
-            - setup
-      - prerelease:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
-      - release:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
-          filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+# version: 2.1
+
+# executors:
+#   golang:
+#     parameters:
+#       version:
+#         type: string
+#         default: "1.15.8"
+#       resource_class:
+#         type: string
+#         default: medium
+#     docker:
+#       - image: circleci/golang:<< parameters.version >>
+#     resource_class: << parameters.resource_class >>
+#     working_directory: /go/src/github.com/molecula/featurebase
+
+# commands:
+#   add-github-auth:
+#     steps:
+#       - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
+#       - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
+#   restore-mod-cache:
+#     steps:
+#       - restore_cache:
+#           key: mod-cache-{{ checksum "go.sum" }}
+#   save-mod-cache:
+#     steps:
+#       - save_cache:
+#           key: mod-cache-{{ checksum "go.sum" }}
+#           paths:
+#             - /go/pkg/mod/
+#   checkout-plus:
+#     steps:
+#       - add-github-auth
+#       - checkout
+#       - restore-mod-cache
+#   skip-if-root-unchanged:
+#     description: "skips the parent job if the PR includes no changes to featurebase"
+#     steps:
+#       - run: |
+#           ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
+#           echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
+#           if [ -z "$ROOT_CHANGED_FILES" ] ; then
+#             echo "halting step"
+#             circleci step halt
+#           fi
+#   skip-if-lattice-unchanged:
+#     description: "skips the parent job if the PR includes no changes to lattice"
+#     steps:
+#       - run: |
+#           LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
+#           echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
+#           if [ -z "$LATTICE_CHANGED_FILES" ] ; then
+#             echo "halting step"
+#             circleci step halt
+#           fi
+
+# jobs:
+#   setup:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - run: go mod download
+#       - save-mod-cache
+#   linter:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
+#       - run: make golangci-lint
+#   go-mod-tidy:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - run: go mod tidy
+#       - run: git diff --exit-code -- go.mod go.sum
+#   check-changelog-label:
+#     executor:
+#       name: golang
+#     steps:
+#       - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
+#       - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
+#   test-build-arm:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - run: make build GOOS=linux GOARCH=arm GOARM=5
+#       - run: make build GOOS=linux GOARCH=arm GOARM=6
+#       - run: make build GOOS=linux GOARCH=arm GOARM=7
+#       - run: make build GOOS=linux GOARCH=arm64
+#   test:
+#     parameters:
+#       resource_class:
+#         type: string
+#         default: medium
+#       golang_version:
+#         type: string
+#         default: "1.15.8"
+#       shard_width:
+#         type: string
+#         default: "20"
+#       test_make_target:
+#         type: string
+#         default: "test"
+#       test_flags:
+#         type: string
+#         default: ""
+#       goarch:
+#         type: string
+#         default: amd64
+#     executor:
+#       name: golang
+#       version: << parameters.golang_version >>
+#       resource_class: << parameters.resource_class >>
+#     environment:
+#       TMPDIR: /mnt/ramdisk
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - run: sudo apt-get update --allow-releaseinfo-change -y
+#       - run: sudo apt-get install lsof
+#       - run:
+#           command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
+#           no_output_timeout: 30m
+#   test-external-lookup:
+#     docker:
+#       - image: circleci/golang:1.15.8
+#       - image: circleci/postgres:13.2-ram
+#         environment:
+#           POSTGRES_PASSWORD=password
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - run: sudo apt-get update --allow-releaseinfo-change -y
+#       - run: sudo apt-get install postgresql-client
+#       - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
+#       - run:
+#           command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
+#           no_output_timeout: 30m
+#   cluster-tests:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - skip-if-root-unchanged
+#       - setup_remote_docker
+#       - run: make clustertests
+#   release:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - attach_workspace:
+#           at: .
+#       - setup_remote_docker:
+#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+#       - run: make docker-release
+#       - store_artifacts:
+#           path: build
+#       - persist_to_workspace:
+#           root: .
+#           paths: build
+#   publish_release:
+#     executor:
+#       name: golang
+#     steps:
+#       - attach_workspace:
+#           at: .
+#       - run: go get github.com/tcnksm/ghr
+#       - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
+#   docker-build:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - setup_remote_docker:
+#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+#       - run: make docker GO_VERSION=1.15.8
+#       - run: docker run featurebase:$(git describe --tags) help
+#   dockerhub-upload-unstable:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - setup_remote_docker:
+#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+#       - run: make docker
+#       - run: docker run featurebase:$(git describe --tags) help
+#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
+#   dockerhub-upload-stable:
+#     executor:
+#       name: golang
+#     steps:
+#       - checkout-plus
+#       - setup_remote_docker:
+#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
+#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+#       - run: make docker
+#       - run: docker run featurebase:$(git describe --tags) help
+#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
+#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
+
+# workflows:
+#   build:
+#     jobs:
+#       - setup:
+#           context: molecula
+#           filters:
+#             tags:
+#               only: /^v.*/
+#       - linter:
+#           context: molecula
+#           requires:
+#             - setup
+#       - go-mod-tidy:
+#           context: molecula
+#           requires:
+#             - setup
+#       - check-changelog-label:
+#           context: molecula
+#           requires:
+#             - setup
+#       - test-build-arm:
+#           context: molecula
+#           requires:
+#             - setup
+#       - test:
+#           name: test-golang-<< matrix.golang_version >>
+#           resource_class: large
+#           context: molecula
+#           requires:
+#             - setup
+#           matrix:
+#             parameters:
+#               golang_version: ["1.15.8", "1.16.10"]
+#       - test:
+#           name: << matrix.test_make_target >>
+#           resource_class: xlarge
+#           context: molecula
+#           requires:
+#             - setup
+#           matrix:
+#             parameters:
+#               test_make_target: ["test-race"]
+#       - test:
+#           name: test-shardwidth-22
+#           context: molecula
+#           shard_width: "22"
+#           resource_class: large
+#           requires:
+#             - setup
+#       - test-external-lookup:
+#           context: molecula
+#           requires:
+#             - setup
+#       - cluster-tests:
+#           context: molecula
+#           requires:
+#             - setup
+#       - docker-build:
+#           context: molecula
+#           requires:
+#             - setup
+#       - release:
+#           context: molecula
+#           requires:
+#             - setup
+#           filters:
+#             tags:
+#               only: /^v.*/
+#       - publish_release:
+#           context: molecula
+#           requires:
+#             - release
+#           filters:
+#             tags:
+#               only: /^v.*/
+#             branches:
+#               ignore: /.*/
+#       - dockerhub-upload-unstable:
+#           context: molecula
+#           requires:
+#             - setup
+#           filters:
+#             branches:
+#               only: master
+#       - dockerhub-upload-stable:
+#           context: molecula
+#           requires:
+#             - setup
+#           filters:
+#             tags:
+#               only: /^v.*/
+#             branches:
+#               ignore: /.*/

.dockerignore

@@ -0,0 +1,5 @@
+lattice/.git
+lattice/node_modules
+lattice/build
+statik/statik.go
+build

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,13 +6,10 @@ Fixes #
 
 ## Pull request checklist
 
-- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md).
-- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa).
-- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs).
+- [ ] I have updated the [documentation](https://github.com/molecula/docs).
 - [ ] I have resolved any merge conflicts.
 - [ ] I have included tests that cover my changes.
 - [ ] All new and existing tests pass.
-- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
 - [ ] Add appropriate changelog label to PR (if applicable).
 
 ## Code review checklist
@@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re
 - [ ] Check that tests have been written and that they cover the new functionality.
 - [ ] Run tests and ensure they pass.
 - [ ] Build and run the code, performing any applicable integration testing.
-- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
 - [ ] Make sure PR is tagged with appropriate changelog label.

.gitignore

@@ -4,3 +4,20 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*
+pilosa
+/featurebase
+*.dot
+.idea/
+.*.swp
+.terraform/
+*.tfstate
+launch.json
+.terraform.lock.hcl
+__pycache__/
+report.xml
+outputs.json
+builds/

.gitlab/.gitlab-ci.yml

@@ -0,0 +1,522 @@
+include:
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+
+variables:
+  GOVERSION: "1.16.13"
+
+stages:
+  - lint
+  - test
+  - build
+  - integration
+  - gauntlet
+  - performance
+  - post build
+
+smoke build:
+  image: golang:$GOVERSION
+  stage: lint
+  allow_failure: false
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+    - go build ./...
+
+golangci-lint:
+  image: golangci/golangci-lint:v1.39.0
+  stage: lint
+  allow_failure: false
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Checking for issues in new code"
+    - golangci-lint run
+
+build lattice:
+  stage: test
+  image: node:14
+  variables:
+    CI: "false"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - cd lattice
+    - yarn install
+    - yarn build
+    - mv build ../
+    - cd ../
+    - rm -r lattice
+    - mv build lattice
+    - tar -czvf lattice.tar.gz lattice
+  artifacts:
+    paths:
+      - lattice.tar.gz
+
+run jest tests:
+  stage: test
+  image: node:14
+  variables:
+    CI: "true"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Testing lattice..."
+    - cd lattice
+    - npm install --force
+    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+  artifacts:
+    paths:
+      - lattice/coverage/lcov.info
+
+run go tests:
+  stage: test
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  retry: 1
+  script:
+    - echo "Running featurebase unit tests..."
+    - go test -timeout=30m ./...
+  tags:
+    - aws
+
+run go tests race:
+  stage: test
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  retry: 1
+  script:
+    - echo "Running featurebase race tests..."
+    - go test -race -v -timeout=90m ./...
+  tags:
+    - aws
+
+run go tests shardwidth22:
+  stage: test
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase race tests..."
+    - go test -timeout=30m -tags=shardwidth22 ./...
+  tags:
+    - aws
+    
+# we do coverage reporting from the future tests because the json
+# output is very difficult to human-read. The alternative would be to
+# run the regular tests twice and also run the future tests.
+run go tests future:
+  stage: test
+  image: golang:1.17.6
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  retry: 1
+  script:
+    - echo "Running featurebase unit tests..."
+    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
+    - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
+  artifacts:
+    paths:
+      - coverage.out
+      - test-report.out
+  tags:
+    - aws
+
+upload to sonarcloud:
+  stage: integration
+  image: sonarsource/sonar-scanner-cli:4.6
+  variables:
+    SONAR_TOKEN: $SONAR_TOKEN
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+  needs:
+    - job: run go tests future
+    - job: run jest tests 
+    - job: clustertests
+    
+build for linux amd64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz 
+    - go get -v -u github.com/rakyll/statik 
+    - /go/bin/statik -src=lattice
+    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
+    - GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
+  artifacts:
+    paths:
+      - featurebase_linux_amd64
+      - roaring-migrate_linux_amd64
+
+build for linux arm64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz 
+    - go get -v -u github.com/rakyll/statik 
+    - /go/bin/statik -src=lattice
+    - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
+    - GOOS="linux" GOARCH="arm64" go build -o roaring-migrate_linux_arm64 ./cmd/roaring-migrate
+  artifacts:
+    paths:
+      - featurebase_linux_arm64
+      - roaring-migrate_linux_arm64
+
+build for darwin amd64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz 
+    - go get -v -u github.com/rakyll/statik 
+    - /go/bin/statik -src=lattice
+    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
+    - GOOS="darwin" GOARCH="amd64" go build -o roaring-migrate_darwin_amd64 ./cmd/roaring-migrate
+  artifacts:
+    paths:
+      - featurebase_darwin_amd64
+      - roaring-migrate_darwin_amd64
+
+build for darwin arm64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz 
+    - go get -v -u github.com/rakyll/statik 
+    - /go/bin/statik -src=lattice
+    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+    - GOOS="darwin" GOARCH="arm64" go build -o roaring-migrate_darwin_arm64 ./cmd/roaring-migrate
+  artifacts:
+    paths:
+      - featurebase_darwin_arm64
+      - roaring-migrate_darwin_arm64
+
+package for linux amd64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "amd64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+package for linux arm64:
+  stage: build
+  image: golang:$GOVERSION
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "arm64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+build amd container fb:
+  stage: build
+  needs:
+    - "build for linux amd64"
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:    
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:    
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+
+build arm container fb:
+  stage: build
+  needs:
+    - "build for linux arm64"
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:    
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:    
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+
+
+# clustertests doesn't run in docker, and requires several things to be set up on the runner to work:
+# 1. Install Go, make sure it's on the path
+# 2. Make sure "make" is installed
+# 3. make sure docker/docker-compose is installed
+# 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"`
+# 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
+# TODO: (I think) get clustertests coverage added to coverage report
+clustertests:
+  variables:
+    PROJECT: clustertests_${CI_CONCURRENT_ID}
+  stage: integration
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
+    - make clustertests
+    - mv internal/clustertests/results/ results/
+  artifacts:
+    paths:
+      - results/coverage*.out
+
+authclustertests:
+  variables:
+    PROJECT: authclustertests_${CI_CONCURRENT_ID}
+  stage: integration
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
+    - make authclustertests
+    - rm -rf internal/clustertests/results
+
+
+external lookup tests:
+  stage: integration
+  image: golang:$GOVERSION
+  # TODO: no rules here, do we need to add the rules line?
+  variables:
+    POSTGRES_DB: $POSTGRES_DB
+    POSTGRES_USER: $POSTGRES_USER
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_HOST_AUTH_METHOD: trust
+  services:
+    - postgres:13.5
+  script:
+    - apt-get update --allow-releaseinfo-change -y
+    - apt-get install -y postgresql-client
+    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
+
+
+smoke test:
+  stage: integration
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "service-terraform"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    TF_VAR_cluster_prefix: ""
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+    - aws configure set region "us-east-2" --profile $PROFILE
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+  script:
+    - ./qa/scripts/setupSmokeTest.sh
+    - ./qa/scripts/testSmokeTest.sh
+  after_script:
+    - ./qa/scripts/teardownSmokeTest.sh
+  needs:
+    - job: build for linux arm64
+  artifacts:
+    when: always
+    paths:
+      - report.xml
+    reports:
+      junit: report.xml
+
+
+gauntlet:
+  stage: gauntlet
+  timeout: 4h
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    FBCI_PROFILE: "service-terraform"
+    INFRA_PROFILE: "service-gitlab"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    ASG_NAME: "gitlab-runners"
+    TF_VAR_cluster_prefix: ""
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
+    - aws configure set region "us-east-2" --profile $FBCI_PROFILE
+    - aws configure set aws_profile $FBCI_PROFILE
+    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
+    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
+  script:
+    - ./qa/scripts/setupSamsungGauntlet.sh
+    - ./qa/scripts/testSamsungGauntlet.sh
+  after_script:
+    - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
+    - export INSTANCE_ID=$(cat instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+  
+s3 dump:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
+    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
+    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64
+    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64
+    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
+    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
+    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64
+    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64
+    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
+    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
+    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64
+    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64
+    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
+    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
+    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
+  needs:
+    - job: build for darwin amd64
+    - job: build for darwin arm64
+    - job: build for linux amd64
+    - job: build for linux arm64
+
+perf_able:
+  stage: performance
+  trigger:
+    include: .gitlab/.perf-able-gitlab-ci.yml
+  rules:
+    - changes:
+      - featurebase/*
+
+s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/featurebase/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp featurebase_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_linux_amd64
+    - aws s3 cp roaring-migrate_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_linux_amd64
+    - aws s3 cp featurebase_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_linux_arm64
+    - aws s3 cp roaring-migrate_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_linux_arm64
+    - aws s3 cp featurebase_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_darwin_amd64
+    - aws s3 cp roaring-migrate_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_darwin_amd64
+    - aws s3 cp featurebase_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_darwin_arm64
+    - aws s3 cp roaring-migrate_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_darwin_arm64
+    - aws s3 cp NOTICE s3://${LOCATION}/${CI_COMMIT_TAG}/NOTICE
+    - aws s3 cp install/featurebase.debian.service s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.debian.service
+    - aws s3 cp install/featurebase.redhat.service s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.redhat.service
+    - aws s3 cp install/featurebase.conf s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.conf
+  needs:
+    - job: build for darwin amd64
+    - job: build for darwin arm64
+    - job: build for linux amd64
+    - job: build for linux arm64

.gitlab/.perf-able-gitlab-ci.yml

@@ -0,0 +1,11 @@
+stages:
+  - loadtest
+
+loadtest:
+  image:
+    name: loadimpact/k6:latest
+    entrypoint: ['']
+  stage: loadtest
+  script:
+    - echo "executing local k6 in k6 container..."
+    - k6 run ./qa/scripts/perf/able/script.js

.gitlab/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine:3.14.2
+
+LABEL maintainer "dev@molecula.com"
+LABEL org.opencontainers.image.authors="dev@molecula.com"
+
+ARG ARCH
+
+WORKDIR /
+
+RUN apk add --no-cache curl jq
+
+COPY NOTICE .
+COPY featurebase_linux_$ARCH featurebase
+
+RUN chmod ugo+x .
+EXPOSE 10101
+VOLUME /data
+
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

.golangci.yml

@@ -0,0 +1,60 @@
+run:
+  #skip the protobuf generated files
+  deadline: 5m
+  timeout: 5m
+  skip-dirs-use-default: true
+  skip-dirs:
+    - pb
+    - proto
+  skip-files:
+    - pql/pql.peg.go
+linters:
+  enable:
+    - govet
+    - gofmt
+  enable-all: false
+  disable-all: true
+
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: tab
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+linters-settings:
+  gofmt:
+    simplify: true
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+
+
+    # settings per analyzer
+    settings:
+      printf: # analyzer name, run `go tool vet help` to see all analyzers
+        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+
+    # enable or disable analyzers by name
+    # run `go tool vet help` to see all analyzers
+    enable:
+      - atomicalign
+    enable-all: false
+    disable:
+      - shadow
+    disable-all: false
+
+
+issues:
+  exclude-use-default: false
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  exclude:
+    - 'declaration of "(err|ctx)" shadows declaration at'
+    - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked'
+

CHANGELOG.md

@@ -1,715 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](http://keepachangelog.com/)
-and this project adheres to [Semantic Versioning](http://semver.org/).
-
-## [1.4.0] - 2019-09-17
-
-This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions.
-
-**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release.
-
-### Added
-- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028))
-- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029))
-- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016))
-- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023))
-- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004))
-- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983))
-- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992))
-- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939))
-- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950))
-- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956))
-
-### Changed
-- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064))
-- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001))
-- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994))
-- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993))
-- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978))
-- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902))
-- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966))
-
-### Fixed
-- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050))
-- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034))
-- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033))
-- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026))
-- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021))
-- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019))
-- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017))
-- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012))
-- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000))
-- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997))
-- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975))
-- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973))
-- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970))
-- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954))
-- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951))
-- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947))
-
-### Performance
-- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988))
-- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048))
-- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041))
-- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024))
-- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974))
-
-### Removed
-- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996))
-
-## [1.3.1] - 2019-05-01
-
-This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions.
-
-### Fixed
-- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964))
-
-## [1.3.0] - 2019-04-16
-
-This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions.
-
-### Added 
-- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940))
-- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921))
-- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915))
-- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859))
-- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910))
-- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906))
-- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903))
-- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881))
-- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761))
-- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832))
-- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839))
-- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831))
-- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763))
-- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820))
-- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818))
-- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811))
-
-### Fixed
-- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944))
-- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943))
-- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938))
-- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936))
-- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931))
-- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876))
-- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863))
-- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887))
-- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848))
-- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846))
-- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842))
-- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834))
-- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835))
-- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836))
-- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826))
-- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827))
-- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822))
-- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802))
-
-### Changed
-- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924))
-- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937))
-- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909))
-- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900))
-- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616))
-- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804))
-- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851))
-- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830))
-
-### Performance
-- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916))
-- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901))
-- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897))
-- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892))
-- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871))
-- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865))
-- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814))
-- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803))
-
-## [1.2.0] - 2018-12-20
-
-This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions.
-
-### Added 
-
-- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773))
-- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774))
-- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771))
-- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753))
-- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751))
-- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756))
-- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684))
-- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733))
-- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743))
-- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717))
-- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713))
-- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707))
-- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647))
-- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699))
-- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663))
-- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683))
-- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666))
-- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671))
-- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673))
-- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645))
-- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658))
-- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653))
-- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646))
-- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738))
-- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635))
-- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631))
-- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625))
-- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628))
-
-### Changed
-
-- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780))
-- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755))
-- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748))
-- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740))
-- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725))
-- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724))
-- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677))
-- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664))
-
-### Performance
-
-- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619))
-- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741))
-
-### Fixed
-
-- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795))
-- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787))
-- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790))
-- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785))
-- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782))
-- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777))
-- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769))
-- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764))
-- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750))
-- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749))
-- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732))
-- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744))
-- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742))
-- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737))
-- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729))
-- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719))
-- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727))
-- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712))
-- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711))
-- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674))
-- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675))
-- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696))
-- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686))
-- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679))
-- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676))
-- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665))
-- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662))
-- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655))
-- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656))
-- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651))
-- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612))
-- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621))
-- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620))
-- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614))
-
-
-## [1.1.0] - 2018-08-21
-
-This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions.
-
-### Added
-
-- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610))
-- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608))
-- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601))
-- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582))
-- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584))
-- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527))
-- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524))
-- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532))
-
-### Fixed
-
-- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607))
-- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586))
-- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603))
-- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602))
-- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552))
-- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558))
-
-## [1.0.2] - 2018-08-01
-
-This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions.
-
-### Fixed
-
-- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551))
-- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547))
-- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552))
-- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541))
-- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542))
-- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533))
-- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502))
-- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555))
-
-## [1.0.1] - 2018-07-11
-
-This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions.
-
-### Fixed
-
-- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491))
-- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485))
-- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486))
-- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483))
-- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493))
-
-## [1.0.0] - 2018-07-09
-
-This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions.
-
-### Added
-
-- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337))
-- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327))
-
-### Changed
-
-- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466))
-- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297))
-- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389))
-- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452))
-- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426))
-- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424))
-- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461))
-- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451))
-- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407))
-- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463))
-- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355))
-- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365))
-- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333))
-- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326))
-- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311))
-- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454))
-
-### Removed
-
-- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397))
-- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462))
-- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396))
-- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363))
-- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347))
-- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346))
-- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341))
-- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335))
-- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332))
-- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468))
-
-### Fixed
-
-- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423))
-- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401))
-- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415))
-- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350))
-
-## [0.10.0] - 2018-05-15
-
-This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions.
-
-### Added
-
-- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285))
-- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236))
-
-### Changed
-
-- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265))
-
-### Removed
-
-- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286))
-- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284))
-- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238))
-- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255))
-
-### Fixed
-
-- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269))
-- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279))
-- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278))
-- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268))
-- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266))
-- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242))
-- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234))
-- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253))
-
-## [0.9.0] - 2018-05-04
-
-This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions.
-
-*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
-
-### Added
-
-- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
-- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
-- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
-- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
-- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
-- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
-- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
-- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
-- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
-- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
-- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
-- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245))
-- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
-- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
-- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191))
-- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246))
-- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250))
-
-### Changed
-
-- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
-- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
-- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
-- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
-- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
-- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
-- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
-- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
-- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
-- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
-- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
-- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
-- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
-- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
-- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220))
-- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223))
-- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216))
-- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221))
-
-### Removed
-
-- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
-- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
-- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
-- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226))
-- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222))
-- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219))
-- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212))
-- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209))
-- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207))
-
-### Fixed
-
-- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
-- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
-- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
-- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
-- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
-- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
-- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
-- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
-- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
-- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233))
-- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229))
-- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228))
-
-### Deprecated
-
-- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
-
-### Performance
-
-- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
-
-## [0.8.8] - 2018-02-19
-
-This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions.
-
-### Fixed
-
-- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
-
-## [0.8.7] - 2018-02-12
-
-This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions.
-
-### Fixed
-
-- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
-
-## [0.8.6] - 2018-02-09
-
-This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions.
-
-### Fixed
-
-- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
-- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
-
-## [0.8.5] - 2018-01-18
-
-This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions.
-
-### Fixed
-
-- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
-
-## [0.8.4] - 2018-01-10
-
-This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions.
-
-### Fixed
-
-- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
-- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
-
-### Performance
-
-- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
-- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
-
-## [0.8.3] - 2017-12-12
-
-This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions.
-
-### Fixed
-
-- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000))
-
-## [0.8.2] - 2017-12-05
-
-This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions.
-
-### Fixed
-
-- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994))
-
-## [0.8.1] - 2017-11-15
-
-This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions.
-
-### Fixed
-
-- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969))
-- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968))
-
-## [0.8.0] - 2017-11-15
-
-This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions.
-
-### Added
-
-- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895))
-- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933))
-- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938))
-- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942))
-- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867))
-- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898))
-- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889))
-- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881))
-- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955))
-
-### Changed
-
-- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943))
-- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884))
-- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892))
-- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945))
-- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
-
-### Fixed
-
-- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941))
-- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934))
-- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919))
-- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887))
-- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882))
-- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949))
-
-### Performance
-
-- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874))
-
-## [0.7.2] - 2017-11-15
-
-This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion.
-
-### Changed
-
-- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920))
-
-## [0.7.1] - 2017-10-09
-
-This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions.
-
-### Changed
-
-- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
-
-### Performance
-
-- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874))
-
-## [0.7.0] - 2017-10-03
-
-This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions.
-
-### Added
-
-- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856))
-- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785))
-- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840))
-- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791))
-- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755))
-- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778))
-- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861))
-- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847))
-- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789))
-- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837))
-- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831))
-- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830))
-- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806))
-- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783))
-
-### Fixed
-
-- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860))
-- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814))
-- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795))
-- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794))
-- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779))
-- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773))
-
-### Deprecated
-
-- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839))
-
-### Performance
-
-- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859))
-- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841))
-- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842))
-- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848))
-- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846))
-- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849))
-- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820))
-- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812))
-
-## [0.6.0] - 2017-08-11
-
-This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions.
-
-### Added
-
-- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758))
-
-### Changed
-
-- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750))
-
-### Fixed
-
-- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759))
-- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674))
-
-## [0.5.0] - 2017-08-02
-
-This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions.
-
-### Added
-
-- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748))
-- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646))
-- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745))
-- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741))
-- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636))
-- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635))
-- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629))
-- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600))
-- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606))
-- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721))
-- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702))
-- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670))
-
-### Changed
-
-- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747))
-- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744))
-- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699))
-- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675))
-
-### Fixed
-
-- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729))
-- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678))
-
-## [0.4.0] - 2017-06-08
-
-This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions.
-
-*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).*
-
-### Added
-- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580))
-- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586))
-- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592))
-- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571))
-- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560))
-- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537))
-- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541))
-- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563))
-- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551))
-- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540))
-- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546))
-- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535))
-- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533))
-- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510))
-
-### Changed
-- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520))
-- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578))
-- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573))
-- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524))
-- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525))
-
-### Fixed
-- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605))
-- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594))
-- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599))
-- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597))
-- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589))
-- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565))
-- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559))
-- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555))
-- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554))
-- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548))
-- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528))
-- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503))
-- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507))
-- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462))
-
-### Performance
-- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590))
-- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577))
-- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562))
-
-[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD
-[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
-[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
-[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
-[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7
-[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8
-[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9
-[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10
-[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0
-[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1
-[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2

CONTRIBUTING.md

@@ -1,177 +0,0 @@
-# Contributing to Pilosa
-
-The workflow components of these instructions apply to all Pilosa repositories.
-
-## Reporting a bug
-
-If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
-
-## Submitting a feature request
-
-Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
-which starts by [opening a new issue][1] that describes the new feature proposal.
-
-## Making code contributions
-
-Before you start working on new features, you should [open a new issue][1] to let others know what
-you're doing, otherwise you run the risk of duplicating effort. This also
-gives others an opportunity to provide input for your feature.
-
-If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
-
-
-### Development Environment
-
-- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information.
-
-- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
-
-- Fork the [Pilosa repository][2] to your own account.
-
-- It will be easier to follow these instructions if you:
-
-    ```sh
-    export GH_USERNAME=<your github username>
-	```
-
-- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa:
-
-    ```sh
-    mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
-    git clone https://github.com/pilosa/pilosa.git
-    ```
-
-- `cd` to your pilosa directory:
-
-    ```sh
-    cd ${GOPATH}/src/github.com/pilosa/pilosa
-    ```
-
-- Install Pilosa command line tools:
-
-    ```sh
-    make install
-    ```
-
-    Running `pilosa` should now run a Pilosa instance.
-
-- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username
-
-    ```sh
-    cd ${GOPATH}/src/github.com/pilosa/pilosa
-    git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git
-    ```
-
-### Makefile
-
-Pilosa includes a Makefile that automates several tasks:
-
-- Install Pilosa:
-
-    ```sh
-    make install
-    ```
-
-- Install build dependencies:
-
-    ```sh
-    make install-build-deps
-    ```
-
-- Create the vendor directory:
-
-    ```sh
-    make vendor
-    ```
-
-- Run the test suite:
-
-    ```sh
-    make test
-    ```
-
-- View the coverage report:
-
-    ```sh
-    make cover-viz
-    ```
-
-- Clear the `vendor/` and `build/` directories:
-
-    ```sh
-    make clean
-    ```
-
-- Create release tarballs:
-
-    ```sh
-    make release
-    ```
-
-- Regenerate protocol buffer files in `internal/`:
-
-    ```sh
-    make generate-protoc
-    ```
-
-- Create tagged Docker image:
-
-    ```sh
-    make docker
-    ```
-
-- Run tests inside Docker container:
-
-    ```sh
-    make docker-test
-    ```
-
-Additional commands are available in the `Makefile`.
-
-### Submitting code changes
-
-- Before starting to work on a task, sync your branch with the upstream:
-
-    ```sh
-    git checkout master
-    git pull
-    ```
-
-- Create a local feature branch:
-
-    ```sh
-    git checkout -b something-amazing
-    ```
-
-- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/).
-
-- Make sure that you've written tests for your new feature, and then run the tests:
-
-    ```sh
-    make test
-    ```
-
-- Verify that your pull request is applied to the latest version of code on github:
-
-    ```sh
-    git checkout master
-    git pull
-    git checkout something-amazing
-    git rebase master
-    ```
-
-- Push to your fork:
-
-    ```sh
-    git push -u $GH_USERNAME something-amazing:something-amazing
-    ```
-
-- Submit a [pull request][3]
-
-
-[1]: https://github.com/pilosa/pilosa/issues/new
-[2]: https://github.com/pilosa/pilosa
-[3]: https://github.com/pilosa/pilosa/compare/
-[4]: https://github.com/pilosa/general/blob/master/proposal.md
-[5]: https://github.com/pilosa/pilosa/issues
-[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

Dockerfile

@@ -1,22 +1,55 @@
-FROM golang:1.13.0 as builder
+ARG GO_VERSION=latest
 
-COPY . pilosa
+#######################
+### Lattice builder ###
+#######################
 
-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+FROM moleculacorp/nodejs:latest as lattice-builder
+WORKDIR /lattice
 
-FROM alpine:3.9.4
+COPY lattice/package.json ./
+COPY lattice/yarn.lock ./
+RUN yarn install
 
-LABEL maintainer "dev@pilosa.com"
+COPY lattice ./
+RUN yarn build
+
+######################
+### Pilosa builder ###
+######################
+
+FROM golang:${GO_VERSION} as pilosa-builder
+ARG MAKE_FLAGS
+WORKDIR /pilosa
+
+RUN go get github.com/rakyll/statik
+
+COPY . ./
+COPY --from=lattice-builder /lattice/build /lattice
+RUN /go/bin/statik -src=/lattice -dest=/pilosa
+
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
+
+#####################
+### Pilosa runner ###
+#####################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@molecula.com"
 
 RUN apk add --no-cache curl jq
 
-COPY --from=builder /go/bin/pilosa /pilosa
+COPY --from=pilosa-builder /pilosa/build/featurebase /
 
-COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
 
 EXPOSE 10101
 VOLUME /data
 
-ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

Dockerfile-clustertests

@@ -1,29 +1,35 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.11
+FROM golang:1.16
 
 LABEL maintainer "dev@pilosa.com"
 
-COPY . /go/src/github.com/pilosa/pilosa/
+COPY . /go/src/github.com/molecula/featurebase/
 
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && GO111MODULE=on make vendor
-
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && CGO_ENABLED=0 make install FLAGS="-a"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
 RUN chmod +x /pumba
 
-RUN cp /go/bin/pilosa /pilosa
+# add docker client to pause/unpause nodes
+RUN apt update
+RUN apt install -y docker.io 
+
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
+# generate an instrumented binary to allow for calculating code coverage for clustertests
+# the entrypoint for the binary is TestRunMain, which is wrapper for main 
+RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
+    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
 
-COPY LICENSE /LICENSE
 COPY NOTICE /NOTICE
 
 EXPOSE 10101
 VOLUME /data
 
 ENTRYPOINT ["bash", "-c"]
-CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-clustertests-client

@@ -0,0 +1,35 @@
+# This Dockerfile is used for cluster testing - it produces a much larger image
+# and includes all of Go as well as some utilities.
+
+FROM golang:1.16
+
+LABEL maintainer "dev@pilosa.com"
+
+COPY . /go/src/github.com/molecula/featurebase/
+
+# download pumba for fault injection
+ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
+RUN chmod +x /pumba
+
+# add docker client to pause/unpause nodes
+RUN apt update
+RUN apt install -y docker.io 
+
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
+RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
+    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+
+
+COPY NOTICE /NOTICE
+
+COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests
+
+EXPOSE 10101
+VOLUME /data
+
+ENTRYPOINT ["bash", "-c"]
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -1,23 +1,34 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
-BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
+VARIANT = Molecula
+GO=go
+GOOS=$(shell $(GO) env GOOS)
+GOARCH=$(shell $(GO) env GOARCH)
+VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
-GO_VERSION=latest
-ENTERPRISE ?= 0
-ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
-RELEASE ?= 0
-RELEASE_ENABLED = $(subst 0,,$(RELEASE))
-BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
-BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
+GO_VERSION=1.16.10
+DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
+TEST_TAGS = roaringparanoia
+UNAME := $(shell uname -s)
+TEST_TIMEOUT=30m
+RACE_TEST_TIMEOUT=90m
+ifeq ($(UNAME), Darwin)
+    IS_MACOS:=1
+else
+    IS_MACOS:=0
+endif
+
 export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export CGO_ENABLED=0
 
 # Run tests and compile Pilosa
 default: test build
@@ -28,14 +39,48 @@ clean:
 
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
-	go mod vendor
+	$(GO) mod vendor
+
+version:
+	@echo $(VERSION)
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
+	$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
+
+# Run test suite with race flag
+test-race:
+	CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
+
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
 
 bench:
-	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+	$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 
 # Run test suite with coverage enabled
 cover:
@@ -44,19 +89,27 @@ cover:
 
 # Run test suite with coverage enabled and view coverage results in browser
 cover-viz: cover
-	go tool cover -html=build/coverage.out
+	$(GO) tool cover -html=build/coverage.out
 
 # Compile Pilosa
 build:
-	go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
 # Create a single release build under the build directory
 release-build:
-	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
-	cp NOTICE README.md build/pilosa-$(VERSION_ID)
-	$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
-	tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
-	@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
+	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase"
+	cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID)
+	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
+	@echo Created release build: build/featurebase-$(VERSION_ID).tar.gz
+
+test-release-build: docker-build
+	mv build/featurebase-$(VERSION_ID).tar.gz install/
+	cd install && docker build -t featurebase:test_installation \
+		-f test_installation.Dockerfile \
+		--build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz .
+	mv install/featurebase-$(VERSION_ID).tar.gz build/
+	docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+		featurebase:test_installation
 
 # Error out if there are untracked changes in Git
 check-clean:
@@ -64,76 +117,174 @@ ifndef SKIP_CHECK_CLEAN
 	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
 endif
 
-# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
-release: check-clean
+# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
+release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
+	$(MAKE) release-build GOOS=darwin GOARCH=arm64
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
-	$(MAKE) release-build GOOS=linux GOARCH=386
-	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
+	$(MAKE) release-build GOOS=linux GOARCH=arm64
+
+# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
+release-sans-ui: check-clean
+	rm -f statik/statik.go
+	$(MAKE) release-build GOOS=darwin GOARCH=amd64
+	$(MAKE) release-build GOOS=darwin GOARCH=arm64
+	$(MAKE) release-build GOOS=linux GOARCH=amd64
+	$(MAKE) release-build GOOS=linux GOARCH=arm64
+
+package:
+	go build -o featurebase ./cmd/featurebase
+	nfpm package --packager deb --target featurebase_$(VERSION_ID).deb
+	nfpm package --packager rpm --target featurebase_$(VERSION_ID).rpm
 
 
-# try (e.g.) internal/clustertests/docker-compose-replication2.yml
-DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= clustertests
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
 
 # Run cluster integration tests using docker. Requires docker daemon to be
-# running. This will catch changes to internal/clustertests/*.go, but if you
-# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
-# pilosa image.
-clustertests:
-	docker-compose -f $(DOCKER_COMPOSE) down
-	docker-compose -f $(DOCKER_COMPOSE) build client1
-	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
+# running and docker-compose to be installed.
+clustertests: vendor
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
-
-# Like clustertests, but rebuilds all images.
-clustertests-build:
-	docker-compose -f $(DOCKER_COMPOSE) down
-	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
-
-# Create prerelease builds
-prerelease:
-	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
-	$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
-
-prerelease-upload:
-	aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
+# Run the cluster tests with authentication enabled 
+AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf"
+authclustertests: vendor
+	$(eval PROJECT=authclustertests)
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 # Install Pilosa
 install:
-	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
+
+install-bench:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
+
+# Build the lattice assets
+build-lattice:
+	docker build -t lattice:build ./lattice
+	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
 
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
-	go generate github.com/pilosa/pilosa/v2/internal
+	$(GO) generate github.com/molecula/featurebase/v3/pb
+
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	$(GO) generate github.com/molecula/featurebase/v3/statik
+
+# `go generate` statik assets (lattice UI) in Docker
+generate-statik-docker: build-lattice
+	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
 
 # `go generate` stringers
 generate-stringer:
-	go generate github.com/pilosa/pilosa/v2
+	$(GO) generate github.com/molecula/featurebase/v3
 
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
+generate-proto-grpc: require-protoc require-protoc-gen-go
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
+	# TODO: Modify above commands and remove the below mv if possible.
+	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
+	# I couldn't get it to work during development - Cody
+	cp -r proto/github.com/molecula/featurebase/v3/proto/ proto/
+	rm -rf proto/github.com
+
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
+
+# Create release using Docker
+docker-release:
+	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS=linux GOARCH=arm64
+	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
+	$(MAKE) docker-build GOOS=darwin GOARCH=arm64
+
+# Build a release in Docker
+docker-build: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --target pilosa-builder \
+	    --tag featurebase:build .
+	docker create --name featurebase-build featurebase:build
+	mkdir -p build/featurebase-$(VERSION_ID)
+	docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
+	cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID)
+	docker rm featurebase-build
+	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
 
 # Create Docker image from Dockerfile
-docker:
-	docker build -t "pilosa:$(VERSION)" .
-	@echo Created docker image: pilosa:$(VERSION)
+docker-image: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
+	    --tag featurebase:$(VERSION) .
+	@echo Created docker image: featurebase:$(VERSION)
 
-# Compile Pilosa inside Docker container
-docker-build:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Create docker image (alias)
+docker: docker-image # alias
+
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
+pilosa-keydump:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
+
+# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
+pilosa-chk:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
+
+pilosa-fsck:
+	cd ./cmd/pilosa-fsck && make install && make release
 
 # Run Pilosa tests inside Docker container
 docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
+	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) ./...
+
+# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
+# run top tests, not subdirs. print summary red/green after.
+# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
+topt:
+	mv log.topt.roar log.topt.roar.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
+	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
+	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
+
+topt-race:
+	mv log.topt.race log.topt.race.prev || true
+	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
+	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
+	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
-	golangci-lint run
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
 
 # Run gometalinter with custom flags
 gometalinter: require-gometalinter vendor
@@ -158,13 +309,6 @@ gometalinter: require-gometalinter vendor
 	    --exclude "^pql/pql.peg.go" \
 	    ./...
 
-# Verify that all Go files have license header
-check-license-headers: SHELL:=/bin/bash
-check-license-headers:
-	@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
-	    do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
-	    grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
-
 ######################
 # Build dependencies #
 ######################
@@ -175,24 +319,33 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go install github.com/rakyll/statik@latest
 
 install-stringer:
-	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
+	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
 
 install-protoc-gen-gofast:
-	GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
+	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc-gen-go:
+	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
 
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 
 install-peg:
-	GO111MODULE=off go get github.com/pointlander/peg
+	GO111MODULE=off $(GO) get github.com/pointlander/peg
 
 install-golangci-lint:
-	GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
+	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 
 install-gometalinter:
-	GO111MODULE=off go get -u github.com/alecthomas/gometalinter
+	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
-	GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
+	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
+
+test-external-lookup:
+	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)

NOTES

@@ -1,26 +0,0 @@
-                                                  
-     Index        Column                          
-        ┌───────────▼────────────────────────────┐
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-   Row──▶0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │────────────────────────────────────────┤
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │────────────────────────────────────────┤
-    F  ▶│0000000000000000000000000000000000000000│
-    i  ││0000000000000000000000000000000000000000│
-    e  ││0000000000000000000000000000000000000000│
-    l  ││0000000000000000000000000000000000000000│
-    d  ▶│0000000000000000000000000000000000000000│
-        └────────────────────────────────────────┘
-                                     ▲───────────▲
-                                         Shard    
-
-
-Fragment=intersection of field & shard

NOTICE

@@ -1,44 +1,12 @@
 Software license
 ================
 
-Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License").
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Enterprise Edition software license
-===================================
-
-Files contained under the directory `enterprise` are subject to the following
-license notice (Full license included in the file `COPYING`):
-
-    Copyright (C) 2018 Pilosa Corp. All rights reserved.
-
-    Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    Pilosa Enterprise Edition is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.
+Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
 
 Third-party software licenses
 =============================
 
-The file /pilosa/lru/lru.go contains a redistribution of lru
+The file /lru/lru.go contains a redistribution of lru
 (github.com/golang/groupcache/lru); the license follows:
 
     Copyright 2013 Google Inc.
@@ -115,3 +83,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
     CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

README.md

@@ -1,80 +1,6 @@
-<p>
-    <a href="https://www.pilosa.com">
-        <img src="https://www.pilosa.com/img/logo.svg" width="50%">
-    </a>
-</p>
+# FeatureBase, a distributed bitmap index
 
-[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master)
-[![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa)
-[![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa)
-[![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE)
-[![CLA Assistant](https://cla-assistant.io/readme/badge/pilosa/pilosa)](https://cla-assistant.io/pilosa/pilosa)
-[![GitHub release](https://img.shields.io/github/release/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/releases)
+See our [internal documentation](https://internal-docs.molecula.cloud), which includes all [external documentation](https://docs.molecula.cloud), plus many internal-only pages, listed under the "Internal" heading in the main navigation bar.
 
-## An open source, distributed bitmap index.
-- [Docs](#docs)
-- [Getting Started](#getting-started)
-- [Data Model](#data-model)
-- [Query Language](#query-language)
-- [Client Libraries](#client-libraries)
-- [Get Support](#get-support)
-- [Contributing](#contributing)
+Follow along with the [Sample Project](https://internal-docs.molecula.cloud/tutorials/getting-started) to get a better understanding of FeatureBase's capabilities.
 
-Want to contribute? One of the easiest ways is to [tell us how you're using (or want to use) Pilosa](https://github.com/pilosa/pilosa/issues/1074). We learn from every discussion!
-
-## Docs
-
-See our [Documentation](https://www.pilosa.com/docs/) for information about installing and working with Pilosa.
-
-
-## Getting Started
-
-1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
-
-2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
-
-    ```shell
-    pilosa server
-    ```
-    
-    and verify that it's running:
-    
-    ```shell
-    curl localhost:10101/nodes
-    ```
-
-3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
-
-
-## Data Model
-
-Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) works.
-
-
-## Query Language
-
-You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
-
-
-## Client Libraries
-
-There are supported libraries for the following languages:
-- [Go](https://www.pilosa.com/docs/client-libraries/#go)
-- [Java](https://www.pilosa.com/docs/client-libraries/#java)
-- [Python](https://www.pilosa.com/docs/client-libraries/#python)
-
-## Licenses
-
-The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
-If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
-Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
-directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
-[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
-
-## Get Support
-
-There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support.
-
-## Contributing
-
-Pilosa is an open source project. Please see our [Contributing Guide](CONTRIBUTING.md) for information about how to get involved.

api/client/grpc.go

@@ -0,0 +1,202 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"sync"
+
+	"github.com/molecula/featurebase/v3/logger"
+	pb "github.com/molecula/featurebase/v3/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+	logger      logger.Logger
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config, logger logger.Logger) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+		logger:      logger,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		c.logger.Errorf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

apimethod_string.go

@@ -19,25 +19,35 @@ func _() {
 	_ = x[apiFragmentBlockData-8]
 	_ = x[apiFragmentBlocks-9]
 	_ = x[apiFragmentData-10]
-	_ = x[apiField-11]
-	_ = x[apiFieldAttrDiff-12]
-	_ = x[apiImport-13]
-	_ = x[apiImportValue-14]
-	_ = x[apiIndex-15]
-	_ = x[apiIndexAttrDiff-16]
+	_ = x[apiTranslateData-11]
+	_ = x[apiFieldTranslateData-12]
+	_ = x[apiField-13]
+	_ = x[apiImport-14]
+	_ = x[apiImportValue-15]
+	_ = x[apiIndex-16]
 	_ = x[apiQuery-17]
 	_ = x[apiRecalculateCaches-18]
 	_ = x[apiRemoveNode-19]
 	_ = x[apiResizeAbort-20]
-	_ = x[apiSetCoordinator-21]
+	_ = x[apiSchema-21]
 	_ = x[apiShardNodes-22]
-	_ = x[apiViews-23]
-	_ = x[apiApplySchema-24]
+	_ = x[apiState-23]
+	_ = x[apiViews-24]
+	_ = x[apiApplySchema-25]
+	_ = x[apiStartTransaction-26]
+	_ = x[apiFinishTransaction-27]
+	_ = x[apiTransactions-28]
+	_ = x[apiGetTransaction-29]
+	_ = x[apiActiveQueries-30]
+	_ = x[apiPastQueries-31]
+	_ = x[apiIDReserve-32]
+	_ = x[apiIDCommit-33]
+	_ = x[apiIDReset-34]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

attr.go

@@ -1,205 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"bytes"
-	"sort"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/v2/internal"
-)
-
-// Attribute data type enum.
-const (
-	attrTypeString = 1
-	attrTypeInt    = 2
-	attrTypeBool   = 3
-	attrTypeFloat  = 4
-)
-
-// AttrStore represents an interface for handling row/column attributes.
-type AttrStore interface {
-	Path() string
-	Open() error
-	Close() error
-	Attrs(id uint64) (m map[string]interface{}, err error)
-	SetAttrs(id uint64, m map[string]interface{}) error
-	SetBulkAttrs(m map[uint64]map[string]interface{}) error
-	Blocks() ([]AttrBlock, error)
-	BlockData(i uint64) (map[uint64]map[string]interface{}, error)
-}
-
-// nopStore represents an AttrStore that doesn't do anything.
-var nopStore AttrStore = nopAttrStore{}
-
-// newNopAttrStore returns an attr store which does nothing. It returns a global
-// object to avoid unnecessary allocations.
-func newNopAttrStore(string) AttrStore { return nopStore }
-
-// nopAttrStore represents a no-op implementation of the AttrStore interface.
-type nopAttrStore struct{}
-
-// Path is a no-op implementation of AttrStore Path method.
-func (s nopAttrStore) Path() string { return "" }
-
-// Open is a no-op implementation of AttrStore Open method.
-func (s nopAttrStore) Open() error { return nil }
-
-// Close is a no-op implementation of AttrStore Close method.
-func (s nopAttrStore) Close() error { return nil }
-
-// Attrs is a no-op implementation of AttrStore Attrs method.
-func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil }
-
-// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
-func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil }
-
-// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
-func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil }
-
-// Blocks is a no-op implementation of AttrStore Blocks method.
-func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
-
-// BlockData is a no-op implementation of AttrStore BlockData method.
-func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }
-
-// AttrBlock represents a checksummed block of the attribute store.
-type AttrBlock struct {
-	ID       uint64 `json:"id"`
-	Checksum []byte `json:"checksum"`
-}
-
-// attrBlocks represents a list of blocks.
-type attrBlocks []AttrBlock
-
-// Diff returns a list of block ids that are different or are new in other.
-// Block lists must be in sorted order.
-func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
-	var ids []uint64
-	for {
-		// Read next block from each list.
-		var blk0, blk1 *AttrBlock
-		if len(a) > 0 {
-			blk0 = &a[0]
-		}
-		if len(other) > 0 {
-			blk1 = &other[0]
-		}
-
-		// Exit if "a" contains no more blocks.
-		if blk0 == nil {
-			return ids
-		}
-
-		// Add block ID if it's different or if it's only in "a".
-		if blk1 == nil || blk0.ID < blk1.ID {
-			ids = append(ids, blk0.ID)
-			a = a[1:]
-		} else if blk1.ID < blk0.ID {
-			other = other[1:]
-		} else {
-			if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
-				ids = append(ids, blk0.ID)
-			}
-			a, other = a[1:], other[1:]
-		}
-	}
-}
-
-func encodeAttrs(m map[string]interface{}) []*internal.Attr {
-	keys := make([]string, 0, len(m))
-	for k := range m {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	a := make([]*internal.Attr, len(keys))
-	for i := range keys {
-		a[i] = encodeAttr(keys[i], m[keys[i]])
-	}
-	return a
-}
-
-func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
-	m := make(map[string]interface{}, len(pb))
-	for i := range pb {
-		key, value := decodeAttr(pb[i])
-		m[key] = value
-	}
-	return m
-}
-
-// encodeAttr converts a key/value pair into an Attr internal representation.
-func encodeAttr(key string, value interface{}) *internal.Attr {
-	pb := &internal.Attr{Key: key}
-	switch value := value.(type) {
-	case string:
-		pb.Type = attrTypeString
-		pb.StringValue = value
-	case float64:
-		pb.Type = attrTypeFloat
-		pb.FloatValue = value
-	case uint64:
-		pb.Type = attrTypeInt
-		pb.IntValue = int64(value)
-	case int64:
-		pb.Type = attrTypeInt
-		pb.IntValue = value
-	case bool:
-		pb.Type = attrTypeBool
-		pb.BoolValue = value
-	}
-	return pb
-}
-
-// decodeAttr converts from an Attr internal representation to a key/value pair.
-func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
-	switch attr.Type {
-	case attrTypeString:
-		return attr.Key, attr.StringValue
-	case attrTypeInt:
-		return attr.Key, attr.IntValue
-	case attrTypeBool:
-		return attr.Key, attr.BoolValue
-	case attrTypeFloat:
-		return attr.Key, attr.FloatValue
-	default:
-		return attr.Key, nil
-	}
-}
-
-// cloneAttrs returns a shallow clone of m.
-func cloneAttrs(m map[string]interface{}) map[string]interface{} {
-	other := make(map[string]interface{}, len(m))
-	for k, v := range m {
-		other[k] = v
-	}
-	return other
-}
-
-// EncodeAttrs encodes an attribute map into a byte slice.
-func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
-	return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
-}
-
-// DecodeAttrs decodes a byte slice into an  attribute map.
-func DecodeAttrs(v []byte) (map[string]interface{}, error) {
-	var pb internal.AttrMap
-	if err := proto.Unmarshal(v, &pb); err != nil {
-		return nil, err
-	}
-	return decodeAttrs(pb.GetAttrs()), nil
-}

attr_test.go

@@ -1,201 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"io/ioutil"
-	"os"
-	"reflect"
-	"runtime"
-	"sync"
-	"testing"
-
-	"github.com/pilosa/pilosa/v2"
-	"github.com/pilosa/pilosa/v2/boltdb"
-)
-
-// Ensure database can set and retrieve column attributes.
-func TestAttrStore_Attrs(t *testing.T) {
-	s := MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Retrieve attributes for column #1.
-	if m, err := s.Attrs(1); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
-		t.Fatalf("unexpected attrs(1): %#v", m)
-	}
-
-	// Retrieve attributes for column #2.
-	if m, err := s.Attrs(2); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
-		t.Fatalf("unexpected attrs(2): %#v", m)
-	}
-}
-
-// Ensure database returns a non-nil empty map if unset.
-func TestAttrStore_Attrs_Empty(t *testing.T) {
-	s := MustOpenAttrStore()
-	defer s.Close()
-
-	if m, err := s.Attrs(100); err != nil {
-		t.Fatal(err)
-	} else if m == nil || len(m) > 0 {
-		t.Fatalf("unexpected attrs: %#v", m)
-	}
-}
-
-// Ensure database can unset attributes if explicitly set to nil.
-func TestAttrStore_Attrs_Unset(t *testing.T) {
-	s := MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify attributes.
-	if m, err := s.Attrs(1); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) {
-		t.Fatalf("unexpected attrs: %#v", m)
-	}
-}
-
-// Ensure attribute block checksums can be returned.
-func TestAttrStore_Blocks(t *testing.T) {
-	s := MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Retrieve blocks.
-	blks0, err := s.Blocks()
-	if err != nil {
-		t.Fatal(err)
-	} else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 {
-		t.Fatalf("unexpected blocks: %#v", blks0)
-	}
-
-	// Change second block.
-	if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Ensure second block changed.
-	blks1, err := s.Blocks()
-	if err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(blks0[0], blks1[0]) {
-		t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0])
-	} else if reflect.DeepEqual(blks0[1], blks1[1]) {
-		t.Fatalf("block 1 match: %#v ", blks0[0])
-	} else if !reflect.DeepEqual(blks0[2], blks1[2]) {
-		t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
-	}
-}
-
-// AttrStore represents a test wrapper for pilosa.AttrStore.
-type AttrStore struct {
-	pilosa.AttrStore
-}
-
-// NewAttrStore returns a new instance of AttrStore.
-func NewAttrStore(string) pilosa.AttrStore {
-	f, err := ioutil.TempFile("", "pilosa-attr-")
-	if err != nil {
-		panic(err)
-	}
-	f.Close()
-	os.Remove(f.Name())
-
-	return &AttrStore{boltdb.NewAttrStore(f.Name())}
-}
-
-func BenchmarkAttrStore_Duplicate(b *testing.B) {
-	s := MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	const n = 5
-	for i := 0; i < n; i++ {
-		if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
-			b.Fatal(err)
-		}
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-
-	// Update attributes with an existing subset.
-	cpuN := runtime.GOMAXPROCS(0)
-	var wg sync.WaitGroup
-	errchan := make(chan error)
-	for i := 0; i < cpuN; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for j := 0; j < b.N/cpuN; j++ {
-				if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
-					errchan <- err
-				}
-			}
-		}()
-	}
-	go func() {
-		wg.Wait()
-		close(errchan)
-	}()
-	if err := <-errchan; err != nil {
-		b.Fatal(err)
-	}
-}
-
-// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
-func MustOpenAttrStore() pilosa.AttrStore {
-	s := NewAttrStore("")
-	if err := s.Open(); err != nil {
-		panic(err)
-	}
-	return s
-}
-
-// Close closes the database and removes the underlying data.
-func (s *AttrStore) Close() error {
-	defer os.RemoveAll(s.Path())
-	return s.AttrStore.Close()
-}

audit.go

@@ -0,0 +1,12 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"github.com/molecula/featurebase/v3/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,39 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/molecula/featurebase/v3/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,94 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

authn/authenticate.go

@@ -0,0 +1,307 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+
+// Package authn handles authentication
+package authn
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/molecula/featurebase/v3/logger"
+	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
+)
+
+// cachedGroups is used to hold groups and when they were last cached
+type cachedGroups struct {
+	cacheTime time.Time
+	groups    []Group
+}
+
+// cacheToken is used to hold tokens and when they were added to the cache
+type cachedToken struct {
+	cacheTime time.Time
+	token     *oauth2.Token
+}
+
+// UserInfo holds the information about the user from the token
+type UserInfo struct {
+	UserID   string    `json:"userid"`
+	UserName string    `json:"username"`
+	Groups   []Group   `json:"groups"`
+	Expiry   time.Time `json:"expiry"`
+	Token    string    `json:"token"`
+}
+
+// Group holds group information for an authenticated user
+type Group struct {
+	GroupID   string `json:"id"`
+	GroupName string `json:"displayName"`
+}
+
+// Groups holds a slice of Group for marshalling from JSON
+type Groups struct {
+	Groups []Group `json:"value"`
+}
+
+// Auth holds state, configuration, and utilities needed for authentication.
+type Auth struct {
+	logger         logger.Logger
+	cookieName     string
+	secretKey      []byte
+	groupEndpoint  string
+	logoutEndpoint string
+	fbURL          string // fbURL is the domain featurebase is hosted on, used for post logout redirection
+	oAuthConfig    *oauth2.Config
+	cacheTTL       time.Duration           // cacheTTL is used to determine if a cached item should be refreshed or not
+	tokenTTR       time.Duration           // tokenTTR (time to refresh) is used to determine if a token should be refreshed or not
+	tokenCache     map[string]cachedToken  // tokenCache is a map of accessToken -> *oauth2.Token which we can use to refresh the tokens
+	groupsCache    map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
+	lastCacheClean time.Time               // last cache clean is the time that the cache was last cleaned
+}
+
+// NewAuth instantiates and returns a new Auth struct
+func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string) (auth *Auth, err error) {
+	auth = &Auth{
+		logger:         logger,
+		cookieName:     "molecula-chip",
+		groupEndpoint:  groupEndpoint,
+		logoutEndpoint: logout,
+		fbURL:          url,
+		oAuthConfig: &oauth2.Config{
+			RedirectURL:  fmt.Sprintf("%s/redirect", url),
+			ClientID:     clientID,
+			ClientSecret: clientSecret,
+			Scopes:       scopes,
+			Endpoint: oauth2.Endpoint{
+				AuthURL:  authURL,
+				TokenURL: tokenURL,
+			},
+		},
+		tokenCache:     map[string]cachedToken{},
+		groupsCache:    map[string]cachedGroups{},
+		cacheTTL:       10 * time.Minute,
+		tokenTTR:       7 * time.Minute,
+		lastCacheClean: time.Now(),
+	}
+
+	if auth.secretKey, err = decodeHex(secretKey); err != nil {
+		return nil, errors.Wrap(err, "decoding secret key")
+	}
+	return auth, nil
+}
+
+// SecretKey is a convenient function to get the SecretKey from an Auth struct
+func (a Auth) SecretKey() []byte {
+	return a.secretKey
+}
+
+// Authenticate takes in a bearer token `bearer` and returns UserInfo from that token
+// it is caller's responsibility to inform the user that the access token has been refreshed
+func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, error) {
+	// clean up the cache every 30 minutes or so
+	if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
+		a.cleanCache()
+	}
+
+	if tkn, ok := a.tokenCache[bearer]; ok && (tkn.token.Expiry.Sub(time.Now()) <= a.tokenTTR || !tkn.token.Valid()) {
+		// refresh the token
+		resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
+			url.Values{
+				"grant_type":    {"refresh_token"},
+				"refresh_token": {tkn.token.RefreshToken},
+				"client_id":     {a.oAuthConfig.ClientID},
+				"client_secret": {a.oAuthConfig.ClientSecret},
+			},
+		)
+		if err != nil {
+			return nil, errors.Wrap(err, "refreshing token")
+		}
+		defer resp.Body.Close()
+		var t oauth2.Token
+		if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
+			return nil, errors.Wrap(err, "decoding refreshed token")
+		}
+
+		// update the cache
+		delete(a.tokenCache, bearer)
+		delete(a.groupsCache, bearer)
+		bearer = t.AccessToken
+		a.tokenCache[bearer] = cachedToken{time.Now(), &t}
+	}
+
+	// NOTE: we are using ParseUnverified here because the IDP validates the
+	// token's signature when we get the user's groups, we just need to make
+	// sure it's not expired and is well-formed
+	token, _, err := new(jwt.Parser).ParseUnverified(bearer, &jwt.MapClaims{})
+	// well-formed-ness check
+	if token == nil || token.Claims == nil || err != nil {
+		return nil, fmt.Errorf("parsing bearer token: %v", err)
+	}
+
+	claims := *token.Claims.(*jwt.MapClaims)
+
+	// expiry check
+	if exp, ok := claims["exp"].(string); ok {
+		if expiry, err := strconv.ParseInt(exp, 10, 64); err != nil || expiry < time.Now().UTC().Unix() {
+			return nil, fmt.Errorf("token is expired")
+		}
+	}
+
+	userInfo := UserInfo{
+		UserID:   claims["oid"].(string),
+		UserName: claims["name"].(string),
+		Token:    bearer,
+		Groups:   []Group{},
+	}
+
+	if userInfo.Groups, err = a.getGroups(bearer); err != nil {
+		return nil, errors.Wrap(err, "getting groups")
+	}
+
+	return &userInfo, nil
+}
+
+// cleanCache removes old items from our cache
+func (a *Auth) cleanCache() {
+	for bearer, tkn := range a.tokenCache {
+		// if it's been more than 24 hours since the token was cached
+		if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
+			// remove it from our cache
+			delete(a.tokenCache, bearer)
+		}
+	}
+	for bearer, tkn := range a.groupsCache {
+		// if it's been more than 24 hours since the groups were cached
+		if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
+			// remove it from our cache
+			delete(a.groupsCache, bearer)
+		}
+	}
+	a.lastCacheClean = time.Now()
+}
+
+// Login redirects a user to login to their configured oAuth authorize endpoint
+func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
+	authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
+	http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
+}
+
+// Logout clears out the user's cookie, removes the token from our cache, and
+// redirects user to IdP's logout endpoint
+func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
+	// remove the bearer token from a.tokenCache and a.groupsCache
+	if bearer, err := r.Cookie(a.cookieName); err == nil {
+		delete(a.tokenCache, bearer.Value)
+		delete(a.groupsCache, bearer.Value)
+	}
+	// clear cookie
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.cookieName,
+		Value:    "",
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  time.Unix(0, 0),
+	})
+
+	http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
+}
+
+// Redirect handles the oAuth /redirect endpoint. It gets an access token and
+// returns it to the user in the form of a cookie
+func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
+	token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
+	if err != nil {
+		a.logger.Warnf("getting token from IdP: %+v", err)
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	a.tokenCache[token.AccessToken] = cachedToken{time.Now(), token}
+
+	a.SetCookie(w, token.AccessToken, token.Expiry)
+	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
+}
+
+// getGroups gets the group membership for a given token from configured IdP
+func (a *Auth) getGroups(token string) ([]Group, error) {
+	var groups Groups
+
+	g, ok := a.groupsCache[token]
+	if ok && (time.Now().Sub(g.cacheTime) < a.cacheTTL) {
+		return g.groups, nil
+	}
+
+	req, err := http.NewRequest("GET", a.groupEndpoint, nil)
+	if err != nil {
+		return groups.Groups, errors.Wrap(err, "creating new request to group endpoint")
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+	response, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return groups.Groups, errors.Wrap(err, "getting group membership info")
+	}
+
+	defer response.Body.Close()
+	if err = json.NewDecoder(response.Body).Decode(&groups); err != nil {
+		return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
+	}
+
+	a.groupsCache[token] = cachedGroups{
+		cacheTime: time.Now(),
+		groups:    groups.Groups,
+	}
+	return groups.Groups, nil
+}
+
+func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time) error {
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.cookieName,
+		Value:    token,
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  expiry,
+	})
+	return nil
+}
+
+func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, token string) error {
+	cookies := []string{}
+	if c, ok := md["cookie"]; ok {
+		for _, cookie := range c {
+			if strings.HasPrefix(cookie, a.cookieName) {
+				cookie = a.cookieName + "=" + token
+			}
+			cookies = append(cookies, cookie)
+		}
+	}
+	md["cookie"] = cookies
+	return grpc.SetHeader(ctx, md)
+}
+
+func decodeHex(hexstr string) ([]byte, error) {
+	data, err := hex.DecodeString(hexstr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding hex string to byte slice")
+	}
+	if len(data) != 32 {
+		return nil, fmt.Errorf("invalid key length")
+	}
+	return data, nil
+}

authn/authenticate_internal_test.go

@@ -0,0 +1,592 @@
+package authn
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+	"github.com/molecula/featurebase/v3/logger"
+	"golang.org/x/oauth2"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+func NewTestAuth(t *testing.T) *Auth {
+	t.Helper()
+	var (
+		ClientID         = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
+		ClientSecret     = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+		AuthorizeURL     = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
+		TokenURL         = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
+		GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
+		LogoutURL        = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
+		Scopes           = []string{"https://graph.microsoft.com/.default", "offline_access"}
+		Key              = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+	)
+
+	a, err := NewAuth(
+		logger.NewStandardLogger(os.Stdout),
+		"http://localhost:10101/",
+		Scopes,
+		AuthorizeURL,
+		TokenURL,
+		GroupEndpointURL,
+		LogoutURL,
+		ClientID,
+		ClientSecret,
+		Key,
+	)
+	if err != nil {
+		t.Fatalf("building auth object%s", err)
+	}
+	return a
+}
+func TestAuth(t *testing.T) {
+	a := NewTestAuth(t)
+	t.Run("SetCookie", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		err := a.SetCookie(w, "a cookie value", time.Now().Add(time.Hour))
+		if err != nil {
+			t.Fatalf("expected no errors, got: %v", err)
+		}
+
+		if w.Result().Cookies()[0].Value == "" {
+			t.Errorf("expected something, got empty string")
+		}
+
+		if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
+			t.Fatalf("path=%s, want %s", got, want)
+		}
+	})
+	t.Run("SetGRPCMetadata", func(t *testing.T) {
+		md := metadata.MD{
+			"cookie": []string{a.cookieName + "=something"},
+		}
+		ctx := grpc.NewContextWithServerTransportStream(
+			metadata.NewIncomingContext(context.TODO(),
+				md,
+			),
+			NewServerTransportStream(),
+		)
+		md, ok := metadata.FromIncomingContext(ctx)
+		if !ok {
+			t.Fatalf("expected ok, got: %v", ok)
+		}
+		err := a.SetGRPCMetadata(ctx, md, "this is a token!")
+		if err != nil {
+			t.Fatalf("expected no errors, got: %v", err)
+		}
+		md, ok = metadata.FromIncomingContext(ctx)
+		if !ok {
+			t.Fatalf("expected ok, got: %v", ok)
+		}
+		c, ok := md["cookie"]
+		if !ok {
+			t.Fatalf("expected ok, got: %v", ok)
+		}
+		var cookie string
+		for _, cookie = range c {
+			if strings.HasPrefix(cookie, a.cookieName) {
+				break
+			}
+		}
+		if exp, got := a.cookieName+"=this is a token!", cookie; got != exp {
+			t.Fatalf("expected '%v', got '%v'", exp, got)
+		}
+	})
+	t.Run("KeyLength", func(t *testing.T) {
+		_, err := NewAuth(
+			logger.NewStandardLogger(os.Stdout),
+			"http://localhost:10101/",
+			[]string{"https://graph.microsoft.com/.default", "offline_access"},
+			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
+			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
+			"https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
+			"https://login.microsoftonline.com/common/oauth2/v2.0/logout",
+			"e9088663-eb08-41d7-8f65-efb5f54bbb71",
+			"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
+			"DEADBEEFD",
+		)
+		if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
+			t.Fatalf("expected error decoding secret key got: %v", err)
+		}
+	})
+	t.Run("GetSecretKey", func(t *testing.T) {
+		want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if got := a.SecretKey(); !bytes.Equal(got, want) {
+			t.Fatalf("expected %v, got %v", got, want)
+		}
+	})
+}
+
+func TestAuthenticate(t *testing.T) {
+	cases := []struct {
+		name         string
+		uid          string
+		uname        string
+		exp          int64
+		refresh      bool
+		errOnRefresh bool
+		malformed    bool
+		groups       []Group
+		err          error
+	}{
+		{
+			name:  "GoodToken",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+		},
+		{
+			name:      "Malformed",
+			malformed: true,
+			err:       fmt.Errorf("parsing bearer token: token contains an invalid number of segments"),
+		},
+
+		{
+			name:  "ExpiredTokenNoRefresh",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			exp: -17764800,
+			err: fmt.Errorf("token is expired"),
+		},
+		{
+			name:  "ExpiredTokenYesRefresh",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			refresh: true,
+			exp:     -17764800,
+		},
+		{
+			name:  "ExpiredTokenYesRefreshButError",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			refresh:      true,
+			errOnRefresh: true,
+			exp:          -17764800,
+			err:          fmt.Errorf("decoding refreshed token: invalid character 'b' looking for beginning of value"),
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			// setup the test
+			a := NewTestAuth(t)
+			token := ""
+			var err error
+			if !test.malformed {
+				tkn := jwt.New(jwt.SigningMethodHS256)
+				claims := tkn.Claims.(jwt.MapClaims)
+				claims["oid"] = test.uid
+				claims["name"] = test.uname
+				if test.exp != 0 {
+					claims["exp"] = strconv.Itoa(int(test.exp))
+				}
+				token, err = tkn.SignedString(a.SecretKey())
+				if err != nil {
+					t.Fatalf("unexpected error when signing token %v", err)
+				}
+			} else {
+				token = "asdfasdfasdfasdF"
+			}
+			if len(test.groups) > 0 {
+				a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
+			}
+			if test.refresh {
+				var srv *httptest.Server
+				if !test.errOnRefresh {
+					srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						tkn := jwt.New(jwt.SigningMethodHS256)
+						claims := tkn.Claims.(jwt.MapClaims)
+						claims["oid"] = test.uid
+						claims["name"] = test.uname
+						expiry := strconv.Itoa(int(time.Now().Add(2 * time.Hour).Unix()))
+						claims["exp"] = expiry
+						fresh, err := tkn.SignedString(a.SecretKey())
+						if err != nil {
+							t.Fatalf("unexpected error when signing token %v", err)
+						}
+
+						a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
+						fmt.Fprintf(w, `{"access_token": "`+fresh+`",  "refresh_token": "blah",  "token_type": "bearer",  "expires": `+expiry+` }`)
+					}))
+				} else {
+					srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						http.Error(w, "bad", http.StatusInternalServerError)
+					}))
+				}
+				defer srv.Close()
+				a.oAuthConfig.Endpoint.TokenURL = srv.URL
+				a.tokenCache[token] = cachedToken{
+					time.Now(),
+					&oauth2.Token{
+						AccessToken:  token,
+						RefreshToken: "blah",
+						Expiry:       time.Unix(test.exp, 0),
+					},
+				}
+			}
+
+			// do the actual testing
+			uinfo, err := a.Authenticate(context.TODO(), token)
+			// okay this part kind of sucks bc we need to check errors and i
+			// dont want to write a whole new test for things that should have
+			// errors just to avoid this mess. errors.Is doesn't work either
+			if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
+				t.Fatalf("expected %v, but got %v", test.err, err)
+			} else if test.err != nil && err != nil {
+				if test.err.Error() != err.Error() {
+					t.Fatalf("expected %v, but got %v", test.err, err)
+				} else {
+					return
+				}
+			}
+
+			if !reflect.DeepEqual(uinfo.Groups, test.groups) {
+				t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups)
+			}
+			if !reflect.DeepEqual(uinfo.UserID, test.uid) {
+				t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID)
+			}
+			if !reflect.DeepEqual(uinfo.UserName, test.uname) {
+				t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName)
+			}
+		})
+	}
+}
+
+func TestAuthenticate_CleanCache(t *testing.T) {
+	// this deserves its own test bc it has gross setup required
+	t.Run("should clean", func(t *testing.T) {
+		a := NewTestAuth(t)
+		now := time.Now()
+		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
+		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
+		a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
+		a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
+		a.lastCacheClean = now.Add(-45 * time.Minute)
+
+		_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
+		if a.lastCacheClean.Sub(now) <= time.Nanosecond {
+			t.Fatalf("cache should have been cleaned")
+		}
+		if _, ok := a.groupsCache["oldy"]; ok {
+			t.Errorf("oldy should have been deleted")
+		}
+		if _, ok := a.groupsCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+		if _, ok := a.tokenCache["oldy"]; ok {
+			t.Errorf("oldy should have been deleted")
+		}
+		if _, ok := a.tokenCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+	})
+	t.Run("shouldn't clean", func(t *testing.T) {
+		a := NewTestAuth(t)
+		now := time.Now()
+		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
+		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
+		a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
+		a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
+		a.lastCacheClean = now
+
+		_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
+		if a.lastCacheClean.Sub(now) >= time.Nanosecond {
+			t.Fatalf("cache should not have been cleaned")
+		}
+		if _, ok := a.groupsCache["oldy"]; !ok {
+			t.Errorf("oldy should not have been deleted")
+		}
+		if _, ok := a.groupsCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+		if _, ok := a.tokenCache["oldy"]; !ok {
+			t.Errorf("oldy should not have been deleted")
+		}
+		if _, ok := a.tokenCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+	})
+
+}
+
+func TestGetGroups(t *testing.T) {
+	a := NewTestAuth(t)
+	a.groupsCache = map[string]cachedGroups{
+		"the world is changed": {
+			cacheTime: time.Now(),
+			groups: []Group{
+				{
+					GroupID:   "i feel it in the water",
+					GroupName: "i feel it in the earth",
+				},
+			},
+		},
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := json.Marshal(
+			Groups{
+				Groups: []Group{
+					{
+						GroupID:   "much that once was is lost",
+						GroupName: "for none now live who remember it",
+					},
+				},
+			},
+		)
+		if err != nil {
+			t.Fatalf("unexpected error marshalling groups response: %v", err)
+		}
+		fmt.Fprintf(w, "%s", body)
+	}))
+	a.groupEndpoint = srv.URL
+
+	for name, test := range map[string]struct {
+		token  string
+		groups []Group
+	}{
+		"InCache": {
+			token: "the world is changed",
+			groups: []Group{
+				{
+					GroupID:   "i feel it in the water",
+					GroupName: "i feel it in the earth",
+				},
+			},
+		},
+		"NotInCache": {
+			token: "i smell it in the air",
+			groups: []Group{
+				{
+					GroupID:   "much that once was is lost",
+					GroupName: "for none now live who remember it",
+				},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
+				t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
+			}
+		})
+	}
+
+}
+
+func TestDecodeHex(t *testing.T) {
+	t.Run("cantDecode", func(t *testing.T) {
+		_, err := decodeHex("gggg")
+		if err == nil {
+			t.Fatalf("expected err cannot decode slice, got nil")
+		}
+	})
+	t.Run("tooSmall", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEF")
+		if err == nil {
+			t.Fatalf("expected err wrong length, got nil")
+		}
+	})
+	t.Run("tooBig", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if err == nil {
+			t.Fatalf("expected err wrong length, got nil")
+		}
+	})
+	t.Run("justRight", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if err != nil {
+			t.Fatalf("expected nil, got %v", err)
+		}
+	})
+}
+
+func TestHandlers(t *testing.T) {
+	a := NewTestAuth(t)
+	t.Run("login", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/login", nil)
+		w := httptest.NewRecorder()
+		a.Login(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
+		if got, err := resp.Location(); err != nil || got.String() != redirect {
+			t.Fatalf("expected %v, got %v", redirect, got.Path)
+		}
+	})
+	t.Run("logout", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/logout", nil)
+		w := httptest.NewRecorder()
+		req.AddCookie(
+			&http.Cookie{
+				Name:     a.cookieName,
+				Value:    "test",
+				Path:     "/",
+				Secure:   true,
+				HttpOnly: true,
+				SameSite: http.SameSiteStrictMode,
+				Expires:  time.Unix(3000000, 0),
+			},
+		)
+		a.groupsCache["test"] = cachedGroups{}
+		a.tokenCache["test"] = cachedToken{time.Now(), &oauth2.Token{}}
+		a.Logout(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
+		if got, err := resp.Location(); err != nil || got.String() != redirect {
+			t.Fatalf("expected %v, got %v", redirect, got.Path)
+		}
+		for _, c := range resp.Cookies() {
+			if c.Name == a.cookieName {
+				if c.Value != "" {
+					t.Fatalf("cookie not set to empty value!")
+				}
+				want := time.Unix(0, 0).Unix()
+				got := c.Expires.Unix()
+				if want != got {
+					t.Fatalf("expected %v, got %v", want, got)
+				}
+				break
+			}
+		}
+		if _, ok := a.groupsCache["test"]; ok {
+			t.Fatalf("groups not deleted!")
+		}
+		if _, ok := a.tokenCache["test"]; ok {
+			t.Fatalf("token not deleted!")
+		}
+	})
+	t.Run("redirectGood", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/redirect", nil)
+		w := httptest.NewRecorder()
+		tkn := jwt.New(jwt.SigningMethodHS256)
+		claims := tkn.Claims.(jwt.MapClaims)
+		claims["oid"] = "user id"
+		claims["name"] = "user name"
+		expiresIn := 2 * time.Hour
+		exp := time.Now().Add(expiresIn)
+		expiry := strconv.Itoa(int(exp.Unix()))
+		claims["exp"] = expiry
+		fresh, err := tkn.SignedString(a.SecretKey())
+		if err != nil {
+			t.Fatalf("unexpected error when signing token %v", err)
+		}
+		freshToken := oauth2.Token{
+			AccessToken:  fresh,
+			RefreshToken: "blah",
+			Expiry:       exp,
+		}
+
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(body))
+		}))
+		a.oAuthConfig.Endpoint.TokenURL = srv.URL
+		a.Redirect(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		if got, err := resp.Location(); err != nil || got.String() != "/" {
+			t.Fatalf("expected %v, got %v", "/", got.Path)
+		}
+		cachedToken := a.tokenCache[fresh].token
+		if cachedToken.AccessToken != freshToken.AccessToken {
+			t.Fatalf("expected %v, got %v", freshToken.AccessToken, cachedToken.AccessToken)
+		}
+		if cachedToken.RefreshToken != freshToken.RefreshToken {
+			t.Fatalf("expected %v, got %v", freshToken.RefreshToken, cachedToken.RefreshToken)
+		}
+		if cachedToken.Expiry.Sub(freshToken.Expiry) > time.Second {
+			t.Fatalf("expected %v, got %v", freshToken.Expiry, cachedToken.Expiry)
+		}
+	})
+
+	t.Run("redirectBad", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/redirect", nil)
+		w := httptest.NewRecorder()
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "Server Error", http.StatusInternalServerError)
+		}))
+		a.oAuthConfig.Endpoint.TokenURL = srv.URL
+		a.Redirect(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusBadRequest {
+			t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
+		}
+	})
+
+}
+
+// This type is used for mocking ServerTransportStreams in tests
+type ServerTransportStream struct {
+	md     metadata.MD
+	method string
+}
+
+func NewServerTransportStream() *ServerTransportStream {
+	return &ServerTransportStream{
+		md:     metadata.MD{},
+		method: "test",
+	}
+}
+
+func (s *ServerTransportStream) Method() string {
+	return s.method
+}
+
+func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
+	s.md = md
+	return nil
+}
+
+func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
+	_ = md
+	return nil
+}
+
+func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
+	_ = md
+	return nil
+}

authz/authorization.go

@@ -0,0 +1,142 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authz
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/molecula/featurebase/v3/authn"
+
+	"gopkg.in/yaml.v2"
+)
+
+type GroupPermissions struct {
+	Permissions map[string]map[string]Permission `yaml:"user-groups"`
+	Admin       string                           `yaml:"admin"`
+}
+
+type Permission string
+
+const (
+	None  Permission = ""
+	Read  Permission = "read"
+	Write Permission = "write"
+	Admin Permission = "admin"
+)
+
+// Satisfies returns whether `p` satisfies the permissions required by `b`
+func (p Permission) Satisfies(b Permission) bool {
+	switch p {
+	case "":
+		return b == ""
+	case "read":
+		return b == "" || b == "read"
+	case "write":
+		return b == "" || b == "read" || b == "write"
+	case "admin":
+		return b == "" || b == "read" || b == "write" || b == "admin"
+	}
+	return false
+}
+
+func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
+	permsData, err := ioutil.ReadAll(permsFile)
+
+	if err != nil {
+		return fmt.Errorf("reading permissions failed with error: %s", err)
+	}
+
+	err = yaml.UnmarshalStrict(permsData, &p)
+	if err != nil {
+		return fmt.Errorf("unmarshalling permissions failed with error: %s", err)
+	}
+
+	return
+}
+
+func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) {
+	groups := user.Groups
+	if admin := p.IsAdmin(groups); admin {
+		return Admin, nil
+	}
+
+	allPermissions := map[Permission]bool{
+		Write: false,
+		Read:  false,
+	}
+
+	if len(groups) == 0 {
+		return None, fmt.Errorf("user is not part of any groups in identity provider")
+	}
+
+	var groupsDenied []string
+	for _, group := range groups {
+		if _, ok := p.Permissions[group.GroupID]; ok {
+			if perm, ok := p.Permissions[group.GroupID][index]; ok {
+				allPermissions[perm] = true
+			} else {
+				return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index)
+			}
+		} else {
+			groupsDenied = append(groupsDenied, group.GroupID)
+		}
+	}
+
+	if len(groupsDenied) == len(groups) {
+		return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
+	}
+
+	if allPermissions[Write] {
+		return Write, nil
+	} else if allPermissions[Read] {
+		return Read, nil
+	} else {
+		return None, fmt.Errorf("no permissions found")
+	}
+}
+
+func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
+	for _, group := range groups {
+		if p.Admin == group.GroupID {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
+	// if user is admin, find all indexes in permissions file and return them
+	if p.IsAdmin(groups) {
+		for groupId := range p.Permissions {
+			for index := range p.Permissions[groupId] {
+				indexList = append(indexList, index)
+			}
+		}
+		return indexList
+	}
+
+	for _, group := range groups {
+		if _, ok := p.Permissions[group.GroupID]; ok {
+			for index, permission := range p.Permissions[group.GroupID] {
+				if permission.Satisfies(desiredPermission) {
+					indexList = append(indexList, index)
+				}
+			}
+		}
+	}
+	return indexList
+}

authz/authorization_test.go

@@ -0,0 +1,316 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package authz_test
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/molecula/featurebase/v3/authn"
+	"github.com/molecula/featurebase/v3/authz"
+)
+
+func TestAuth_ReadPermissionsFile(t *testing.T) {
+
+	singleInput := `user-groups:
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	multiInput := `user-groups:
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+    "test2": "write"
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "write"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	singlePermission := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	multiPermission := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
+			"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		input  string
+		output authz.GroupPermissions
+	}{
+		{singleInput, singlePermission},
+		{multiInput, multiPermission},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			permFile := strings.NewReader(test.input)
+
+			var p authz.GroupPermissions
+			err := p.ReadPermissionsFile(permFile)
+			if err != nil {
+				t.Fatalf("readPermissionsFile error: %s", err)
+			}
+
+			if !reflect.DeepEqual(p, test.output) {
+				t.Fatalf("expected output %s, but got %s", test.output, p)
+			}
+		},
+		)
+	}
+}
+
+func TestAuth_GetPermissions(t *testing.T) {
+
+	// initializes different example of permissions file in yaml
+	permissions1 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions2 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "write"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions3 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "write"
+    "test2": "read"
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions4 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": ""
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	// initializes groups that are returned from identity provider
+	groupName := "name"
+	groupsList1 := []authn.Group{}
+	groupsList2 := []authn.Group{{
+		GroupID:   "fake-group",
+		GroupName: groupName}}
+	groupsList3 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName},
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName},
+	}
+	groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}}
+
+	tests := []struct {
+		yamlData   string
+		groups     []authn.Group
+		index      string
+		userAccess authz.Permission
+		err        string
+	}{
+		{
+			permissions1,
+			groupsList1,
+			"test",
+			authz.None,
+			"user is not part of any groups in identity provider",
+		},
+		{
+			permissions1,
+			groupsList3,
+			"test1",
+			authz.None,
+			"does not have permission to index",
+		},
+		{
+			permissions2,
+			groupsList2,
+			"test",
+			authz.None,
+			"does not have permission to FeatureBase",
+		},
+		{
+			permissions1,
+			groupsList3,
+			"test",
+			authz.Read,
+			"",
+		},
+		{
+			permissions2,
+			groupsList3,
+			"test",
+			authz.Write,
+			"",
+		},
+		{
+			permissions3,
+			groupsList4,
+			"test",
+			authz.Admin,
+			"",
+		},
+		{
+			permissions4,
+			groupsList3,
+			"test",
+			authz.None,
+			"no permissions found",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+
+			permFile := strings.NewReader(test.yamlData)
+
+			var p authz.GroupPermissions
+			if err := p.ReadPermissionsFile(permFile); err != nil {
+				t.Errorf("Error: %s", err)
+			}
+
+			p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index)
+
+			if p1 != test.userAccess {
+				t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1)
+			}
+
+			if err != nil {
+				if !strings.Contains(err.Error(), test.err) {
+					t.Errorf("expected error to contain %s, but got %s", test.err, err.Error())
+				}
+			}
+
+		})
+	}
+}
+
+func TestAuth_IsAdmin(t *testing.T) {
+
+	group1 := []authn.Group{
+		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
+	}
+
+	group2 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
+	}
+
+	groupPermissions := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		groups           []authn.Group
+		groupPermissions authz.GroupPermissions
+		output           bool
+	}{
+		{
+			group1, groupPermissions, true,
+		},
+		{
+			group2, groupPermissions, false,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			p := test.groupPermissions
+			resp := p.IsAdmin(test.groups)
+			if resp != test.output {
+				t.Errorf("expected %t, but got %t", test.output, resp)
+			}
+		})
+	}
+}
+
+func TestAuth_GetAuthorizedIndexList(t *testing.T) {
+
+	group1 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
+	}
+
+	group2 := []authn.Group{
+		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
+	}
+
+	group3 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"},
+	}
+
+	p := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {
+				"test1": authz.Read,
+				"test2": authz.Write,
+			},
+			"dca35310-ecda-4f23-86cd-876aee559900": {
+				"test3": authz.Read,
+			},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		groups     []authn.Group
+		permission authz.Permission
+		output     []string
+	}{
+		{
+			group1,
+			authz.Read,
+			[]string{"test1", "test2"},
+		},
+		{
+			group1,
+			authz.Write,
+			[]string{"test2"},
+		},
+		{
+			group3,
+			authz.Write,
+			nil,
+		},
+		{
+			group2,
+			authz.Read,
+			[]string{"test1", "test2", "test3"},
+		},
+		{
+			group2,
+			authz.Write,
+			[]string{"test1", "test2", "test3"},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+
+			indexList := p.GetAuthorizedIndexList(test.groups, test.permission)
+			sort.Strings(indexList)
+
+			if !reflect.DeepEqual(indexList, test.output) {
+				t.Errorf("expected %s, but got %s", test.output, indexList)
+			}
+		})
+	}
+
+}

boltdb/attrstore.go

@@ -1,423 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package boltdb
-
-import (
-	"bytes"
-
-	"encoding/binary"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/cespare/xxhash"
-
-	"github.com/boltdb/bolt"
-	"github.com/pilosa/pilosa/v2"
-	"github.com/pkg/errors"
-)
-
-// attrBlockSize is the size of attribute blocks for anti-entropy.
-const attrBlockSize = 100
-
-// attrCache represents a cache for attributes.
-type attrCache struct {
-	mu    sync.RWMutex
-	attrs map[uint64]map[string]interface{}
-}
-
-// Get returns the cached attributes for a given id.
-func (c *attrCache) Get(id uint64) map[string]interface{} {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	attrs := c.attrs[id]
-	if attrs == nil {
-		return nil
-	}
-
-	// Make a copy for safety
-	ret := make(map[string]interface{})
-	for k, v := range attrs {
-		ret[k] = v
-	}
-	return ret
-}
-
-// Set updates the cached attributes for a given id.
-func (c *attrCache) Set(id uint64, attrs map[string]interface{}) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.attrs[id] = attrs
-}
-
-// attrStore represents a storage layer for attributes.
-type attrStore struct {
-	mu        sync.RWMutex
-	path      string
-	db        *bolt.DB
-	attrCache *attrCache
-}
-
-// newAttrCache returns a new instance of AttrCache.
-func newAttrCache() *attrCache {
-	return &attrCache{
-		attrs: make(map[uint64]map[string]interface{}),
-	}
-}
-
-// NewAttrStore returns a new instance of AttrStore.
-func NewAttrStore(path string) pilosa.AttrStore {
-	return &attrStore{
-		path:      path,
-		attrCache: newAttrCache(),
-	}
-}
-
-// Path returns path to the store's data file.
-func (s *attrStore) Path() string { return s.path }
-
-// Open opens and initializes the store.
-func (s *attrStore) Open() error {
-	// Open storage.
-	db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return errors.Wrap(err, "opening storage")
-	}
-	s.db = db
-
-	// Initialize database.
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte("attrs"))
-		return err
-	}); err != nil {
-		return errors.Wrap(err, "initializing")
-	}
-
-	return nil
-}
-
-// Close closes the store.
-func (s *attrStore) Close() error {
-	if s.db != nil {
-		s.db.Close()
-	}
-	return nil
-}
-
-// Attrs returns a set of attributes by ID.
-func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	// Check cache for map.
-	if m = s.attrCache.Get(id); m != nil {
-		return m, nil
-	}
-
-	// Find attributes from storage.
-	if err = s.db.View(func(tx *bolt.Tx) error {
-		m, err = txAttrs(tx, id)
-		return err
-	}); err != nil {
-		return nil, errors.Wrap(err, "finding attributes")
-	}
-
-	// Add to cache.
-	s.attrCache.Set(id, m)
-
-	return m, nil
-}
-
-// SetAttrs sets attribute values for a given ID.
-func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error {
-	// Ignore empty maps.
-	if len(m) == 0 {
-		return nil
-	}
-
-	// Check if the attributes already exist under a read-only lock.
-	if attr, err := s.Attrs(id); err != nil {
-		return errors.Wrap(err, "checking attrs")
-	} else if attr != nil && mapContains(attr, m) {
-		return nil
-	}
-
-	// Obtain write lock.
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	var attr map[string]interface{}
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		tmp, err := txUpdateAttrs(tx, id, m)
-		if err != nil {
-			return err
-		}
-		attr = tmp
-
-		return nil
-	}); err != nil {
-		return errors.Wrap(err, "updating store")
-	}
-
-	// Swap attributes map in cache.
-	s.attrCache.Set(id, attr)
-
-	return nil
-}
-
-// SetBulkAttrs sets attribute values for a set of ids.
-func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	attrs := make(map[uint64]map[string]interface{})
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		// Collect and sort keys.
-		ids := make([]uint64, 0, len(m))
-		for id := range m {
-			ids = append(ids, id)
-		}
-		sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
-
-		// Update attributes for each id.
-		for _, id := range ids {
-			attr, err := txUpdateAttrs(tx, id, m[id])
-			if err != nil {
-				return err
-			}
-			attrs[id] = attr
-		}
-
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	// Swap attributes map in cache.
-	for id, attr := range attrs {
-		s.attrCache.Set(id, attr)
-	}
-
-	return nil
-}
-
-// Blocks returns a list of all blocks in the store.
-func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) {
-	err = s.db.View(func(tx *bolt.Tx) error {
-		// Wrap cursor to segment by block.
-		cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize)
-
-		// Iterate over each block.
-		for cur.nextBlock() {
-			block := pilosa.AttrBlock{ID: cur.blockID()}
-
-			// Compute checksum of every key/value in block.
-			h := xxhash.New()
-			for k, v := cur.next(); k != nil; k, v = cur.next() {
-				// hash function writes don't usually need to be checked
-				_, _ = h.Write(k)
-				_, _ = h.Write(v)
-			}
-			block.Checksum = h.Sum(nil)
-
-			// Append block.
-			blocks = append(blocks, block)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "getting blocks")
-	}
-	return blocks, nil
-}
-
-// BlockData returns all data for a single block.
-func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) {
-	m = make(map[uint64]map[string]interface{})
-
-	// Start read-only transaction.
-	err = s.db.View(func(tx *bolt.Tx) error {
-		// Move to the start of the block.
-		min := u64tob(i * attrBlockSize)
-		max := u64tob((i + 1) * attrBlockSize)
-		cur := tx.Bucket([]byte("attrs")).Cursor()
-		for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
-			// Exit if we're past the end of the block.
-			if bytes.Compare(k, max) != -1 {
-				break
-			}
-
-			// Decode attribute map and associate with id.
-			attrs, err := pilosa.DecodeAttrs(v)
-			if err != nil {
-				return errors.Wrap(err, "decoding attrs")
-			}
-			m[btou64(k)] = attrs
-
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "getting block data")
-	}
-	return m, nil
-}
-
-// txAttrs returns a map of attributes for an id.
-func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
-	v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
-	if v == nil {
-		return emptyMap, nil
-	}
-	return pilosa.DecodeAttrs(v)
-}
-
-// txUpdateAttrs updates the attributes for an id.
-// Returns the new combined set of attributes for the id.
-func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
-	attr, err := txAttrs(tx, id)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create a new map if it is empty so we don't update emptyMap.
-	if len(attr) == 0 {
-		attr = make(map[string]interface{}, len(m))
-	}
-
-	// Merge attributes with original values.
-	// Nil values should delete keys.
-	for k, v := range m {
-		if v == nil {
-			delete(attr, k)
-			continue
-		}
-
-		switch v := v.(type) {
-		case int:
-			attr[k] = int64(v)
-		case uint:
-			attr[k] = int64(v)
-		case uint64:
-			attr[k] = int64(v)
-		case string, int64, bool, float64:
-			attr[k] = v
-		default:
-			return nil, fmt.Errorf("invalid attr type: %T", v)
-		}
-	}
-
-	// Marshal and save new values.
-	buf, err := pilosa.EncodeAttrs(attr)
-	if err != nil {
-		return nil, errors.Wrap(err, "encoding attrs")
-	}
-	if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
-		return nil, errors.Wrap(err, "saving attrs")
-	}
-	return attr, nil
-}
-
-// u64tob encodes v to big endian encoding.
-func u64tob(v uint64) []byte {
-	b := make([]byte, 8)
-	binary.BigEndian.PutUint64(b, v)
-	return b
-}
-
-// btou64 decodes b from big endian encoding.
-func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
-
-// emptyMap is a reusable map that contains no keys.
-var emptyMap = make(map[string]interface{})
-
-// mapContains returns true if all keys & values of subset are in m.
-func mapContains(m, subset map[string]interface{}) bool {
-	for k, v := range subset {
-		value, ok := m[k]
-		if !ok || value != v {
-			return false
-		}
-	}
-	return true
-}
-
-// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
-type blockCursor struct {
-	cur  *bolt.Cursor
-	base uint64
-	n    uint64
-
-	buf struct {
-		key    []byte
-		value  []byte
-		filled bool
-	}
-}
-
-// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
-func newBlockCursor(c *bolt.Cursor, n int) blockCursor { // nolint: unparam
-	cur := blockCursor{
-		cur: c,
-		n:   uint64(n),
-	}
-	cur.buf.key, cur.buf.value = c.First()
-	cur.buf.filled = true
-	return cur
-}
-
-// blockID returns the current block ID. Only valid after call to nextBlock().
-func (cur *blockCursor) blockID() uint64 { return cur.base }
-
-// nextBlock moves the cursor to the next block.
-// Returns true if another block exists, otherwise returns false.
-func (cur *blockCursor) nextBlock() bool {
-	if cur.buf.key == nil {
-		return false
-	}
-
-	cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
-	return true
-}
-
-// next returns the next key/value within the block.
-// Returns nils at the end of the block.
-func (cur *blockCursor) next() (key, value []byte) {
-	// Use buffered value, if set.
-	if cur.buf.filled {
-		key, value = cur.buf.key, cur.buf.value
-		cur.buf.filled = false
-		return key, value
-	}
-
-	// Read next key.
-	key, value = cur.cur.Next()
-
-	// Fill buffer for EOF.
-	if key == nil {
-		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
-		return nil, nil
-	}
-
-	// Parse key and buffer if outside of block.
-	id := binary.BigEndian.Uint64(key)
-	if id/cur.n > cur.base {
-		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
-		return nil, nil
-	}
-
-	return key, value
-}

boltdb/translate.go

@@ -1,39 +1,49 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Copyright 2021 Molecula Corp. All rights reserved.
 package boltdb
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/boltdb/bolt"
-	"github.com/pilosa/pilosa/v2"
+	"github.com/molecula/featurebase/v3"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
+
+	"runtime/pprof"
 )
 
+var _ = pprof.StartCPUProfile
+
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
 	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
+
+	// ErrTranslateKeyNotFound is returned when translating key
+	// and the underlying store returns an empty set
+	ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set")
+
+	bucketKeys = []byte("keys")
+	bucketIDs  = []byte("ids")
+)
+
+const (
+	// snapshotExt is the file extension used for an in-process snapshot.
+	snapshotExt = ".snapshotting"
+
+	errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found"
 )
 
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
-	s := NewTranslateStore(index, field)
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) {
+	s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@@ -45,46 +55,69 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
 var _ pilosa.TranslateStore = &TranslateStore{}
 
 // TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// An empty string will be converted into the sentinel byte slice:
+// var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
+// }
 type TranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
 
-	index string
-	field string
+	index       string
+	field       string
+	partitionID int
+	partitionN  int
 
 	once    sync.Once
 	closing chan struct{}
 
-	readOnly    bool
-	writeNotify chan struct{}
+	readOnly     bool
+	fsyncEnabled bool
+	writeNotify  chan struct{}
 
 	// File path to database file.
 	Path string
 }
 
 // NewTranslateStore returns a new instance of TranslateStore.
-func NewTranslateStore(index, field string) *TranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore {
 	return &TranslateStore{
-		index:       index,
-		field:       field,
-		closing:     make(chan struct{}),
-		writeNotify: make(chan struct{}),
+		index:        index,
+		field:        field,
+		partitionID:  partitionID,
+		partitionN:   partitionN,
+		closing:      make(chan struct{}),
+		writeNotify:  make(chan struct{}),
+		fsyncEnabled: fsyncEnabled,
 	}
 }
 
 // Open opens the translate file.
 func (s *TranslateStore) Open() (err error) {
+
+	// add the path to the problem database if we panic handling it.
+	defer func() {
+		r := recover()
+		if r != nil {
+			panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r))
+		}
+	}()
+
 	if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
-	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
+	} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil {
 		return errors.Wrapf(err, "open file: %s", err)
 	}
 
 	// Initialize buckets.
 	if err := s.db.Update(func(tx *bolt.Tx) error {
-		if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil {
+		if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
 			return err
-		} else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil {
+		} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
 			return err
 		}
 		return nil
@@ -108,6 +141,11 @@ func (s *TranslateStore) Close() (err error) {
 	return nil
 }
 
+// PartitionID returns the partition id the store was initialized with.
+func (s *TranslateStore) PartitionID() int {
+	return s.partitionID
+}
+
 // ReadOnly returns true if the store is in read-only mode.
 func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
@@ -135,109 +173,127 @@ func (s *TranslateStore) Size() int64 {
 	return tx.Size()
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
-func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
-	// Find id by key under read lock.
-	if err := s.db.View(func(tx *bolt.Tx) error {
-		id = findIDByKey(tx.Bucket([]byte("keys")), key)
-		return nil
-	}); err != nil {
-		return 0, err
-	} else if id != 0 {
-		return id, nil
-	}
-
-	if s.ReadOnly() {
-		return 0, pilosa.ErrTranslateStoreReadOnly
-	}
-
-	// Find or create id under write lock.
-	var written bool
-	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		if id = findIDByKey(bkt, key); id != 0 {
-			return nil
-		} else if id, err = bkt.NextSequence(); err != nil {
-			return err
-		} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
-			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
-			return err
+// FindKeys looks up the ID for each key.
+// Keys are not created if they do not exist.
+// Missing keys are not considered errors, so the length of the result may be less than that of the input.
+func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
+	result := make(map[string]uint64, len(keys))
+	err := s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
+		}
+		for _, key := range keys {
+			id, _ := findIDByKey(bkt, key)
+			if id == 0 {
+				// The key does not exist.
+				continue
+			}
+
+			result[key] = id
 		}
-		written = true
 		return nil
-	}); err != nil {
-		return 0, err
+	})
+	if err != nil {
+		return nil, err
 	}
 
-	if written {
-		s.notifyWrite()
-	}
-
-	return id, nil
+	return result, nil
 }
 
-// TranslateKeys converts a string key to an integer ID.
-// If key does not have an associated id then one is created.
-func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
-	if len(keys) == 0 {
-		return nil, nil
-	}
-
-	// Allocate slice for ID mapping.
-	ids = make([]uint64, len(keys))
-
-	// Find ids by key under read lock.
-	var found int
-	if err := s.db.View(func(tx *bolt.Tx) error {
-		bkt := tx.Bucket([]byte("keys"))
-		for i, key := range keys {
-			if id := findIDByKey(bkt, key); id != 0 {
-				ids[i] = id
-				found++
-			}
-		}
-		return nil
-	}); err != nil {
-		return nil, err
-	} else if found == len(keys) {
-		return ids, nil
-	}
+// translateTransactionSize governs the number of writes to a single
+// boltDB bucket we will make in a single db.Update(), before starting
+// a new Update. We do this because Put() is quadratic, but Commit is
+// expensive enough that we want to do a fair number of updates before
+// paying for it.
+const translateTransactionSize = 16384
 
+// CreateKeys maps all keys to IDs, creating the IDs if they do not exist.
+// If the translator is read-only, this will return an error.
+func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
 	if s.ReadOnly() {
-		return ids, pilosa.ErrTranslateStoreReadOnly
+		return nil, pilosa.ErrTranslateStoreReadOnly
 	}
 
-	// Find or create ids under write lock if any keys were not found.
-	var written bool
-	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		bkt := tx.Bucket([]byte("keys"))
-		for i, key := range keys {
-			if ids[i] != 0 {
-				continue
+	written := false
+	result := make(map[string]uint64, len(keys))
+	idScratch := make([]byte, translateTransactionSize*8)
+	for len(keys) > 0 {
+		// boltdb performs badly if you write really large numbers of
+		// keys all at once...
+		err := s.db.Update(func(tx *bolt.Tx) error {
+			keyBucket := tx.Bucket(bucketKeys)
+			if keyBucket == nil {
+				return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
 			}
-
-			if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
-				continue
-			} else if ids[i], err = bkt.NextSequence(); err != nil {
-				return err
-			} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
-				return err
-			} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
-				return err
+			idBucket := tx.Bucket(bucketIDs)
+			if idBucket == nil {
+				return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
 			}
-			written = true
+			puts := 0
+			for idx, key := range keys {
+				id, boltKey := findIDByKey(keyBucket, key)
+				if id != 0 {
+					result[key] = id
+					continue
+				}
+				id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+				idBytes := idScratch[puts*8 : puts*8+8]
+				binary.BigEndian.PutUint64(idBytes, id)
+				puts++
+				if err := keyBucket.Put(boltKey, idBytes); err != nil {
+					return err
+				} else if err := idBucket.Put(idBytes, boltKey); err != nil {
+					return err
+				}
+				result[key] = id
+				written = true
+				if puts == translateTransactionSize {
+					keys = keys[idx+1:]
+					return nil
+				}
+			}
+			keys = keys[len(keys):]
+			return nil
+		})
+		if err != nil {
+			return nil, err
 		}
-		return nil
-	}); err != nil {
-		return nil, err
 	}
-
 	if written {
 		s.notifyWrite()
 	}
-	return ids, nil
+
+	return result, nil
+}
+
+// Match finds the IDs of all keys matching a filter.
+func (s *TranslateStore) Match(filter func([]byte) bool) ([]uint64, error) {
+	var matches []uint64
+	err := s.db.View(func(tx *bolt.Tx) error {
+		// This uses the id bucket instead of the key bucket so that matches are produced in sorted order.
+		idBucket := tx.Bucket(bucketIDs)
+		if idBucket == nil {
+			return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
+		}
+
+		return idBucket.ForEach(func(id, key []byte) error {
+			if bytes.Equal(key, emptyKey) {
+				key = nil
+			}
+
+			if filter(key) {
+				matches = append(matches, btou64(id))
+			}
+
+			return nil
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return matches, nil
 }
 
 // TranslateID converts an integer ID to a string key.
@@ -248,7 +304,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) {
 		return "", err
 	}
 	defer func() { _ = tx.Rollback() }()
-	return findKeyByID(tx.Bucket([]byte("ids")), id), nil
+	return findKeyByID(tx.Bucket(bucketIDs), id), nil
 }
 
 // TranslateIDs converts a list of integer IDs to a list of string keys.
@@ -263,9 +319,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 	}
 	defer func() { _ = tx.Rollback() }()
 
+	bucket := tx.Bucket(bucketIDs)
+
 	keys := make([]string, len(ids))
 	for i, id := range ids {
-		keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id)
+		keys[i] = findKeyByID(bucket, id)
 	}
 	return keys, nil
 }
@@ -273,9 +331,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 // ForceSet writes the id/key pair to the store even if read only. Used by replication.
 func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
-		if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil {
+		if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
 			return err
-		} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
+		} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil {
 			return err
 		}
 		return nil
@@ -286,7 +344,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	return nil
 }
 
-// Reader returns a reader that streams the underlying data file.
+// EntryReader returns a reader that streams the underlying data file.
 func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@@ -311,9 +369,7 @@ func (s *TranslateStore) notifyWrite() {
 // MaxID returns the highest id in the store.
 func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
-		if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
-			max = btou64(key)
-		}
+		max = maxID(tx)
 		return nil
 	}); err != nil {
 		return 0, err
@@ -321,6 +377,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
 
+// WriteTo writes the contents of the store to the writer.
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+	return tx.WriteTo(w)
+}
+
+// ReadFrom reads the content and overwrites the existing store.
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+	// Close store.
+	if err := s.Close(); err != nil {
+		return 0, errors.Wrap(err, "closing store")
+	}
+
+	// Create a temporary file to snapshot to.
+	snapshotPath := s.Path + snapshotExt
+	file, err := os.Create(snapshotPath)
+	if err != nil {
+		return n, errors.Wrap(err, "creating snapshot file")
+	}
+
+	// Write payload to snapshot.
+	if n, err = io.Copy(file, r); err != nil {
+		file.Close()
+		return n, errors.Wrap(err, "snapshot write to")
+	}
+
+	// we close the file here so we don't still have it open when trying
+	// to open it in a moment.
+	file.Close()
+
+	// Move snapshot to data file location.
+	if err := os.Rename(snapshotPath, s.Path); err != nil {
+		return n, errors.Wrap(err, "renaming snapshot")
+	}
+
+	// Re-open the store.
+	if err := s.Open(); err != nil {
+		return n, errors.Wrap(err, "re-opening store")
+	}
+
+	return n, nil
+}
+
+// MaxID returns the highest id in the store.
+func maxID(tx *bolt.Tx) uint64 {
+	if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil {
+		return btou64(key)
+	}
+	return 0
+}
+
 type TranslateEntryReader struct {
 	ctx    context.Context
 	store  *TranslateStore
@@ -353,7 +464,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 		var found bool
 		if err := r.store.db.View(func(tx *bolt.Tx) error {
 			// Find ID/key lookup at offset or later.
-			cur := tx.Bucket([]byte("ids")).Cursor()
+			cur := tx.Bucket(bucketIDs).Cursor()
 			key, value := cur.Seek(u64tob(r.offset))
 			if key == nil {
 				return nil
@@ -387,13 +498,43 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	}
 }
 
-func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
-	if value := bkt.Get([]byte(key)); value != nil {
-		return btou64(value)
+// emptyKey is a sentinel byte slice which stands for "" as a key.
+var emptyKey = []byte{
+	0x00, 0x00, 0x00,
+	0x4d, 0x54, 0x4d, 0x54, // MTMT
+	0x00,
+	0xc2, 0xa0, // NO-BREAK SPACE
+	0x00,
+}
+
+func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
+	var boltKey []byte
+	if key == "" {
+		boltKey = emptyKey
+	} else {
+		boltKey = []byte(key)
 	}
-	return 0
+
+	if value := bkt.Get(boltKey); value != nil {
+		return btou64(value), boltKey
+	}
+	return 0, boltKey
 }
 
 func findKeyByID(bkt *bolt.Bucket, id uint64) string {
-	return string(bkt.Get(u64tob(id)))
+	boltKey := bkt.Get(u64tob(id))
+	if bytes.Equal(boltKey, emptyKey) {
+		return ""
+	}
+	return string(boltKey)
 }
+
+// u64tob encodes v to big endian encoding.
+func u64tob(v uint64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, v)
+	return b
+}
+
+// btou64 decodes b from big endian encoding.
+func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }

boltdb/translate_test.go

@@ -1,125 +1,97 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Copyright 2021 Molecula Corp. All rights reserved.
 package boltdb_test
 
 import (
+	"bytes"
 	"context"
-	"io/ioutil"
-	"os"
+	"fmt"
+	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
-	"github.com/pilosa/pilosa/v2"
-	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/boltdb"
+	"github.com/molecula/featurebase/v3/testhook"
+	"github.com/molecula/featurebase/v3/topology"
 )
 
-func TestTranslateStore_TranslateKey(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+//var vv = pilosa.VV
+
+func TestTranslateStore_CreateKeys(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
-	// Ensure initial key translates to ID 1.
-	if id, err := s.TranslateKey("foo"); err != nil {
+	ids, err := s.CreateKeys("abc", "abc")
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
+	} else if _, ok := ids["abc"]; !ok {
+		t.Fatalf(`missing "abc"; got %v`, ids)
+	} else if len(ids) > 1 {
+		t.Fatalf("expected one key, got %d in %v", len(ids), ids)
 	}
 
-	// Ensure next key autoincrements.
-	if id, err := s.TranslateKey("bar"); err != nil {
+	// Ensure different keys translate to different IDs.
+	ids1, err := s.CreateKeys("foo", "bar")
+	if err != nil {
 		t.Fatal(err)
-	} else if got, want := id, uint64(2); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
-	}
-
-	// Ensure retranslating existing key returns original ID.
-	if id, err := s.TranslateKey("foo"); err != nil {
-		t.Fatal(err)
-	} else if got, want := id, uint64(1); got != want {
-		t.Fatalf("TranslateKey()=%d, want %d", got, want)
-	}
-}
-
-func TestTranslateStore_TranslateKeys(t *testing.T) {
-	s := MustOpenNewTranslateStore()
-	defer MustCloseTranslateStore(s)
-
-	// Ensure initial keys translate to incrementing IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
-		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
-		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
+	} else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar {
+		t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo)
 	}
 
 	// Ensure retranslation returns original IDs.
-	if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	if ids, err := s.CreateKeys("bar", "foo"); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(2); got != want {
-		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
+	} else if !reflect.DeepEqual(ids, ids1) {
+		t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1)
 	}
 
 	// Ensure retranslating with existing and non-existing keys returns correctly.
-	if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
+	if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil {
 		t.Fatal(err)
-	} else if got, want := ids[0], uint64(1); got != want {
-		t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
-	} else if got, want := ids[1], uint64(3); got != want {
-		t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
-	} else if got, want := ids[2], uint64(2); got != want {
-		t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
+	} else if got, want := ids["foo"], ids1["foo"]; got != want {
+		t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want)
+	} else if _, ok := ids["baz"]; !ok {
+		t.Fatalf(`missing translation for "baz"; got %v`, ids)
+	} else if got, want := ids["bar"], ids1["bar"]; got != want {
+		t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want)
 	}
 }
 
 func TestTranslateStore_TranslateID(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKey("foo"); err != nil {
-		t.Fatal(err)
-	} else if _, err := s.TranslateKey("bar"); err != nil {
+	ids, err := s.CreateKeys("foo", "bar", "")
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if key, err := s.TranslateID(1); err != nil {
-		t.Fatal(err)
-	} else if got, want := key, "foo"; got != want {
-		t.Fatalf("TranslateID()=%s, want %s", got, want)
-	}
-
-	if key, err := s.TranslateID(2); err != nil {
-		t.Fatal(err)
-	} else if got, want := key, "bar"; got != want {
-		t.Fatalf("TranslateID()=%s, want %s", got, want)
+	for key, id := range ids {
+		k, err := s.TranslateID(id)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if k != key {
+			t.Fatalf("TranslateID()=%s, want %s", k, key)
+		}
 	}
 }
 
 func TestTranslateStore_TranslateIDs(t *testing.T) {
-	s := MustOpenNewTranslateStore()
+	s := MustOpenNewTranslateStore(t)
 	defer MustCloseTranslateStore(s)
 
 	// Setup initial keys.
-	if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+	ids, err := s.CreateKeys("foo", "bar")
+	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Ensure IDs can be translated back to keys.
-	if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
+	if keys, err := s.TranslateIDs([]uint64{ids["foo"], ids["bar"], 1}); err != nil {
 		t.Fatal(err)
 	} else if got, want := keys[0], "foo"; got != want {
 		t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@@ -130,13 +102,114 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
 	}
 }
 
+func TestTranslateStore_FindKeys(t *testing.T) {
+	cases := []struct {
+		name   string
+		data   []string
+		lookup []string
+	}{
+		{
+			name:   "All",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h"},
+		},
+		{
+			name:   "Extra",
+			data:   []string{"plugh", "xyzzy", "h"},
+			lookup: []string{"plugh", "xyzzy", "h", "65"},
+		},
+		{
+			name:   "None",
+			data:   []string{"a", "b", "c"},
+			lookup: []string{"d", "e"},
+		},
+		{
+			name:   "Empty",
+			lookup: []string{"h"},
+		},
+		{
+			name: "LookupNothing",
+		},
+	}
+
+	for _, c := range cases {
+		c := c
+		t.Run(c.name, func(t *testing.T) {
+			s := MustOpenNewTranslateStore(t)
+			defer MustCloseTranslateStore(s)
+
+			var naiveMap map[string]uint64
+			if c.data != nil {
+				// Load in key data.
+				keys := c.data
+				ids, err := s.CreateKeys(keys...)
+				if err != nil {
+					t.Errorf("failed to import keys: %v", err)
+					return
+				}
+				if len(ids) != len(keys) {
+					t.Errorf("mapped %d keys to %d ids", len(keys), len(ids))
+					return
+				}
+				naiveMap = ids
+			}
+
+			// Compute expected lookup result.
+			result := map[string]uint64{}
+			for _, key := range c.lookup {
+				id, ok := naiveMap[key]
+				if !ok {
+					// The key is expected to be missing.
+					continue
+				}
+
+				result[key] = id
+			}
+
+			// Find the keys.
+			found, err := s.FindKeys(c.lookup...)
+			if err != nil {
+				t.Errorf("failed to find keys: %v", err)
+			} else if !reflect.DeepEqual(result, found) {
+				t.Errorf("expected %v but found %v", result, found)
+			}
+		})
+	}
+}
+
+func TestTranslateStore_MaxID(t *testing.T) {
+	s := MustOpenNewTranslateStore(t)
+	defer MustCloseTranslateStore(s)
+
+	// Generate a bunch of keys.
+	var lastk uint64
+	for i := 0; i < 1026; i++ {
+		key := strconv.Itoa(i)
+		ids, err := s.CreateKeys(key)
+		if err != nil {
+			t.Fatalf("translating %d: %v", i, err)
+		}
+		lastk = ids[key]
+	}
+
+	// Verify the max ID.
+	max, err := s.MaxID()
+	if err != nil {
+		t.Fatalf("checking max ID: %v", err)
+	}
+	if max != lastk {
+		t.Fatalf("last key is %d but max is %d", lastk, max)
+	}
+}
+
 func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Create multiple new keys.
-		if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
+		ids1, err := s.CreateKeys("foo", "bar")
+		if err != nil {
 			t.Fatal(err)
 		}
 
@@ -151,7 +224,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read first entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, ids1["foo"]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -160,21 +233,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		// Read next entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(2); got != want {
+		} else if got, want := entry.ID, ids1["bar"]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "bar"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
 		}
 
 		// Insert next key while reader is open.
-		if _, err := s.TranslateKey("baz"); err != nil {
+		ids2, err := s.CreateKeys("baz")
+		if err != nil {
 			t.Fatal(err)
 		}
 
 		// Read newly created entry.
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(3); got != want {
+		} else if got, want := entry.ID, ids2["baz"]; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "baz"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -188,7 +262,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure reader will read as soon as a new write comes in using WriteNotify().
 	t.Run("WriteNotify", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -198,20 +272,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 		}
 		defer r.Close()
 
+		// cache holds the translated key id so we can check it later
+		cache := make(chan uint64)
+
 		// Insert key in separate goroutine.
 		// Sleep momentarily to reader hangs.
 		translateErr := make(chan error)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			if _, err := s.TranslateKey("foo"); err != nil {
+			ids, err := s.CreateKeys("foo")
+			if err != nil {
 				translateErr <- err
 			}
+			cache <- ids["foo"]
 		}()
 
 		var entry pilosa.TranslateEntry
 		if err := r.ReadEntry(&entry); err != nil {
 			t.Fatal(err)
-		} else if got, want := entry.ID, uint64(1); got != want {
+		} else if got, want := entry.ID, <-cache; got != want {
 			t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
 		} else if got, want := entry.Key, "foo"; got != want {
 			t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@@ -226,7 +305,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on close.
 	t.Run("Close", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -260,7 +339,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 	// Ensure exits read on store close.
 	t.Run("StoreClose", func(t *testing.T) {
-		s := MustOpenNewTranslateStore()
+		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
 
 		// Start reader from initial position.
@@ -294,22 +373,90 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 }
 
 // MustNewTranslateStore returns a new TranslateStore with a temporary path.
-func MustNewTranslateStore() *boltdb.TranslateStore {
-	f, err := ioutil.TempFile("", "")
+func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
+	f, err := testhook.TempFile(tb, "translate-store")
 	if err != nil {
 		panic(err)
 	} else if err := f.Close(); err != nil {
 		panic(err)
 	}
 
-	s := boltdb.NewTranslateStore("I", "F")
+	s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN, false)
 	s.Path = f.Name()
 	return s
 }
 
+func TestTranslateStore_ReadWrite(t *testing.T) {
+	t.Run("WriteTo_ReadFrom", func(t *testing.T) {
+		s := MustOpenNewTranslateStore(t)
+		defer MustCloseTranslateStore(s)
+
+		batch0 := []string{}
+		for i := 0; i < 100; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+		batch1 := []string{}
+		for i := 100; i < 200; i++ {
+			batch1 = append(batch1, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.CreateKeys(batch0...)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Put the contents of the store into a buffer.
+		buf := bytes.NewBuffer(nil)
+		expN := int64(32768)
+
+		// After this, the buffer should contain batch0.
+		if n, err := s.WriteTo(buf); err != nil {
+			t.Fatalf("writing to buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// Populate the store with the keys in batch1.
+		batch1IDs, err := s.CreateKeys(batch1...)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expIDs := map[string]uint64{
+			"key50":  batch0IDs["key50"],
+			"key150": batch1IDs["key150"],
+		}
+
+		// Check the IDs for a key from each batch.
+		if ids, err := s.FindKeys("key50", "key150"); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(expIDs, ids) {
+			t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
+		}
+
+		// Reset the contents of the store with the data in the buffer.
+		if n, err := s.ReadFrom(buf); err != nil {
+			t.Fatalf("reading from buffer: %s", err)
+		} else if n != expN {
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
+		}
+
+		// This time, we expect the second key to be different because
+		// we overwrote the store, and then just set that key.
+		if ids, err := s.CreateKeys("key50", "key150"); err != nil {
+			t.Fatal(err)
+		} else if ids["key50"] != expIDs["key50"] {
+			t.Fatalf("last expected ids[key50]: %d, but got: %d", expIDs["key50"], ids["key50"])
+		} else if ids["key150"] == expIDs["key150"] {
+			t.Fatalf("last expected different ids[key150]: %d, but got: %d", expIDs["key150"], ids["key150"])
+		}
+	})
+}
+
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
-func MustOpenNewTranslateStore() *boltdb.TranslateStore {
-	s := MustNewTranslateStore()
+func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
+	s := MustNewTranslateStore(tb)
 	if err := s.Open(); err != nil {
 		panic(err)
 	}
@@ -320,7 +467,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore {
 func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 	if err := s.Close(); err != nil {
 		panic(err)
-	} else if err := os.Remove(s.Path); err != nil {
-		panic(err)
 	}
 }

broadcast.go

@@ -1,22 +1,10 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 
 import (
 	"fmt"
 
+	"github.com/molecula/featurebase/v3/topology"
 	"github.com/pkg/errors"
 )
 
@@ -26,11 +14,22 @@ type Serializer interface {
 	Unmarshal([]byte, Message) error
 }
 
+// NopSerializer represents a Serializer that doesn't do anything.
+var NopSerializer Serializer = &nopSerializer{}
+
+type nopSerializer struct{}
+
+// Marshal is a no-op implementation of Serializer Marshal method.
+func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
+
+// Unmarshal is a no-op implementation of Serializer Unmarshal method.
+func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
+
 // broadcaster is an interface for broadcasting messages.
 type broadcaster interface {
 	SendSync(Message) error
 	SendAsync(Message) error
-	SendTo(*Node, Message) error
+	SendTo(*topology.Node, Message) error
 }
 
 // Message is the interface implemented by all core pilosa types which can be serialized to messages.
@@ -49,7 +48,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
 func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
+func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
@@ -63,12 +62,14 @@ const (
 	messageTypeClusterStatus
 	messageTypeResizeInstruction
 	messageTypeResizeInstructionComplete
-	messageTypeSetCoordinator
-	messageTypeUpdateCoordinator
 	messageTypeNodeState
 	messageTypeRecalculateCaches
+	messageTypeLoadSchemaMessage
 	messageTypeNodeEvent
 	messageTypeNodeStatus
+	messageTypeTransaction
+	messageTypeResizeNodeMessage
+	messageTypeResizeAbortMessage
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -104,18 +105,22 @@ func getMessage(typ byte) Message {
 		return &ResizeInstruction{}
 	case messageTypeResizeInstructionComplete:
 		return &ResizeInstructionComplete{}
-	case messageTypeSetCoordinator:
-		return &SetCoordinatorMessage{}
-	case messageTypeUpdateCoordinator:
-		return &UpdateCoordinatorMessage{}
 	case messageTypeNodeState:
 		return &NodeStateMessage{}
 	case messageTypeRecalculateCaches:
 		return &RecalculateCaches{}
+	case messageTypeLoadSchemaMessage:
+		return &LoadSchemaMessage{}
 	case messageTypeNodeEvent:
 		return &NodeEvent{}
 	case messageTypeNodeStatus:
 		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
+	case messageTypeResizeNodeMessage:
+		return &ResizeNodeMessage{}
+	case messageTypeResizeAbortMessage:
+		return &ResizeAbortMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -143,18 +148,22 @@ func getMessageType(m Message) byte {
 		return messageTypeResizeInstruction
 	case *ResizeInstructionComplete:
 		return messageTypeResizeInstructionComplete
-	case *SetCoordinatorMessage:
-		return messageTypeSetCoordinator
-	case *UpdateCoordinatorMessage:
-		return messageTypeUpdateCoordinator
 	case *NodeStateMessage:
 		return messageTypeNodeState
 	case *RecalculateCaches:
 		return messageTypeRecalculateCaches
+	case *LoadSchemaMessage:
+		return messageTypeLoadSchemaMessage
 	case *NodeEvent:
 		return messageTypeNodeEvent
 	case *NodeStatus:
 		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
+	case *ResizeNodeMessage:
+		return messageTypeResizeNodeMessage
+	case *ResizeAbortMessage:
+		return messageTypeResizeAbortMessage
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

bsi.go

@@ -0,0 +1,283 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"math/bits"
+
+	"github.com/molecula/featurebase/v3/roaring"
+)
+
+// bsiData contains BSI-structured data.
+type bsiData []*Row
+
+// pivotDescending loops over nonzero BSI values in descending order.
+// For each value, the provided function is called with the value and a slice of the associated columns.
+// If limit or offset are not-nil, they will be applied.
+// Applying a limit or offset may modify the pointed-to value.
+func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+	// This "pivot" algorithm works by treating the BSI data as a tree.
+	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
+	// Each range is subdivided into 2 ranges of half size, which form lower branches.
+	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
+	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
+	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
+	// This function uses a depth-first search over this virtual tree.
+
+	switch {
+	case !filter.Any():
+		// There are no remaining data.
+
+	case offset != nil && *offset >= filter.Count():
+		// Skip this entire branch.
+		*offset -= filter.Count()
+
+	case limit != nil && *limit == 0:
+		// The limit has been reached.
+		// No more data is necessary.
+
+	case len(bsi) == 0:
+		// This is a leaf node.
+		cols := filter.Columns()
+		if offset != nil {
+			cols = cols[*offset:]
+			*offset = 0
+		}
+		if limit != nil {
+			if *limit < uint64(len(cols)) {
+				cols = cols[:*limit]
+			}
+			*limit -= uint64(len(cols))
+		}
+		fn(branch, cols...)
+
+	default:
+		// Pivot over the highest bit.
+		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
+		splitBit := bsi[len(bsi)-1]
+		lowerBits := bsi[:len(bsi)-1]
+		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+	}
+}
+
+/*
+// distribution generates a BSI histogram for the input.
+// TODO: I forgot what I was going to use this for.
+// Could probbably use this for:
+// - quartile queries
+// - TopN on int
+func (bsi bsiData) distribution(filter *Row) bsiData {
+	var dist bsiData
+	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+		dist.insert(count, uint64(len(values)))
+	})
+	return dist
+}
+*/
+
+var placeholderBitmap = roaring.NewBitmap()
+
+// addBSI adds two BSI bitmaps together.
+// It does not handle sign and has no concept of overflow.
+func addBSI(x, y bsiData) bsiData {
+	// Accumulate row segments.
+	segments := make([][]rowSegment, len(x)+len(y))
+	xsegs, ysegs := segments[:len(x)], segments[len(x):]
+	for i, r := range x {
+		xsegs[i] = r.segments
+	}
+	for i, r := range y {
+		ysegs[i] = r.segments
+	}
+
+	var dst bsiData
+	var xbitmaps, ybitmaps []*roaring.Bitmap
+	for {
+		// Find the next shard.
+		next := ^uint64(0)
+		for _, s := range segments {
+			if len(s) == 0 {
+				continue
+			}
+			shard := s[0].shard
+			if shard < next {
+				next = shard
+			}
+		}
+		if next == ^uint64(0) {
+			// There are no remaining shards.
+			break
+		}
+
+		// Accumulate bitmaps for this shard.
+		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
+		for i, segs := range xsegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			xsegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(xbitmaps) < i {
+				xbitmaps = append(xbitmaps, placeholderBitmap)
+			}
+			xbitmaps = append(xbitmaps, bm)
+		}
+		for i, segs := range ysegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			ysegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(ybitmaps) < i {
+				ybitmaps = append(ybitmaps, placeholderBitmap)
+			}
+			ybitmaps = append(ybitmaps, bm)
+		}
+
+		// Add the shard values together.
+		var out []*roaring.Bitmap
+		switch {
+		case len(xbitmaps) == 0:
+			// There are no values in x.
+			out = ybitmaps
+		case len(ybitmaps) == 0:
+			// There are no values in y.
+			out = xbitmaps
+		default:
+			out = roaring.Add(xbitmaps, ybitmaps)
+		}
+
+		// Convert the bitmaps to output segments.
+		for i, b := range out {
+			if !b.Any() {
+				continue
+			}
+			for len(dst) <= i {
+				dst = append(dst, NewRow())
+			}
+			dst[i].segments = append(dst[i].segments, rowSegment{
+				shard:    next,
+				writable: true,
+				data:     b,
+				n:        b.Count(),
+			})
+		}
+	}
+
+	return dst
+}
+
+// rowBuilder builds a row quickly from individual values.
+// It is optimized for the case in which values are generated sequentially.
+type rowBuilder struct {
+	bm    *roaring.Bitmap
+	mask  *[1024]uint64
+	array []uint16
+	key   uint64
+	n     int32
+}
+
+// flushKey flushes the data at the current key to the bitmap.
+func (b *rowBuilder) flushKey() {
+	var c *roaring.Container
+	switch {
+	case b.mask != nil:
+		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
+		b.mask = nil
+	case len(b.array) > 0:
+		c = roaring.NewContainerArrayCopy(b.array)
+		b.array = b.array[:0]
+	default:
+		return
+	}
+
+	if b.bm == nil {
+		b.bm = roaring.NewBitmap()
+	}
+	if old := b.bm.Containers.Get(b.key); old != nil {
+		c = roaring.Union(c, old)
+	}
+	b.bm.Containers.Put(b.key, c)
+}
+
+// Add a value to the bitmap.
+// Values must be added sequentially.
+func (b *rowBuilder) Add(v uint64) {
+	vkey := v / (1 << 16)
+	if b.key != vkey {
+		// This is a new key, so flush the old one.
+		b.flushKey()
+		b.key = vkey
+	}
+
+	if b.mask != nil {
+		// Add to the mask.
+		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
+		b.mask[uint16(v)/64] |= 1 << (v % 64)
+		return
+	}
+
+	// Add to an array.
+	b.array = append(b.array, uint16(v))
+	if len(b.array) >= roaring.ArrayMaxSize {
+		// The array is too big.
+		// Convert it to a bitmask.
+		m := [1024]uint64{}
+		for _, v := range b.array {
+			m[v/64] |= 1 << (v % 64)
+		}
+		b.n = int32(len(b.array))
+		b.array = b.array[:0]
+		b.mask = &m
+	}
+}
+
+// Build a Row from stored data.
+// This resets the builder.
+func (b *rowBuilder) Build() *Row {
+	// Flush the active key to the bitmap.
+	b.flushKey()
+
+	// Remove the bitmap and convert it to a Row.
+	bm := b.bm
+	b.bm = nil
+	if bm == nil {
+		return NewRow()
+	}
+	return NewRowFromBitmap(bm)
+}
+
+// bsiBuilder assembles BSI data.
+// It is optimized for the case in which values are generated sequentially.
+type bsiBuilder []rowBuilder
+
+// Insert a value into the BSI data.
+// Columns must be inserted sequentially, and duplicates are not allowed.
+func (b *bsiBuilder) Insert(col, val uint64) {
+	for val != 0 {
+		i := bits.TrailingZeros64(val)
+		val &^= 1 << i
+		for len(*b) <= i {
+			*b = append(*b, rowBuilder{})
+		}
+		(*b)[i].Add(col)
+	}
+}
+
+// Build BSI data.
+// This resets the builder.
+func (b *bsiBuilder) Build() bsiData {
+	builders := *b
+	*b = builders[:0]
+	rows := make(bsiData, len(builders))
+	for i := range builders {
+		rows[i] = builders[i].Build()
+	}
+	return rows
+}

bsi_test.go

@@ -0,0 +1,159 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+)
+
+// TestBSIAdd does a number of iterations. For each iteration, it
+// generates a random number of ids, and two random values for each id
+// to add together.
+func TestBSIAdd(t *testing.T) {
+	// TODO wouldn't it be cool if our test suite had a randomized
+	// burn-in mode where you could run any test which supported it
+	// with a random seed and way more iterations?
+	rnd := rand.New(rand.NewSource(99))
+	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
+	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
+	var builderA, builderB bsiBuilder
+	// a and b are generated slices of numbers to add together
+	var a, b []uint64
+	// idToIndex maps record ids to indexes in a and b
+	idToIndex := make(map[int]int)
+	// indexToID has the record id for each value in a and b
+	indexToID := []uint64{}
+
+	min := 999999999
+	max := 0
+
+	for iteration := 0; iteration < 1; iteration++ {
+		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
+			// reset generated data
+			a, b = a[:0], b[:0]
+			indexToID = indexToID[:0]
+			for k := range idToIndex {
+				delete(idToIndex, k)
+			}
+
+			// z generates the values, they can be fairly large, but are usually small
+			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
+			id := -1
+			for i := 0; true; i++ {
+				// get the next id, skipping a random amount
+				id = id + int(idZipf.Uint64()+1)
+				if id >= ShardWidth {
+					if i < min {
+						min = i
+					}
+					if max < i {
+						max = i
+					}
+					break
+				}
+				idToIndex[id] = int(i)
+				indexToID = append(indexToID, uint64(id))
+
+				// append a random value to each data slice
+				a = append(a, z.Uint64())
+				b = append(b, z.Uint64())
+			}
+
+			// build the BSIs based on the data slices and generated IDs
+			for index, id := range indexToID {
+				va, vb := a[index], b[index]
+				builderA.Insert(uint64(id), va)
+				builderB.Insert(uint64(id), vb)
+			}
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+
+			// build results from added bsiData; results[i] should hold a[i]+b[i]
+			results := make([]uint64, len(a))
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[idToIndex[int(id)]] = count
+				}
+			})
+
+			for i, res := range results {
+				if res != a[i]+b[i] {
+					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
+				}
+			}
+		})
+	}
+}
+
+type bsiAddCase struct {
+	positions []uint64
+	a         []uint64
+	b         []uint64
+}
+
+func (b bsiAddCase) Len() int {
+	return len(b.positions)
+}
+
+// Less reports whether the element with
+// index i should sort before the element with index j.
+func (b bsiAddCase) Less(i, j int) bool {
+	return b.positions[i] < b.positions[j]
+}
+
+// Swap swaps the elements with indexes i and j.
+func (b bsiAddCase) Swap(i, j int) {
+	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
+	b.a[i], b.a[j] = b.a[j], b.a[i]
+	b.b[i], b.b[j] = b.b[j], b.b[i]
+}
+
+// TestBSIAddCases tests specific cases of bsiAdd (would generally be
+// pulled from randomly generated ones from TestBSIAdd upon failure).
+func TestBSIAddCases(t *testing.T) {
+	tests := []bsiAddCase{
+		{
+			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
+			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
+			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
+		},
+		{
+			positions: []uint64{17570, 54607},
+			a:         []uint64{1010, 71},
+			b:         []uint64{25, 150},
+		},
+	}
+
+	var builderA, builderB bsiBuilder
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
+				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
+			}
+			sort.Sort(tst)
+
+			for i := 0; i < len(tst.a); i++ {
+				builderA.Insert(tst.positions[i], tst.a[i])
+				builderB.Insert(tst.positions[i], tst.b[i])
+			}
+
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := addBSI(dataA, dataB)
+			// maps id to count
+			results := make(map[uint64]uint64)
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[id] = count
+				}
+			})
+
+			for i, id := range tst.positions {
+				if results[id] != tst.a[i]+tst.b[i] {
+					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
+				}
+			}
+		})
+	}
+}

cache.go

@@ -1,29 +1,19 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
 	"sync"
 	"time"
 
-	"github.com/pilosa/pilosa/v2/lru"
-	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/molecula/featurebase/v3/lru"
+	pb "github.com/molecula/featurebase/v3/proto"
+	"github.com/molecula/featurebase/v3/stats"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -118,7 +108,8 @@ func (c *lruCache) Top() []bitmapPair {
 			Count: n,
 		})
 	}
-	sort.Sort(bitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
@@ -134,9 +125,11 @@ var _ cache = &lruCache{}
 
 // rankCache represents a cache with sorted entries.
 type rankCache struct {
-	mu       sync.Mutex
-	entries  map[uint64]uint64
-	rankings []bitmapPair // cached, ordered list
+	mu           sync.Mutex
+	entries      map[uint64]uint64
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -168,10 +161,16 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	// Ignore if the column count is below the threshold,
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -184,11 +183,25 @@ func (c *rankCache) Add(id uint64, n uint64) {
 func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
 	c.entries[id] = n
+
+	// FB-1206: Periodically invalidate the cache when we are bulk loading
+	// as this can take up an upbounded amount of memory. This is especially
+	// true when restoring shards as all rows will be added.
+	if len(c.entries) > int(2*c.maxEntries) {
+		c.stats.Count(MetricRecalculateCache, 1, 1.0)
+		c.recalculate()
+	}
 }
 
 // Get returns a count for a given id.
@@ -209,12 +222,15 @@ func (c *rankCache) Len() int {
 func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
-	for id := range c.entries {
-		a = append(a, id)
+	if len(c.entries) == 0 {
+		return nil
 	}
-	sort.Sort(uint64Slice(a))
-	return a
+	ids := make([]uint64, 0, len(c.entries))
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
@@ -228,7 +244,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -236,27 +252,42 @@ func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
 	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
+		// Ensure that we're marked as dirty even if we weren't otherwise.
+		c.dirty = true
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
 func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]bitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
 		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(bitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -272,11 +303,14 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
+
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // SetStats defines the stats client used in the cache.
@@ -285,7 +319,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
 }
 
 // Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
 func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -309,17 +355,68 @@ type bitmapPair struct {
 // bitmapPairs is a sortable list of BitmapPair objects.
 type bitmapPairs []bitmapPair
 
-func (p bitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p bitmapPairs) Len() int           { return len(p) }
-func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
+func (p *bitmapPairs) Len() int           { return len(*p) }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
-	Key   string `json:"key,omitempty"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
+	}
+}
+
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	}
+}
+
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -395,6 +492,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
+}
+
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
+	}
+	copy(r.Pairs, p.Pairs)
+	return
+}
+
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -402,66 +575,6 @@ func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
-// merge combines p and other to a unique sorted set of values.
-// p and other must both have unique sets and be sorted.
-func (p uint64Slice) merge(other []uint64) []uint64 {
-	ret := make([]uint64, 0, len(p))
-
-	i, j := 0, 0
-	for i < len(p) && j < len(other) {
-		a, b := p[i], other[j]
-		if a == b {
-			ret = append(ret, a)
-			i, j = i+1, j+1
-		} else if a < b {
-			ret = append(ret, a)
-			i++
-		} else {
-			ret = append(ret, b)
-			j++
-		}
-	}
-
-	if i < len(p) {
-		ret = append(ret, p[i:]...)
-	} else if j < len(other) {
-		ret = append(ret, other[j:]...)
-	}
-
-	return ret
-}
-
-// bitmapCache provides an interface for caching full bitmaps.
-type bitmapCache interface {
-	Fetch(id uint64) (*Row, bool)
-	Add(id uint64, b *Row)
-}
-
-// simpleCache implements BitmapCache
-// it is meant to be a short-lived cache for cases where writes are continuing to access
-// the same row within a short time frame (i.e. good for write-heavy loads)
-// A read-heavy use case would cause the cache to get bigger, potentially causing the
-// node to run out of memory.
-type simpleCache struct {
-	cache map[uint64]*Row
-}
-
-// Fetch retrieves the bitmap at the id in the cache.
-func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
-	m, ok := s.cache[id]
-	return m, ok
-}
-
-// Add adds the bitmap to the cache, keyed on the id. A nil row means
-// deleting the row from the cache.
-func (s *simpleCache) Add(id uint64, b *Row) {
-	if b != nil {
-		s.cache[id] = b
-	} else {
-		delete(s.cache, id)
-	}
-}
-
 // nopCache represents a no-op Cache implementation.
 type nopCache struct {
 	stats stats.StatsClient

cache_test.go

@@ -1,27 +1,15 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
-	"github.com/pilosa/pilosa/v2"
+	"github.com/molecula/featurebase/v3"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +19,66 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
+}
+
+func TestCache_Rank_BulkAdd(t *testing.T) {
+	const cacheSize = 10
+	cache := pilosa.NewRankCache(uint32(cacheSize))
+
+	for i := uint64(0); i < 1000; i++ {
+		cache.BulkAdd(i, i)
+		if n := cache.Len(); n > cacheSize*2 {
+			t.Fatalf("entry count exceed 2x cache size: %d", n)
+		}
+	}
 }

catcher.go

@@ -0,0 +1,243 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"github.com/molecula/featurebase/v3/roaring"
+	txkey "github.com/molecula/featurebase/v3/short_txkey"
+	"github.com/molecula/featurebase/v3/vprint"
+)
+
+// catcher is useful to report error locations with a
+// Stack dump before the complexity
+// of the executor_test swallows up
+// the location of a PanicOn.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	return c.b.NewTxIterator(index, field, view, shard)
+}
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ForEach(index, field, view, shard, fn)
+}
+
+func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
+	return c.b.GetSortedFieldViewList(idx, shard)
+}
+
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

client.go

@@ -1,174 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"context"
-	"io"
-)
-
-// Bit represents the intersection of a row and a column. It can be specified by
-// integer ids or string keys.
-type Bit struct {
-	RowID     uint64
-	ColumnID  uint64
-	RowKey    string
-	ColumnKey string
-	Timestamp int64
-}
-
-// FieldValue represents the value for a column within a
-// range-encoded field.
-type FieldValue struct {
-	ColumnID  uint64
-	ColumnKey string
-	Value     int64
-}
-
-// InternalClient should be implemented by any struct that enables any transport between nodes
-// TODO: Refactor
-// Note from Travis: Typically an interface containing more than two or three methods is an indication that
-// something hasn't been architected correctly.
-// While I understand that putting the entire Client behind an interface might require this many methods,
-// I don't want to let it go unquestioned.
-type InternalClient interface {
-	MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
-	Schema(ctx context.Context) ([]*IndexInfo, error)
-	PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
-	CreateIndex(ctx context.Context, index string, opt IndexOptions) error
-	FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
-	Nodes(ctx context.Context) ([]*Node, error)
-	Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-	Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
-	ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
-	EnsureIndex(ctx context.Context, name string, options IndexOptions) error
-	EnsureField(ctx context.Context, indexName string, fieldName string) error
-	EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
-	ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
-	ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
-	ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
-	CreateField(ctx context.Context, index, field string) error
-	CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
-	FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
-	BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
-	ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
-	SendMessage(ctx context.Context, uri *URI, msg []byte) error
-	RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
-	ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
-}
-
-//===============
-
-// InternalQueryClient is the internal interface for querying a node.
-type InternalQueryClient interface {
-	QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
-}
-
-type nopInternalQueryClient struct{}
-
-func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
-
-func newNopInternalQueryClient() *nopInternalQueryClient {
-	return &nopInternalQueryClient{}
-}
-
-var _ InternalQueryClient = newNopInternalQueryClient()
-
-//===============
-
-type nopInternalClient struct{}
-
-func newNopInternalClient() nopInternalClient {
-	return nopInternalClient{}
-}
-
-var _ InternalClient = newNopInternalClient()
-
-func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
-	return nil, nil
-}
-func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
-func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error {
-	return nil
-}
-
-func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
-	return nil
-}
-func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
-	return nil, nil
-}
-func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
-	return nil, nil
-}
-func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
-func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
-	return nil, nil
-}
-func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
-	return nil
-}
-func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
-	return nil
-}
-func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
-	return nil
-}
-func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
-	return nil
-}
-func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
-	return nil
-}
-func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
-	return nil
-}
-func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error {
-	return nil
-}
-func (n nopInternalClient) ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error {
-	return nil
-}
-func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
-	return nil
-}
-func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
-func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
-	return nil
-}
-func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
-	return nil, nil
-}
-func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
-	return nil, nil, nil
-}
-func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
-	return nil, nil
-}
-func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
-	return nil, nil
-}
-func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
-	return nil
-}
-func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
-	return nil, nil
-}

client/README.md

@@ -0,0 +1,85 @@
+# Go Client for Pilosa
+
+Go client for Pilosa high performance distributed index.
+
+## Usage
+
+If you have the pilosa repo in your `GOPATH`,
+you can import the library in your code using:
+
+```go
+import "github.com/pilosa/pilosa/v2/client"
+```
+
+
+### Quick overview
+
+Assuming [Pilosa](https://github.com/pilosa/pilosa) server is running at `localhost:10101` (the default):
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/pilosa/pilosa/v2/client"
+)
+
+func main() {
+	// Create the default client
+	cli := client.DefaultClient()
+
+	// Retrieve the schema
+	schema, err := cli.Schema()
+
+	// Create an Index object
+	myindex := schema.Index("myindex")
+
+	// Create a Field object
+	myfield := myindex.Field("myfield")
+
+	// make sure the index and the field exists on the server
+	err := cli.SyncSchema(schema)
+
+	// Send a Set query. If err is non-nil, response will be nil.
+	response, err := cli.Query(myfield.Set(5, 42))
+
+	// Send a Row query. If err is non-nil, response will be nil.
+	response, err = cli.Query(myfield.Row(5))
+
+	// Get the result
+	result := response.Result()
+	// Act on the result
+	if result != nil {
+		columns := result.Row().Columns
+		fmt.Println("Got columns: ", columns)
+	}
+
+	// You can batch queries to improve throughput
+	response, err = cli.Query(myindex.BatchQuery(
+		myfield.Row(5),
+		myfield.Row(10)))
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	for _, result := range response.Results() {
+		// Act on the result
+		fmt.Println(result.Row().Columns)
+	}
+}
+```
+
+## Documentation
+
+### Data Model and Queries
+
+See: [Data Model and Queries](docs/data-model-queries.md)
+
+### Executing Queries
+
+See: [Server Interaction](docs/server-interaction.md)
+
+### Other Documentation
+
+* [Tracing](docs/tracing.md)

client/client_it_test.go

@@ -0,0 +1,753 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/molecula/featurebase/v3/disco"
+	pnet "github.com/molecula/featurebase/v3/net"
+	"github.com/molecula/featurebase/v3/shardwidth"
+	"github.com/molecula/featurebase/v3/test"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	testIndex                *Index
+	testIndexWithKeys        *Index
+	testIndexWithKeysNoTrack *Index
+	testIndexAtomicRecord    *Index
+	testIndexKeyTranslation  *Index
+
+	testField            *Field
+	testFieldTimeQuantum *Field
+	testFieldInt0        *Field
+	testFieldInt1        *Field
+)
+
+func setup(t *testing.T, cli *Client) {
+	t.Helper()
+
+	testSchema := NewSchema()
+	testIndex = testSchema.Index("test-index")
+	testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true))
+	testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack",
+		OptIndexKeys(true),
+		OptIndexTrackExistence(false),
+	)
+	testField = testIndex.Field("test-field")
+	testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear))
+	testIndexKeyTranslation = testSchema.Index("test-index-key-translation", OptIndexKeys(true))
+
+	testIndexAtomicRecord = testSchema.Index("test-index-atomic-record")
+	testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000))
+	testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000))
+
+	require.NoErrorf(t, cli.SyncSchema(testSchema), "SyncSchema")
+}
+
+func tearDown(t *testing.T, cli *Client) {
+	t.Helper()
+
+	for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord, testIndexKeyTranslation} {
+		require.NoErrorf(t, cli.DeleteIndex(i), "DeleteIndex(%s)", i.name)
+	}
+}
+
+func TestClientAgainstCluster(t *testing.T) {
+	for size, replicaN := 3, 1; replicaN <= 2; replicaN++ {
+		testName := fmt.Sprintf("%d.%d", size, replicaN)
+		t.Run(testName, func(t *testing.T) {
+
+			// Start size.replicaN cluster
+			c := test.MustNewCluster(t, size)
+			for _, n := range c.Nodes {
+				n.Config.Cluster.ReplicaN = replicaN
+			}
+			err := c.Start()
+			require.NoError(t, err, "Start cluster "+testName)
+
+			urls := make([]string, len(c.Nodes))
+			for i, n := range c.Nodes {
+				urls[i] = n.URL()
+			}
+			defer c.Close()
+
+			// Create a new client for the cluster
+			cli, err := newClientFromAddresses(urls, &ClientOptions{})
+			require.NoErrorf(t, err, "newClientFromAddresses(%v): %v", urls, err)
+			defer cli.Close()
+
+			t.Run("GetStatus", func(t *testing.T) {
+				status, err := cli.Status()
+				require.NoErrorf(t, err, "GET /status")
+
+				require.Equalf(t, disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status")
+			})
+
+			t.Run("QueryRow", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				resp, err := cli.Query(testField.Row(1))
+				require.NoErrorf(t, err, "Query Row")
+				require.NotNil(t, resp, "Response should not be nil")
+			})
+
+			t.Run("QueryWithShards", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				shardWidth := uint64(1 << shardwidth.Exponent)
+
+				_, err := cli.Query(testField.Set(1, 1))
+				require.NoErrorf(t, err, "Set(1, %d)", 1)
+
+				_, err = cli.Query(testField.Set(1, shardWidth))
+				require.NoErrorf(t, err, "Set(1, %d)", shardWidth)
+
+				_, err = cli.Query(testField.Set(1, shardWidth*3))
+				require.NoErrorf(t, err, "Set(1, %d)", shardWidth*3)
+
+				resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3))
+				require.NoErrorf(t, err, "Row(1) OptQueryShards(0, 3)")
+
+				cols := resp.Result().Row().Columns
+				require.Equalf(t, []uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols)
+			})
+
+			t.Run("OrmCount", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldCount := testIndex.Field("test-field-count")
+				err := cli.EnsureField(testFieldCount)
+				require.NoError(t, err)
+
+				qry := testIndex.BatchQuery(
+					testFieldCount.Set(10, 20),
+					testFieldCount.Set(10, 21),
+					testFieldCount.Set(15, 25),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(t, err, "BatchQuery")
+
+				resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10)))
+				require.NoErrorf(t, err, "Count")
+				require.Equalf(t, int64(2), resp.Result().Count(), "Count")
+			})
+
+			t.Run("DecimalField", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3))
+				err := cli.EnsureField(testFieldDec)
+				require.NoError(t, err)
+
+				sch, err := cli.Schema()
+				require.NoErrorf(t, err, "Schema")
+
+				idx := sch.indexes[testIndex.name]
+				opts := idx.Field(testFieldDec.name).Options()
+				require.Equalf(t, int64(3), opts.scale, "%s scale", testFieldDec.name)
+			})
+
+			t.Run("IntersectReturns", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldSegments := testIndex.Field("test-field-segments")
+				err := cli.EnsureField(testFieldSegments)
+				require.NoError(t, err)
+
+				qry1 := testIndex.BatchQuery(
+					testFieldSegments.Set(2, 10),
+					testFieldSegments.Set(2, 15),
+					testFieldSegments.Set(3, 10),
+					testFieldSegments.Set(3, 20),
+				)
+				_, err = cli.Query(qry1)
+				require.NoErrorf(t, err, "BatchQuery")
+
+				qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3))
+				resp, err := cli.Query(qry2)
+				require.NoErrorf(t, err, "Intersect")
+
+				require.Equalf(t, 1, len(resp.Results()), "Intersect number of results")
+				require.Equalf(t, []uint64{10}, resp.Result().Row().Columns, "Intersect columns results")
+			})
+
+			t.Run("TopNReturns", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldTopN := testIndex.Field("test-field-topn")
+				err := cli.EnsureField(testFieldTopN)
+				require.NoError(t, err)
+
+				qry := testIndex.BatchQuery(
+					testFieldTopN.Set(10, 5),
+					testFieldTopN.Set(10, 10),
+					testFieldTopN.Set(10, 15),
+					testFieldTopN.Set(20, 5),
+					testFieldTopN.Set(30, 5),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(t, err, "BatchQuery")
+
+				// XXX: The following is required to make this test pass. See: https://github.com/molecula/featurebase/issues/625
+				_, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil)
+				require.NoErrorf(t, err, "POST /recalculate-caches")
+
+				resp, err := cli.Query(testFieldTopN.TopN(2))
+				require.NoErrorf(t, err, "TopN(2)")
+
+				items := resp.Result().CountItems()
+				require.Equalf(t, 2, len(items), "TopN result CountItems")
+
+				item := items[0]
+				require.Equalf(t, uint64(10), item.ID, "TopN result item[0].ID")
+				require.Equalf(t, uint64(3), item.Count, "TopN result item[0].Count")
+			})
+
+			t.Run("MinMaxRow", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldMinMax := testIndex.Field("test-field-minmax")
+				err := cli.EnsureField(testFieldMinMax)
+				require.NoError(t, err)
+
+				qry := testIndex.BatchQuery(
+					testFieldMinMax.Set(10, 5),
+					testFieldMinMax.Set(10, 10),
+					testFieldMinMax.Set(10, 15),
+					testFieldMinMax.Set(20, 5),
+					testFieldMinMax.Set(30, 5),
+				)
+				_, err = cli.Query(qry)
+				require.NoErrorf(t, err, "Setting bits")
+
+				resp, err := cli.Query(testFieldMinMax.MinRow())
+				require.NoErrorf(t, err, "MinRow")
+
+				min := resp.Result().CountItem().ID
+				require.Equalf(t, uint64(10), min, "Min")
+
+				resp, err = cli.Query(testFieldMinMax.MaxRow())
+				require.NoErrorf(t, err, "MaxRow")
+
+				max := resp.Result().CountItem().ID
+				require.Equalf(t, uint64(30), max, "Max")
+			})
+
+			t.Run("SetMutexField", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0))
+				err := cli.EnsureField(testFieldMutex)
+				require.NoError(t, err)
+
+				// can set mutex
+				_, err = cli.Query(testFieldMutex.Set(1, 100))
+				require.NoErrorf(t, err, "Set(1, 100)")
+
+				resp, err := cli.Query(testFieldMutex.Row(1))
+				require.NoErrorf(t, err, "Row(1)")
+
+				target := []uint64{100}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+
+				// setting another row removes the previous
+				_, err = cli.Query(testFieldMutex.Set(42, 100))
+				require.NoErrorf(t, err, "Set(42, 100)")
+
+				resp, err = cli.Query(testIndex.BatchQuery(
+					testFieldMutex.Row(1),
+					testFieldMutex.Row(42),
+				))
+				require.NoErrorf(t, err, "BatchQuery")
+
+				target1 := []uint64(nil)
+				target42 := []uint64{100}
+				require.Equalf(t, target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns")
+				require.Equalf(t, target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns")
+			})
+
+			t.Run("SetBoolField", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool())
+				err := cli.EnsureField(testFieldBool)
+				require.NoError(t, err)
+
+				// can set bool
+				_, err = cli.Query(testFieldBool.Set(true, 100))
+				require.NoErrorf(t, err, "Set(true, 100)")
+
+				resp, err := cli.Query(testFieldBool.Row(true))
+				require.NoErrorf(t, err, "Row(true)")
+
+				target := []uint64{100}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("ClearRowQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldClear := testIndex.Field("test-field-clear")
+				err := cli.EnsureField(testFieldClear)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldClear.Set(1, 100),
+					testFieldClear.Set(1, 200),
+				))
+				require.NoErrorf(t, err, "Set(1, 100) Set(1, 200)")
+
+				resp, err := cli.Query(testFieldClear.Row(1))
+				require.NoErrorf(t, err, "Row(1)")
+
+				target := []uint64{100, 200}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+
+				_, err = cli.Query(testFieldClear.ClearRow(1))
+				require.NoErrorf(t, err, "ClearRow(1)")
+
+				resp, err = cli.Query(testFieldClear.Row(1))
+				require.NoErrorf(t, err, "Row(1)")
+
+				target = []uint64(nil)
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("RowsQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldRows := testIndex.Field("test-field-rows")
+				err := cli.EnsureField(testFieldRows)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRows.Set(1, 100),
+					testFieldRows.Set(1, 200),
+					testFieldRows.Set(2, 200),
+				))
+				require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testFieldRows.Rows())
+				require.NoErrorf(t, err, "Rows")
+
+				target := RowIdentifiersResult{
+					IDs: []uint64{1, 2},
+				}
+				require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
+			})
+
+			t.Run("UnionRowsQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldRows := testIndex.Field("test-field-rows")
+				err := cli.EnsureField(testFieldRows)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRows.Set(1, 100),
+					testFieldRows.Set(1, 200),
+					testFieldRows.Set(2, 200),
+				))
+				require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testFieldRows.Rows().Union())
+				require.NoErrorf(t, err, "Rows Union")
+
+				target := []uint64{100, 200}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("LikeQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true))
+				err := cli.EnsureField(testFieldLike)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldLike.Set("a", 100),
+					testFieldLike.Set("b", 200),
+					testFieldLike.Set("bc", 200),
+				))
+				require.NoErrorf(t, err, "Set(a, 100) Set(b, 200) Set(bc, 200)")
+
+				resp, err := cli.Query(testFieldLike.Like("b%"))
+				require.NoErrorf(t, err, `Like(b%)`)
+
+				target := RowIdentifiersResult{
+					Keys: []string{"b", "bc"},
+				}
+				require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
+			})
+
+			t.Run("GroupByQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldGroupBy := testIndex.Field("test-field-group-by")
+				err := cli.EnsureField(testFieldGroupBy)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldGroupBy.Set(1, 100),
+					testFieldGroupBy.Set(1, 200),
+					testFieldGroupBy.Set(2, 200),
+				))
+				require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
+
+				resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
+				require.NoErrorf(t, err, `Like(b%)`)
+
+				target := []GroupCount{
+					{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1},
+				}
+
+				assertGroupBy(t, target, resp.Result().GroupCounts())
+			})
+
+			t.Run("GroupByQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10))
+				err := cli.EnsureField(testFieldGroupBy)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.RawQuery(`
+		Set(0,      test-field-group-by-int=1)
+		Set(1,      test-field-group-by-int=2)
+
+		Set(2,      test-field-group-by-int=-2)
+		Set(3,      test-field-group-by-int=-1)
+
+		Set(4,      test-field-group-by-int=4)
+
+		Set(10,     test-field-group-by-int=0)
+		Set(100,    test-field-group-by-int=0)
+		Set(1000,   test-field-group-by-int=0)
+		Set(10000,  test-field-group-by-int=0)
+		Set(100000, test-field-group-by-int=0)
+		`))
+				require.NoError(t, err, "Set(0..100000)")
+
+				resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
+				require.NoErrorf(t, err, `GroupBy(Rows)`)
+
+				var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4
+				target := []GroupCount{
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1},
+					{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1},
+				}
+				assertGroupBy(t, target, resp.Result().GroupCounts())
+			})
+
+			t.Run("CreateDeleteIndexField", func(t *testing.T) {
+				tmpIndex := NewIndex("tmp-index")
+				tmpField := tmpIndex.Field("tmp-field")
+
+				err := cli.CreateIndex(tmpIndex)
+				require.NoError(t, err)
+
+				err = cli.CreateField(tmpField)
+				require.NoError(t, err)
+
+				err = cli.DeleteField(tmpField)
+				require.NoError(t, err)
+
+				err = cli.DeleteIndex(tmpIndex)
+				require.NoError(t, err)
+			})
+
+			t.Run("ErrorCreatingIndexField", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				require.ErrorIs(t, cli.CreateIndex(testIndex), ErrIndexExists)
+				require.ErrorIs(t, cli.CreateField(testField), ErrFieldExists)
+			})
+
+			t.Run("Failover", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar")
+				tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0))
+
+				_, err := tmpcli.Query(testIndex.All())
+				require.Error(t, err, ErrTriedMaxHosts)
+			})
+
+			t.Run("InvalidQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				_, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil)
+				require.Error(t, err)
+
+				_, err = cli.Query(testIndex.RawQuery("Invalid query"))
+				require.Error(t, err)
+			})
+
+			t.Run("Sync", func(t *testing.T) {
+				testIndexRemote := NewIndex("test-index-remote")
+				err := cli.EnsureIndex(testIndexRemote)
+				require.NoError(t, err)
+
+				testFieldRemote := testIndexRemote.Field("test-field-remote")
+				err = cli.EnsureField(testFieldRemote)
+				require.NoError(t, err)
+
+				schema := NewSchema()
+				idx1 := schema.Index("index-1")
+				idx1.Field("field-1-1")
+				idx1.Field("field-1-2")
+
+				idx2 := schema.Index("index-2")
+				idx2.Field("field-2-1")
+				schema.Index(testIndexRemote.Name())
+
+				err = cli.SyncSchema(schema)
+				require.NoError(t, err)
+
+				err = cli.DeleteIndex(testIndexRemote)
+				require.NoError(t, err)
+
+				err = cli.DeleteIndex(idx1)
+				require.NoError(t, err)
+
+				err = cli.DeleteIndex(idx2)
+				require.NoError(t, err)
+			})
+
+			t.Run("FetchFragmentNodes", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0)
+				require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
+				require.Equalf(t, replicaN, len(nodes), "len(nodes)")
+
+				// running the same for coverage
+				nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0)
+				require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
+				require.Equalf(t, replicaN, len(nodes), "len(nodes)")
+			})
+
+			t.Run("RowRangeQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour))
+				err := cli.EnsureField(testFieldRange)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)),
+					testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)),
+					testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)),
+				))
+				require.NoErrorf(t, err, "BatchQuery SetTimestamp")
+
+				start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC)
+				end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC)
+				resp, err := cli.Query(testFieldRange.RowRange(10, start, end))
+				require.NoErrorf(t, err, "RowRange(10, %v, %v)", start, end)
+
+				target := []uint64{100}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+			t.Run("StoreQuery", func(t *testing.T) {
+				schema := NewSchema()
+				testIndexStore := schema.Index("test-index-store")
+				testFieldFrom := testIndexStore.Field("test-field-from")
+				testFieldTo := testIndexStore.Field("test-field-to")
+				err := cli.SyncSchema(schema)
+				require.NoError(t, err)
+
+				defer func() {
+					cerr := cli.DeleteIndex(testIndexStore)
+					require.NoErrorf(t, cerr, "failed to delete index: %v", testIndexStore.name)
+				}()
+
+				_, err = cli.Query(testIndexStore.BatchQuery(
+					testFieldFrom.Set(10, 100),
+					testFieldFrom.Set(10, 200),
+					testFieldTo.Store(testFieldFrom.Row(10), 1),
+				))
+				require.NoErrorf(t, err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)")
+
+				resp, err := cli.Query(testFieldTo.Row(1))
+				require.NoErrorf(t, err, "Row(1)")
+
+				target := []uint64{100, 200}
+				require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
+			})
+
+			t.Run("MultipleClientKeyQuery", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient")
+				err := cli.EnsureField(testFieldMultiClient)
+				require.NoError(t, err)
+
+				eg := &errgroup.Group{}
+				for i := 0; i < 10; i++ {
+					rowID := uint64(i)
+					eg.Go(func() error {
+						_, e := cli.Query(testFieldMultiClient.Set(rowID, "col"))
+						return e
+					})
+				}
+				require.NoError(t, eg.Wait())
+			})
+
+			t.Run("ExportRowIDColumnID", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldExport := testIndex.Field("test-field-export")
+				err := cli.EnsureField(testFieldExport)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndex.BatchQuery(
+					testFieldExport.Set(1, 1),
+					testFieldExport.Set(1, 10),
+					testFieldExport.Set(2, 1048577),
+				), nil)
+				require.NoErrorf(t, err, "Set(1, 1) Set(1, 10) Set(2, 1048577)")
+
+				r, err := cli.ExportField(testFieldExport)
+				require.NoErrorf(t, err, "ExportField")
+
+				b, err := ioutil.ReadAll(r)
+				require.NoError(t, err)
+
+				target := "1,1\n1,10\n2,1048577\n"
+				require.Equalf(t, target, string(b), "Export Field Response")
+			})
+
+			t.Run("ExportRowIDColumnKey", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldExport := testIndexWithKeys.Field("test-field-export")
+				err := cli.EnsureField(testFieldExport)
+				require.NoError(t, err)
+
+				_, err = cli.Query(testIndexWithKeys.BatchQuery(
+					testFieldExport.Set(1, "one"),
+					testFieldExport.Set(1, "ten"),
+					testFieldExport.Set(2, "big-number"),
+				), nil)
+				require.NoErrorf(t, err, "Set(1, one) Set(1, ten) Set(2, big-number)")
+
+				r, err := cli.ExportField(testFieldExport)
+				require.NoErrorf(t, err, "ExportField")
+
+				b, err := ioutil.ReadAll(r)
+				require.NoError(t, err)
+
+				target := "1,one\n1,ten\n2,big-number\n"
+				require.Equalf(t, target, string(b), "Export Field Response")
+			})
+
+			t.Run("TranslateRowKeys", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				testFieldTranslate := testIndexKeyTranslation.Field("test-field-translate", OptFieldKeys(true))
+				err := cli.EnsureField(testFieldTranslate)
+				require.NoError(t, err)
+
+				trans, err := cli.CreateFieldKeys(testFieldTranslate, "key1", "key2")
+				require.NoErrorf(t, err, "CreateFieldKeys")
+
+				target := map[string]uint64{"key1": 1, "key2": 2}
+				require.Equalf(t, target, trans, "CreateFieldKeys")
+
+				trans, err = cli.FindFieldKeys(testFieldTranslate, "key1", "key2", "key3")
+				require.NoErrorf(t, err, "FindFieldKeys")
+
+				require.Equalf(t, target, trans, "FindFieldKeys")
+			})
+
+			t.Run("TranslateColKeys", func(t *testing.T) {
+				setup(t, cli)
+				defer tearDown(t, cli)
+
+				created, err := cli.CreateIndexKeys(testIndexKeyTranslation, "key1", "key2")
+				require.NoErrorf(t, err, "CreateIndexKeys")
+				if _, ok := created["key1"]; !ok {
+					t.Error("key1 missing")
+				}
+				if _, ok := created["key2"]; !ok {
+					t.Error("key2 missing")
+				}
+
+				found, err := cli.FindIndexKeys(testIndexKeyTranslation, "key1", "key2", "key3")
+				require.NoErrorf(t, err, "FindIndexKeys")
+
+				require.Equalf(t, created, found, "IndexKeys")
+			})
+
+			t.Run("Transactions", func(t *testing.T) {
+				trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute)
+				require.NoErrorf(t, err, "StartTransaction(blah)")
+				require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(t, trns.Active, "TranslateColumnKeys Active")
+
+				trnsMap, err := cli.Transactions()
+				require.NoErrorf(t, err, "Transactions")
+				require.Equalf(t, 1, len(trnsMap), "Transactions len")
+				require.Truef(t, trnsMap["blah"].Active, "Transactions Active")
+
+				trns, err = cli.GetTransaction("blah")
+				require.NoErrorf(t, err, "GetTransaction(blah)")
+				require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(t, trns.Active, "TranslateColumnKeys Active")
+
+				trns, err = cli.FinishTransaction("blah")
+				require.NoErrorf(t, err, "FinishTransaction(blah)")
+				require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
+				require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
+				require.Truef(t, trns.Active, "TranslateColumnKeys Active")
+			})
+		})
+	}
+}
+
+func assertGroupBy(t *testing.T, expected, results []GroupCount) {
+	t.Helper()
+
+	require.Equalf(t, len(expected), len(results), "number of groupings mismatch")
+
+	for i, result := range results {
+		require.Equalf(t, expected[i], result, "unexpected result at %d", i)
+	}
+}

client/client_test.go

@@ -0,0 +1,209 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"crypto/tls"
+	"errors"
+	"reflect"
+	"testing"
+
+	pnet "github.com/molecula/featurebase/v3/net"
+)
+
+func TestQueryWithError(t *testing.T) {
+	var err error
+	client := DefaultClient()
+	index := NewIndex("foo")
+	invalid := NewPQLRowQuery("", index, errors.New("invalid"))
+	_, err = client.Query(invalid, nil)
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestClientOptions(t *testing.T) {
+	targets := []*ClientOptions{
+		{SocketTimeout: 10},
+		{ConnectTimeout: 5},
+		{PoolSizePerRoute: 7},
+		{TotalPoolSize: 17},
+		{TLSConfig: &tls.Config{InsecureSkipVerify: true}},
+	}
+	optionsList := [][]ClientOption{
+		{OptClientSocketTimeout(10)},
+		{OptClientConnectTimeout(5)},
+		{OptClientPoolSizePerRoute(7)},
+		{OptClientTotalPoolSize(17)},
+		{OptClientTLSConfig(&tls.Config{InsecureSkipVerify: true})},
+	}
+
+	for i := 0; i < len(targets); i++ {
+		options := &ClientOptions{}
+		err := options.addOptions(optionsList[i]...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		target := targets[i]
+		if !reflect.DeepEqual(target, options) {
+			t.Fatalf("%v != %v", target, options)
+		}
+	}
+}
+
+func TestNewClientWithErrorredOption(t *testing.T) {
+	_, err := NewClient(":8888", ClientOptionErr(0))
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestNewClient(t *testing.T) {
+	client, err := NewClient(":9999", OptClientManualServerAddress(true))
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI, err := pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(targetURI, client.manualServerURI) {
+		t.Fatalf("%v != %v", targetURI, client.manualServerURI)
+	}
+	targetFragmentNode := &fragmentNode{
+		Scheme: "http",
+		Host:   "localhost",
+		Port:   9999,
+	}
+	if !reflect.DeepEqual(targetFragmentNode, client.manualFragmentNode) {
+		t.Fatalf("%v != %v", targetFragmentNode, client.manualFragmentNode)
+	}
+	client, err = NewClient(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	targetURI, err = pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	target := []*pnet.URI{targetURI}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+	client, err = NewClient([]string{":9999"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	targetURI1, err := pnet.NewURIFromAddress(":8888")
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI2, err := pnet.NewURIFromAddress(":9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client, err = NewClient([]*pnet.URI{targetURI1, targetURI2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	target = []*pnet.URI{targetURI1, targetURI2}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	client, err = NewClient([]*pnet.URI{targetURI})
+	if err != nil {
+		t.Fatal(err)
+	}
+	target = []*pnet.URI{targetURI}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+
+	client, err = NewClient(DefaultCluster())
+	if err != nil {
+		t.Fatal(err)
+	}
+	target = []*pnet.URI{}
+	if !reflect.DeepEqual(target, client.cluster.hosts) {
+		t.Fatalf("%v != %v", target, client.cluster.hosts)
+	}
+}
+
+func TestNewClientWithInvalidAddr(t *testing.T) {
+	_, err := NewClient(10)
+	if err != ErrAddrURIClusterExpected {
+		t.Fatalf("%v != %v", ErrAddrURIClusterExpected, err)
+	}
+	_, err = NewClient(":invalid")
+	if err == nil {
+		t.Fatalf("should have failed: %+v", err)
+	}
+	_, err = NewClient([]string{"valid:8000", ":invalid"})
+	if err != pnet.ErrInvalidAddress {
+		t.Fatalf("Should have failed '%v, got '%v'", pnet.ErrInvalidAddress, err)
+	}
+}
+
+func TestNewClientManualAddressWithNoURIs(t *testing.T) {
+	_, err := NewClient([]string{}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+	_, err = NewClient([]*pnet.URI{}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+}
+
+func TestNewClientManualAddressWithMultipleURIs(t *testing.T) {
+	_, err := NewClient([]string{":9000", ":5000"}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+
+	targetURI1, err := pnet.NewURIFromAddress(":9000")
+	if err != nil {
+		t.Fatal(err)
+	}
+	targetURI2, err := pnet.NewURIFromAddress(":5000")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewClient([]*pnet.URI{targetURI1, targetURI2}, OptClientManualServerAddress(true))
+	if err != ErrSingleServerAddressRequired {
+		t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
+	}
+}
+
+func ClientOptionErr(int) ClientOption {
+	return func(*ClientOptions) error {
+		return errors.New("Some error")
+	}
+}
+
+func TestQueryOptionsError(t *testing.T) {
+	client := DefaultClient()
+	index := NewIndex("foo")
+	_, err := client.Query(index.RawQuery(""), QueryOptionErr(0))
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func QueryOptionErr(int) QueryOption {
+	return func(*QueryOptions) error {
+		return errors.New("Some error")
+	}
+}

client/cluster.go

@@ -0,0 +1,99 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"sync"
+
+	pnet "github.com/molecula/featurebase/v3/net"
+)
+
+// Cluster contains hosts in a Pilosa cluster.
+type Cluster struct {
+	hosts       []*pnet.URI
+	okList      []bool
+	mutex       *sync.RWMutex
+	lastHostIdx int
+}
+
+// DefaultCluster returns the default Cluster.
+func DefaultCluster() *Cluster {
+	return &Cluster{
+		hosts:  make([]*pnet.URI, 0),
+		okList: make([]bool, 0),
+		mutex:  &sync.RWMutex{},
+	}
+}
+
+// NewClusterWithHost returns a cluster with the given URIs.
+func NewClusterWithHost(hosts ...*pnet.URI) *Cluster {
+	cluster := DefaultCluster()
+	for _, host := range hosts {
+		cluster.AddHost(host)
+	}
+	return cluster
+}
+
+// AddHost adds a host to the cluster.
+func (c *Cluster) AddHost(address *pnet.URI) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.hosts = append(c.hosts, address)
+	c.okList = append(c.okList, true)
+}
+
+// Host returns a host in the cluster.
+func (c *Cluster) Host() *pnet.URI {
+	c.mutex.Lock()
+	var host *pnet.URI
+	for i := range c.okList {
+		idx := (i + c.lastHostIdx) % len(c.okList)
+		ok := c.okList[idx]
+		if ok {
+			host = c.hosts[idx]
+			break
+		}
+	}
+	c.lastHostIdx++
+	c.mutex.Unlock()
+	if host != nil {
+		return host
+	}
+	c.reset()
+	return host
+}
+
+// RemoveHost black lists the host with the given pnet.URI from the cluster.
+func (c *Cluster) RemoveHost(address *pnet.URI) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	for i, uri := range c.hosts {
+		if uri.Equals(address) {
+			c.okList[i] = false
+			break
+		}
+	}
+}
+
+// Hosts returns all available hosts in the cluster.
+func (c *Cluster) Hosts() []pnet.URI {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	hosts := make([]pnet.URI, 0, len(c.hosts))
+	for i, host := range c.hosts {
+		if c.okList[i] {
+			hosts = append(hosts, *host)
+		}
+	}
+	return hosts
+}
+
+func (c *Cluster) reset() {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	for i := range c.okList {
+		c.okList[i] = true
+	}
+}

client/cluster_test.go

@@ -0,0 +1,70 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"testing"
+
+	pnet "github.com/molecula/featurebase/v3/net"
+)
+
+func TestNewClusterWithHost(t *testing.T) {
+	c := NewClusterWithHost(pnet.DefaultURI())
+	hosts := c.Hosts()
+	if len(hosts) != 1 || !hosts[0].Equals(pnet.DefaultURI()) {
+		t.Fail()
+	}
+}
+
+func TestAddHost(t *testing.T) {
+	const addr = "http://localhost:3000"
+	c := DefaultCluster()
+	if c.Hosts() == nil {
+		t.Fatalf("Hosts should not be nil")
+	}
+	uri, err := pnet.NewURIFromAddress(addr)
+	if err != nil {
+		t.Fatalf("Cannot parse address")
+	}
+	target, err := pnet.NewURIFromAddress(addr)
+	if err != nil {
+		t.Fatalf("Cannot parse address")
+	}
+	c.AddHost(uri)
+	hosts := c.Hosts()
+	if len(hosts) != 1 || !hosts[0].Equals(target) {
+		t.Fail()
+	}
+}
+
+func TestHosts(t *testing.T) {
+	c := DefaultCluster()
+	if c.Host() != nil {
+		t.Fatalf("Hosts with empty cluster should return nil")
+	}
+	c = NewClusterWithHost(pnet.DefaultURI())
+	if !c.Host().Equals(pnet.DefaultURI()) {
+		t.Fatalf("Host should return a value if there are hosts in the cluster")
+	}
+}
+
+func TestRemoveHost(t *testing.T) {
+	uri, err := pnet.NewURIFromAddress("index1.pilosa.com:9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := NewClusterWithHost(uri)
+	if len(c.hosts) != 1 {
+		t.Fatalf("The cluster should contain the host")
+	}
+	uri, err = pnet.NewURIFromAddress("index1.pilosa.com:9999")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.RemoveHost(uri)
+	if len(c.Hosts()) != 0 {
+		t.Fatalf("The cluster should not contain the host")
+	}
+}

client/csv/csv.go

@@ -0,0 +1,181 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package csv
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/molecula/featurebase/v3/client"
+)
+
+// Format is the format of the data in the CSV file.
+type Format uint
+
+const (
+	// RowIDColumnID formatted data is ROW_ID,COLUMN_ID.
+	RowIDColumnID Format = iota
+	// RowIDColumnKey formatted data is ROW_ID,COLUMN_KEY.
+	RowIDColumnKey
+	// RowKeyColumnID formatted data is ROW_KEY,COLUMN_ID.
+	RowKeyColumnID
+	// RowKeyColumnKey formatted data is ROW_KEY,COLUMN_ID.
+	RowKeyColumnKey
+	// ColumnID formatted data is COLUMN_ID. Valid only for value import.
+	ColumnID
+	// ColumnKey formatted data is COLUMN_KEY. Valud only for value import.
+	ColumnKey
+)
+
+// ColumnUnmarshaller creates a RecordUnmarshaller for importing columns with the given format.
+func ColumnUnmarshaller(format Format) RecordUnmarshaller {
+	return ColumnUnmarshallerWithTimestamp(format, "")
+}
+
+// ColumnUnmarshallerWithTimestamp creates a RecordUnmarshaller for importing columns with the given format and timestamp format.
+func ColumnUnmarshallerWithTimestamp(format Format, timestampFormat string) RecordUnmarshaller {
+	return func(text string) (client.Record, error) {
+		var err error
+		column := client.Column{}
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			return nil, errors.New("invalid CSV line")
+		}
+
+		hasRowKey := format == RowKeyColumnID || format == RowKeyColumnKey
+		hasColumnKey := format == RowIDColumnKey || format == RowKeyColumnKey
+
+		if hasRowKey {
+			column.RowKey = parts[0]
+		} else {
+			column.RowID, err = strconv.ParseUint(parts[0], 10, 64)
+			if err != nil {
+				return nil, errors.New("invalid row ID")
+			}
+		}
+
+		if hasColumnKey {
+			column.ColumnKey = parts[1]
+		} else {
+			column.ColumnID, err = strconv.ParseUint(parts[1], 10, 64)
+			if err != nil {
+				return nil, errors.New("invalid column ID")
+			}
+		}
+
+		timestamp := int64(0)
+		if len(parts) == 3 {
+			if timestampFormat == "" {
+				if tsInt, err := strconv.Atoi(parts[2]); err != nil {
+					return nil, err
+				} else {
+					timestamp = int64(tsInt)
+				}
+			} else {
+				t, err := time.Parse(timestampFormat, parts[2])
+				if err != nil {
+					return nil, err
+				}
+				timestamp = t.Unix() * int64(time.Second) // Casting a duration to int64 gives the number of nanoseconds in that duration.
+			}
+		}
+		column.Timestamp = timestamp
+
+		return column, nil
+	}
+}
+
+// RecordUnmarshaller is a function which creates a Record from a CSV file line with column data.
+type RecordUnmarshaller func(text string) (client.Record, error)
+
+// Iterator reads records from a Reader.
+// Each line should contain a single record in the following form:
+// field1,field2,...
+type Iterator struct {
+	reader       io.Reader
+	line         int
+	scanner      *bufio.Scanner
+	unmarshaller RecordUnmarshaller
+}
+
+// NewIterator creates a CSVIterator from a Reader.
+func NewIterator(reader io.Reader, unmarshaller RecordUnmarshaller) *Iterator {
+	return &Iterator{
+		reader:       reader,
+		line:         0,
+		scanner:      bufio.NewScanner(reader),
+		unmarshaller: unmarshaller,
+	}
+}
+
+// NewColumnIterator creates a new iterator for column data.
+func NewColumnIterator(format Format, reader io.Reader) *Iterator {
+	return NewIterator(reader, ColumnUnmarshaller(format))
+}
+
+// NewColumnIteratorWithTimestampFormat creates a new iterator for column data with timestamp.
+func NewColumnIteratorWithTimestampFormat(format Format, reader io.Reader, timestampFormat string) *Iterator {
+	return NewIterator(reader, ColumnUnmarshallerWithTimestamp(format, timestampFormat))
+}
+
+// NewValueIterator creates a new iterator for value data.
+func NewValueIterator(format Format, reader io.Reader) *Iterator {
+	return NewIterator(reader, FieldValueUnmarshaller(format))
+}
+
+// NextRecord iterates on lines of a Reader.
+// Returns io.EOF on end of iteration.
+func (c *Iterator) NextRecord() (client.Record, error) {
+	if ok := c.scanner.Scan(); ok {
+		c.line++
+		text := strings.TrimSpace(c.scanner.Text())
+		if text != "" {
+			rc, err := c.unmarshaller(text)
+			if err != nil {
+				return nil, fmt.Errorf("%s at line: %d", err.Error(), c.line)
+			}
+			return rc, nil
+		}
+	}
+	err := c.scanner.Err()
+	if err != nil {
+		return nil, err
+	}
+	return nil, io.EOF
+}
+
+// FieldValueUnmarshaller is a function which creates a Record from a CSV file line with value data.
+func FieldValueUnmarshaller(format Format) RecordUnmarshaller {
+	return func(text string) (client.Record, error) {
+		parts := strings.Split(text, ",")
+		if len(parts) < 2 {
+			return nil, errors.New("invalid CSV")
+		}
+		value, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, errors.New("invalid value")
+		}
+		switch format {
+		case ColumnID:
+			columnID, err := strconv.ParseUint(parts[0], 10, 64)
+			if err != nil {
+				return nil, errors.New("invalid column ID at line: %d")
+			}
+			return client.FieldValue{
+				ColumnID: uint64(columnID),
+				Value:    value,
+			}, nil
+		case ColumnKey:
+			return client.FieldValue{
+				ColumnKey: parts[0],
+				Value:     value,
+			}, nil
+		default:
+			return nil, fmt.Errorf("invalid format: %d", format)
+		}
+	}
+}

client/csv/csv_it_test.go

@@ -0,0 +1,48 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+//go:build integration
+// +build integration
+
+package csv_test
+
+import (
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/molecula/featurebase/v3/client"
+	"github.com/molecula/featurebase/v3/client/csv"
+)
+
+func TestCSVIterate(t *testing.T) {
+	text := `10,7
+		10,5
+		2,3
+		7,1`
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
+	recs := consumeIterator(t, iterator)
+	target := []client.Record{
+		client.Column{RowID: 10, ColumnID: 7},
+		client.Column{RowID: 10, ColumnID: 5},
+		client.Column{RowID: 2, ColumnID: 3},
+		client.Column{RowID: 7, ColumnID: 1},
+	}
+	if !reflect.DeepEqual(target, recs) {
+		t.Fatalf("%v != %v", target, recs)
+	}
+}
+
+func consumeIterator(t *testing.T, it *csv.Iterator) []client.Record {
+	recs := []client.Record{}
+	for {
+		r, err := it.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		recs = append(recs, r)
+	}
+	return recs
+}

client/csv/csv_test.go

@@ -0,0 +1,254 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package csv_test
+
+import (
+	"errors"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/client"
+	"github.com/molecula/featurebase/v3/client/csv"
+)
+
+func TestCSVColumnIterator(t *testing.T) {
+	reader := strings.NewReader(`1,10,683793200
+		5,20,683793300
+		3,41,683793385`)
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, reader)
+	columns := []client.Record{}
+	for {
+		column, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		columns = append(columns, column)
+	}
+	if len(columns) != 3 {
+		t.Fatalf("There should be 3 columns")
+	}
+	target := []client.Column{
+		{RowID: 1, ColumnID: 10, Timestamp: 683793200},
+		{RowID: 5, ColumnID: 20, Timestamp: 683793300},
+		{RowID: 3, ColumnID: 41, Timestamp: 683793385},
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], columns[i]) {
+			t.Fatalf("%v != %v", target[i], columns[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatRowIDColumnID(t *testing.T) {
+	format := "2006-01-02T03:04"
+	reader := strings.NewReader(`1,10,1991-09-02T09:33
+		5,20,1991-09-02T09:35
+		3,41,1991-09-02T09:36`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
+	records := []client.Record{}
+	for {
+		record, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		records = append(records, record)
+	}
+	target := []client.Column{
+		{RowID: 1, ColumnID: 10, Timestamp: 683803980000000000},
+		{RowID: 5, ColumnID: 20, Timestamp: 683804100000000000},
+		{RowID: 3, ColumnID: 41, Timestamp: 683804160000000000},
+	}
+	if len(records) != len(target) {
+		t.Fatalf("There should be %d columns", len(target))
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], records[i]) {
+			t.Fatalf("%v != %v", target[i], records[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatRowKeyColumnKey(t *testing.T) {
+	format := "2006-01-02T03:04"
+	reader := strings.NewReader(`one,ten,1991-09-02T09:33
+		five,twenty,1991-09-02T09:35
+		three,forty-one,1991-09-02T09:36`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowKeyColumnKey, reader, format)
+	records := []client.Record{}
+	for {
+		record, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		records = append(records, record)
+	}
+	target := []client.Column{
+		{RowKey: "one", ColumnKey: "ten", Timestamp: 683803980000000000},
+		{RowKey: "five", ColumnKey: "twenty", Timestamp: 683804100000000000},
+		{RowKey: "three", ColumnKey: "forty-one", Timestamp: 683804160000000000},
+	}
+	if len(records) != len(target) {
+		t.Fatalf("There should be %d columns", len(target))
+	}
+	for i := range target {
+		if !reflect.DeepEqual(target[i], records[i]) {
+			t.Fatalf("%v != %v", target[i], records[i])
+		}
+	}
+}
+
+func TestCSVColumnIteratorWithTimestampFormatFail(t *testing.T) {
+	format := "2014-07-16"
+	reader := strings.NewReader(`1,10,X`)
+	iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatalf("Should have failed")
+	}
+}
+
+func TestCSVValueIteratorWithColumnID(t *testing.T) {
+	reader := strings.NewReader(`1,10
+		5,-20
+		3,41
+	`)
+	iterator := csv.NewValueIterator(csv.ColumnID, reader)
+	values := []client.Record{}
+	for {
+		value, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		values = append(values, value)
+	}
+	target := []pilosa.FieldValue{
+		{ColumnID: 1, Value: 10},
+		{ColumnID: 5, Value: -20},
+		{ColumnID: 3, Value: 41},
+	}
+	if len(values) != len(target) {
+		t.Fatalf("There should be %d values, got %d", len(target), len(values))
+	}
+	for i := range target {
+		v := values[i].(client.FieldValue)
+		if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
+			t.Fatalf("'%+v' != '%+v'", target[i], values[i])
+		}
+	}
+}
+
+func TestCSVValueIteratorWithColumnKey(t *testing.T) {
+	reader := strings.NewReader(`one,10
+		five,-20
+		three,41
+	`)
+	iterator := csv.NewValueIterator(csv.ColumnKey, reader)
+	values := []client.Record{}
+	for {
+		value, err := iterator.NextRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		values = append(values, value)
+	}
+	target := []pilosa.FieldValue{
+		{ColumnKey: "one", Value: 10},
+		{ColumnKey: "five", Value: -20},
+		{ColumnKey: "three", Value: 41},
+	}
+	if len(values) != len(target) {
+		t.Fatalf("There should be %d values, got %d", len(target), len(values))
+	}
+	for i := range target {
+		v := values[i].(client.FieldValue)
+		if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
+			t.Fatalf("%v != %v", target[i], values[i])
+		}
+	}
+}
+
+func TestCSValueIteratorWithInvalidFormat(t *testing.T) {
+	reader := strings.NewReader("1,2")
+	iterator := csv.NewValueIterator(csv.RowIDColumnID, reader)
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatalf("should have failed")
+	}
+}
+
+func TestCSVColumnIteratorInvalidInput(t *testing.T) {
+	invalidInputs := []string{
+		// less than 2 columns
+		"155",
+		// invalid row ID
+		"a5,155",
+		// invalid column ID
+		"155,a5",
+		// invalid timestamp
+		"155,255,a5",
+	}
+	for _, text := range invalidInputs {
+		iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
+		_, err := iterator.NextRecord()
+		if err == nil {
+			t.Fatalf("CSVColumnIterator input: %s should fail", text)
+		}
+	}
+}
+
+func TestCSVValueIteratorInvalidInput(t *testing.T) {
+	invalidInputs := []string{
+		// less than 2 columns
+		"155",
+		// invalid column ID
+		"a5,155",
+		// invalid value
+		"155,a5",
+	}
+	for _, text := range invalidInputs {
+		iterator := csv.NewValueIterator(csv.ColumnID, strings.NewReader(text))
+		_, err := iterator.NextRecord()
+		if err == nil {
+			t.Fatalf("CSVValueIterator input: %s should fail", text)
+		}
+	}
+}
+
+func TestCSVColumnIteratorError(t *testing.T) {
+	iterator := csv.NewColumnIterator(csv.RowIDColumnID, &BrokenReader{})
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatal("CSVColumnIterator should fail with error")
+	}
+}
+
+func TestCSVValueIteratorError(t *testing.T) {
+	iterator := csv.NewValueIterator(csv.ColumnID, &BrokenReader{})
+	_, err := iterator.NextRecord()
+	if err == nil {
+		t.Fatal("CSVValueIterator should fail with error")
+	}
+}
+
+type BrokenReader struct{}
+
+func (r BrokenReader) Read(p []byte) (n int, err error) {
+	return 0, errors.New("broken reader")
+}

client/doc.go

@@ -0,0 +1,55 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+/*
+Package client enables querying a Pilosa server.
+
+This client uses Pilosa's http+protobuf API.
+
+Usage:
+
+	import (
+		"fmt"
+		"github.com/molecula/featurebase/v3/client"
+	)
+
+	// Create a Client instance
+	cli := client.DefaultClient()
+
+	// Create a Schema instance
+	schema, err := cli.Schema()
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an Index instance
+	index, err := schema.Index("repository")
+	if err != nil {
+		panic(err)
+	}
+
+	// Create a Field instance
+	stargazer, err := index.Field("stargazer")
+	if err != nil {
+		panic(err)
+	}
+
+	// Sync the schema with the server-side, so non-existing indexes/fields are created on the server-side.
+	err = cli.SyncSchema(schema)
+	if err != nil {
+		panic(err)
+	}
+
+	// Execute a query
+	response, err := cli.Query(stargazer.Row(5))
+	if err != nil {
+		panic(err)
+	}
+
+	// Act on the result
+	fmt.Println(response.Result())
+
+See also https://www.pilosa.com/docs/api-reference/ and https://www.pilosa.com/docs/query-language/.
+*/
+package client

client/docs/data-model-queries.md

@@ -0,0 +1,149 @@
+# Data Model and Queries
+
+## Indexes and Fields
+
+*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs/latest/data-model/) for more detail about the data model.
+
+The `schema.Index` function is used to create an index instance. Note that this does not create an index on the server; the index object simply defines the schema.
+
+```go
+schema := client.NewSchema()
+repository := schema.Index("repository")
+```
+
+You can pass options while creating index instances:
+```go
+repository := schema.Index("repository", pilosa.OptIndexKeys(true))
+```
+
+Field definitions are created with a call to the `Field` function of an index:
+
+```go
+stargazer := repository.Field("stargazer")
+```
+
+You can pass options to fields:
+
+```go
+stargazer := repository.Field("stargazer", pilosa.OptFieldTypeTime(TimeQuantumYearMonthDay))
+```
+
+In case the schema already exists on the server, you can retrieve that instead of creating the schema:
+```go
+cli := client.DefaultClient()
+schema, err := cli.Schema()
+if err != nil {
+    // act on the error
+}
+repository := schema.Index("repository")
+```
+
+## Queries
+
+Once you have indexes and field definitions, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows with related methods attached to fields.
+
+For instance, `Row` queries work on rows; use a `Field` object to create those queries:
+
+```go
+rowQuery := stargazer.Row(1)  // corresponds to PQL: Row(stargazer=1)
+```
+
+`Union` queries work on columns; use the index to create them:
+
+```go
+query := repository.Union(rowQuery1, rowQuery2)
+```
+
+In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.BatchQuery` function is used for that purpose:
+
+```go
+query := repository.BatchQuery(
+    stargazer.Row(1),
+    repository.Union(stargazer.Row(100), stargazer.Row(5)))
+```
+
+The recommended way of creating query instances is using dedicated functions attached to index and field objects, but sometimes it would be desirable to send raw queries to Pilosa. You can use `index.RawQuery` method for that. Note that query string is not validated before sending to the server:
+
+```go
+query := repository.RawQuery("Row(stargazer=5)")
+```
+
+Raw queries are only sent to the coordinator node of a Pilosa cluster, so currently there's a possible performance hit using them instead of ORM functions attached to index or field instances.
+
+This client supports [range queries using bit sliced indexes (BSI)](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read the [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa.
+
+In order to use BSI range queries, an integer field should be created. The field should have its minimum and maximum set. Here's how you would do that:
+```go
+index := schema.Index("animals")
+captivity := index.Field("captivity", pilosa.OptFieldTypeInt(0, 956))
+```
+
+If the field with the necessary field already exists on the server, you don't need to create the field instance, `cli.SyncSchema(schema)` would load that to `schema`. You can then add some data:
+```go
+// Add the captivity values to the field.
+data := []int{3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318}
+query := index.BatchQuery()
+for i, x := range data {
+	column := uint64(i + 1)
+	query.Add(captivity.SetIntValue(column, x))
+}
+cli.Query(query)
+```
+
+Let's write a range query:
+```go
+// Query for all animals with more than 100 specimens
+response, _ := cli.Query(captivity.GT(100))
+fmt.Println(response.Result().Row().Columns)
+
+// Query for the total number of animals in captivity
+response, _ = cli.Query(captivity.Sum(nil))
+fmt.Println(response.Result().Value())
+```
+
+If you pass a row query to `Sum` as a filter, then only the columns matching the filter will be considered in the `Sum` calculation:
+```go
+// Let's run a few set queries first
+cli.Query(index.BatchQuery(
+    field.Set(42, 1),
+    field.Set(42, 6)))
+// Query for the total number of animals in captivity where row 42 is set
+response, _ = cli.Query(captivity.Sum(field.Row(42)))
+fmt.Println(response.Result().Value())
+```
+
+See the functions further below for the list of functions that can be used with a `Field`.
+
+Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details. Here is a list of methods corresponding to PQL calls:
+
+Index:
+
+* `Union(rows *PQLRowQuery...) *PQLRowQuery`
+* `Intersect(rows *PQLRowQuery...) *PQLRowQuery`
+* `Difference(rows *PQLRowQuery...) *PQLRowQuery`
+* `Xor(rows ...*PQLRowQuery) *PQLRowQuery`
+* `Not(row) *PQLRowQuery`
+* `Count(row *PQLRowQuery) *PQLBaseQuery`
+* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery`
+
+Field:
+
+* `Row(rowID uint64) *PQLRowQuery`
+* `Set(rowID uint64, columnID uint64) *PQLBaseQuery`
+* `SetTimestamp(rowID uint64, columnID uint64, timestamp time.Time) *PQLBaseQuery`
+* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery`
+* `TopN(n uint64) *PQLRowQuery`
+* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery`
+* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
+* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
+* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery`
+* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery`
+* `LT(n int) *PQLRowQuery`
+* `LTE(n int) *PQLRowQuery`
+* `GT(n int) *PQLRowQuery`
+* `GTE(n int) *PQLRowQuery`
+* `Between(a int, b int) *PQLRowQuery`
+* `Sum(row *PQLRowQuery) *PQLBaseQuery`
+* `Min(row *PQLRowQuery) *PQLBaseQuery`
+* `Max(row *PQLRowQuery) *PQLBaseQuery`
+* `SetIntValue(columnID uint64, value int) *PQLBaseQuery`

client/docs/server-interaction.md

@@ -0,0 +1,160 @@
+# Server Interaction
+
+## Pilosa URI
+
+A Pilosa URI has the `${SCHEME}://${HOST}:${PORT}` format:
+* **Scheme**: Protocol of the URI. Default: `http`.
+* **Host**: Hostname or ipv4/ipv6 IP address. Default: localhost.
+* **Port**: Port number. Default: `10101`.
+
+All parts of the URI are optional, but at least one of them must be specified. The following are equivalent:
+
+* `http://localhost:10101`
+* `http://localhost`
+* `http://:10101`
+* `localhost:10101`
+* `localhost`
+* `:10101`
+
+A Pilosa URI is represented by the `github.com/pilosa/pilosa/v2/net URI` struct. Below are a few ways to create `URI` objects:
+
+```go
+import pnet "github.com/pilosa/pilosa/v2/net"
+
+// create the default URI: http://localhost:10101
+uri1 := pnet.DefaultURI()
+
+// create a URI from string address
+uri2, err := pnet.NewURIFromAddress("index1.pilosa.com:20202");
+
+// create a URI with the given host and port
+uri3, err := pnet.NewURIFromHostPort("index1.pilosa.com", 20202);
+```
+
+## Pilosa Client
+
+In order to interact with a Pilosa server, an instance of `client.Client` should be created. The client is thread-safe and uses a pool of connections to the server, so we recommend creating a single instance of the client and sharing it when necessary.
+
+If the Pilosa server is running at the default address (`http://localhost:10101`) you can create the client with default options using:
+
+```go
+import "github.com/pilosa/pilosa/v2/client"
+
+cli := client.DefaultClient()
+```
+
+To use a custom server address, use the `NewClient` function:
+
+```go
+uri, err := pnet.NewURIFromAddress("http://index1.pilosa.com:15000")
+if err != nil {
+    // Act on the error
+}
+cli, err := client.NewClient(uri)
+```
+
+Equivalently:
+```go
+cli, err := client.NewClient("http://index1.pilosa.com:15000")
+```
+
+If you are running a cluster of Pilosa servers, you can create a `Cluster` struct that keeps addresses of those servers:
+
+```go
+uri1, err := pnet.NewURIFromAddress(":10101")
+uri2, err := pnet.NewURIFromAddress(":10110")
+uri3, err := pnet.NewURIFromAddress(":10111")
+cluster := client.NewClusterWithHost(uri1, uri2, uri3)
+
+// Create a client with the cluster
+cli, err := client.NewClient(cluster)
+```
+
+That is equivalent to:
+```go
+cli, err := client.NewClient([]string{":10101", ":10110", ":10111"})
+
+```
+
+It is possible to customize the behaviour of the underlying HTTP client by passing `ClientOption` structs to the `NewClient` function:
+
+```go
+cli, err := client.NewClient(cluster,
+    client.OptClientConnectTimeout(1000),  // if can't connect in  a second, close the connection
+    client.OptClientSocketTimeout(10000),  // if no response received in 10 seconds, close the connection
+    client.OptClientPoolSizePerRoute(3),  // number of connections in the pool per host
+    client.OptClientTotalPoolSize(10))   // number of total connections in the pool
+```
+
+Once you create a client, you can create indexes, fields or start sending queries.
+
+Here is how you would create a index and field:
+
+```go
+// materialize repository index definition and stargazer field definition initialized before
+err := cli.SyncSchema(schema)
+```
+
+You can send queries to a Pilosa server using the `Query` function of the `Client` struct:
+
+```go
+response, err := cli.Query(field.Row(5));
+```
+
+## Server Response
+
+When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned.
+
+A `QueryResponse` struct may contain zero or more results of `QueryResult` type. You can access all results using the `Results` function of `QueryResponse` (which returns a list of `QueryResult` objects), or you can use the `Result` method (which returns either the first result or `nil` if there are no results):
+
+```go
+response, err := cli.Query(field.Row(5))
+if err != nil {
+    // Act on the error
+}
+
+// check that there's a result and act on it
+result := response.Result()
+if result != nil {
+    // Act on the result
+}
+
+// iterate over all results
+for _, result := range response.Results() {
+    // Act on the result
+}
+```
+
+`QueryResult` objects contain:
+
+* `Row()` function to retrieve a row result,
+* `CountItems()` function to retrieve column count per row ID entries returned from `TopN` queries,
+* `Count()` function to retrieve the number of rows per the given row ID returned from `Count` queries.
+* `Value()` function to retrieve the result of `Min`, `Max` or `Sum` queries.
+* `Changed()` function returns whether a `Set` or `Clear` query changed a column.
+
+```go
+row := result.Row()
+columns := row.Columns
+
+countItems := result.CountItems()
+
+count := result.Count()
+
+value := result.Value()
+
+changed := result.Changed()
+```
+
+## SSL/TLS
+
+Make sure the Pilosa server runs on a TLS address. [How To Set Up a Secure Cluster](https://www.pilosa.com/docs/latest/tutorials/#how-to-set-up-a-secure-cluster) tutorial explains how to do that.
+
+In order to enable TLS support on the client side, the scheme of the address should be explicitly specified as `https`, e.g.: `https://01.pilosa.local:10501`
+
+This client library uses the `net/http` module of Go standard library. You can pass a [tls.Config](https://golang.org/pkg/crypto/tls/#Config) struct in a `pilosa.TLSConfig` option to the client. If the Pilosa server is using a certificate from a recognized authority, you can use the defaults.
+
+If you are using a self signed certificate, just pass `pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})` to `pilosa.NewClient` function:
+```go
+client, _ := pilosa.NewClient("https://01.pilosa.local:10501", pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true}))
+```

client/docs/tracing.md

@@ -0,0 +1,111 @@
+# Tracing
+
+Pilosa client supports distributed tracing via the [OpenTracing](https://opentracing.io/) API.
+
+In order to use a tracer with Go-Pilosa, you should:
+1. Create the tracer,
+2. Pass the `OptClientOption(tracer)` to `NewClient`.
+
+In this document, we will be using the [Jaeger](https://www.jaegertracing.io) tracer, but OpenTracing has support for [other tracing systems](https://opentracing.io/docs/supported-tracers/).
+
+## Running the Pilosa Server
+
+Let's run a temporary Pilosa container:
+
+    $ docker run -it --rm -p 10101:10101 pilosa/pilosa:v1.2.0
+
+Check that you can access Pilosa:
+
+    $ curl localhost:10101
+    Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.
+
+## Running the Jaeger Server
+
+Let's run a Jaeger Server container:
+
+    $ docker run -it --rm -p 5775:5775/udp -p 16686:16686 jaegertracing/all-in-one:latest
+    ...<title>Jaeger UI</title>...
+
+## Writing the Sample Code
+
+The sample code depdends on the Jaeger Go client, so let's install it first:
+
+    $ go get -u github.com/uber/jaeger-client-go/
+
+Save the following sample code as `gopilosa-tracing.go`:
+```go
+package main
+
+import (
+	"log"
+	"time"
+
+	"github.com/pilosa/pilosa/v2/client"
+	"github.com/uber/jaeger-client-go"
+	"github.com/uber/jaeger-client-go/config"
+)
+
+func main() {
+	// Create the tracer.
+	cfg := config.Configuration{
+		Sampler: &config.SamplerConfig{
+			Type:  "const",
+			Param: 1,
+		},
+		Reporter: &config.ReporterConfig{
+			LogSpans:            true,
+			BufferFlushInterval: 1 * time.Second,
+			// Jaeger Server address
+			LocalAgentHostPort:  "127.0.0.1:5775",
+		},
+	}
+	tracer, closer, err := cfg.New(
+		"go_pilosa_test",
+		config.Logger(jaeger.StdLogger),
+	)
+
+	// Don't forget to close the tracer.
+	defer closer.Close()
+
+	// Create the client, and pass the tracer.
+	cli, err := client.NewClient(":10101", pilosa.OptClientTracer(tracer))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Read the schema from the server.
+	// This should create a trace on the Jaeger server.
+	schema, err := cli.Schema()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Create and sync the sample schema.
+	// This should create a trace on the Jaeger server.
+	myIndex := schema.Index("my-index")
+	myField := myIndex.Field("my-field")
+	err = cli.SyncSchema(schema)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Run a query on Pilosa.
+	// This should create a trace on the Jaeger server.
+	_, err = cli.Query(myField.Set(1, 1000))
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+```
+
+## Checking the Tracing Data
+
+Run the sample code:
+
+    $ go run gopilosa-tracing.go
+
+
+* Open http://localhost:16686 in your web browser to visit Jaeger UI.
+* Click on the *Search* tab and select `go_pilosa_test` in the *Service* dropdown on the right.
+* Click on *Find Traces* button at the bottom left.
+* You should see a couple of traces, such as: `Client.Query`, `Client.CreateField`, `Client.Schema`, etc.

client/egpool/egpool.go

@@ -0,0 +1,110 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package egpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+type Group struct {
+	PoolSize int
+
+	jobs chan func() error
+
+	sema     chan struct{}
+	errMu    sync.Mutex
+	firstErr error
+	errs     []error
+}
+
+func (eg *Group) Go(f func() error) {
+	if eg.PoolSize <= 0 {
+		eg.PoolSize = 1
+	}
+
+	if eg.jobs == nil {
+		eg.jobs = make(chan func() error)
+		eg.sema = make(chan struct{}, eg.PoolSize)
+	}
+
+	// Start the job in an idle worker if possible.
+	select {
+	case eg.jobs <- f:
+		return
+	default:
+	}
+
+	// Start a new worker if necessary.
+	select {
+	case eg.jobs <- f:
+		// A worker finished its previous job and took this one over.
+		return
+	case eg.sema <- struct{}{}:
+		// Start a new worker.
+		go eg.processJobs()
+		eg.jobs <- f
+	}
+}
+
+func (eg *Group) err(err error) {
+	eg.errMu.Lock()
+	defer eg.errMu.Unlock()
+
+	if eg.firstErr == nil {
+		eg.firstErr = err
+	}
+	eg.errs = append(eg.errs, err)
+}
+
+type ErrPanic struct {
+	Value interface{}
+}
+
+func (p ErrPanic) Error() string {
+	return fmt.Sprintf("panic: %v", p.Value)
+}
+
+var ErrGoexit = errors.New("runtime.Goexit used in job function")
+
+func (eg *Group) processJobs() {
+	// Notify pool of shutdown.
+	defer func() { <-eg.sema }()
+
+	// Handle panic and Goexit.
+	var finished bool
+	defer func() {
+		if !finished {
+			if p := recover(); p != nil {
+				eg.err(ErrPanic{p})
+			} else {
+				eg.err(ErrGoexit)
+			}
+		}
+	}()
+
+	// Run jobs from queue.
+	for jobFn := range eg.jobs {
+		err := jobFn()
+		if err != nil {
+			eg.err(err)
+		}
+	}
+
+	finished = true
+}
+
+func (eg *Group) Wait() error {
+	if eg.jobs == nil {
+		return nil
+	}
+	close(eg.jobs)
+	for i := 0; i < eg.PoolSize; i++ {
+		eg.sema <- struct{}{}
+	}
+	return eg.firstErr
+}
+
+func (eg *Group) Errors() []error {
+	return eg.errs
+}

client/egpool/egpool_test.go

@@ -0,0 +1,37 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package egpool_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/molecula/featurebase/v3/client/egpool"
+)
+
+func TestEGPool(t *testing.T) {
+	eg := egpool.Group{}
+
+	a := make([]int, 10)
+
+	for i := 0; i < 10; i++ {
+		i := i
+		eg.Go(func() error {
+			a[i] = i
+			if i == 7 {
+				return errors.New("blah")
+			}
+			return nil
+		})
+	}
+
+	err := eg.Wait()
+	if err == nil || err.Error() != "blah" {
+		t.Errorf("expected err blah, got: %v", err)
+	}
+
+	for i := 0; i < 10; i++ {
+		if a[i] != i {
+			t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i])
+		}
+	}
+}

client/error.go

@@ -0,0 +1,25 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+import "github.com/pkg/errors"
+
+// Predefined Pilosa errors.
+var (
+	ErrEmptyCluster                = errors.New("No usable addresses in the cluster")
+	ErrIndexExists                 = errors.New("Index exists")
+	ErrFieldExists                 = errors.New("Field exists")
+	ErrInvalidIndexName            = errors.New("Invalid index name")
+	ErrInvalidFieldName            = errors.New("Invalid field name")
+	ErrInvalidLabel                = errors.New("Invalid label")
+	ErrInvalidKey                  = errors.New("Invalid key")
+	ErrTriedMaxHosts               = errors.New("Tried max hosts, still failing")
+	ErrAddrURIClusterExpected      = errors.New("Addresses, URIs or a cluster is expected")
+	ErrInvalidQueryOption          = errors.New("Invalid query option")
+	ErrInvalidIndexOption          = errors.New("Invalid index option")
+	ErrInvalidFieldOption          = errors.New("Invalid field option")
+	ErrNoFragmentNodes             = errors.New("No fragment nodes")
+	ErrNoShard                     = errors.New("Index has no shards")
+	ErrUnknownType                 = errors.New("Unknown type")
+	ErrSingleServerAddressRequired = errors.New("OptClientManualServerAddress requires a single URI or address")
+	ErrPreconditionFailed          = errors.New("Precondition failed")
+)

client/ingest_api_batch.go

@@ -0,0 +1,145 @@
+package client
+
+import (
+	"time"
+
+	"github.com/molecula/featurebase/v3/logger"
+	"github.com/pkg/errors"
+)
+
+// NewIngestAPIBatch creates an alternate implementation of
+// RecordBatch which exists to aid in testing the new Ingest API and
+// is likely far slower than the Batch.
+func NewIngestAPIBatch(client *Client, size int, logger logger.Logger, fields []*Field) *ingestAPIBatch {
+	if len(fields) == 0 {
+		return nil
+	}
+
+	return &ingestAPIBatch{
+		client:    client,
+		log:       logger,
+		fields:    fields,
+		keyed:     fields[0].index.Opts().Keys(),
+		index:     fields[0].index.Name(),
+		batchSize: size,
+
+		recordsK: make(map[string]map[string]interface{}),
+		records:  make(map[uint64]map[string]interface{}),
+	}
+}
+
+type ingestAPIBatch struct {
+	client    *Client
+	log       logger.Logger
+	batchSize int
+
+	fields []*Field
+	keyed  bool
+	index  string
+
+	// map[recordKey][fieldName]value
+	recordsK map[string]map[string]interface{}
+	records  map[uint64]map[string]interface{}
+}
+
+func (b *ingestAPIBatch) Add(row Row) error {
+	if len(row.Clears) > 0 {
+		return errors.New("ingest api batch does not support clears")
+	}
+	values := make(map[string]interface{})
+	for i, val := range row.Values {
+		field := b.fields[i]
+		// val can be string, uint64, int64, []string, []uint64, nil
+		// TODO timestamp field might need special handling
+		// TODO check that the Row.Clears field is only used for packed bools, and then issue a warning/error (in IDK) if the ingest API mode is used in conjunction w/ packed bools.
+		if val == nil {
+			continue
+		}
+		zero := QuantizedTime{}
+		if field.Options().Type() == FieldTypeTime && row.Time != zero {
+			timeq, err := row.Time.Time()
+			if err != nil {
+				return errors.Wrap(err, "parsing row time")
+			}
+			values[field.Name()] = map[string]interface{}{"time": timeq.Format(time.RFC3339), "values": val}
+		} else {
+			values[field.Name()] = val
+		}
+	}
+
+	if b.keyed {
+		switch rowID := row.ID.(type) {
+		case string:
+			b.recordsK[rowID] = values
+		case []byte:
+			b.recordsK[string(rowID)] = values
+		default:
+			return errors.Errorf("unsupported rowID %v of type %[1]T, must be string, or []byte for keyed index", rowID)
+		}
+		if len(b.recordsK) >= b.batchSize {
+			return ErrBatchNowFull
+		}
+	} else {
+		rowID, ok := row.ID.(uint64)
+		if !ok {
+			return errors.Errorf("unsupported rowID %v of type %[1]T, must be uint64 for unkeyed index", row.ID)
+		}
+		b.records[rowID] = values
+		if len(b.records) >= b.batchSize {
+			return ErrBatchNowFull
+		}
+	}
+	return nil
+}
+
+func (b *ingestAPIBatch) Import() error {
+	if b.keyed {
+		return b.importKeyed()
+	}
+	return b.importUnkeyed()
+}
+
+func (b *ingestAPIBatch) importKeyed() error {
+	req := []map[string]interface{}{
+		{
+			"action":  "set",
+			"records": b.recordsK,
+		},
+	}
+	bod, err := b.client.IngestData(b.index, req)
+	if err != nil {
+		return errors.Wrapf(err, "importKeyed, body: %s", bod)
+	}
+
+	for k := range b.recordsK {
+		delete(b.recordsK, k)
+	}
+	return nil
+}
+
+func (b *ingestAPIBatch) importUnkeyed() error {
+	req := []map[string]interface{}{
+		{
+			"action":  "set",
+			"records": b.records,
+		},
+	}
+	bod, err := b.client.IngestData(b.index, req)
+	if err != nil {
+		return errors.Wrapf(err, "importKeyed, body: %s", bod)
+	}
+
+	for v := range b.records {
+		delete(b.records, v)
+	}
+	return nil
+}
+
+func (b *ingestAPIBatch) Len() int {
+	if b.keyed {
+		return len(b.recordsK)
+	}
+	return len(b.records)
+}
+
+func (b *ingestAPIBatch) Flush() error { return nil }

client/ingest_api_batch_test.go

@@ -0,0 +1,306 @@
+package client
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/molecula/featurebase/v3/logger"
+	"github.com/molecula/featurebase/v3/test"
+)
+
+func TestIngestAPIBatchAdd(t *testing.T) {
+	t.Run("unkeyed", func(t *testing.T) {
+		batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
+			{
+				name:  "a",
+				index: &Index{name: "idxname", options: &IndexOptions{}},
+				options: &FieldOptions{
+					fieldType: FieldTypeSet,
+				},
+			},
+			{
+				name:  "b",
+				index: &Index{name: "idxname", options: &IndexOptions{}},
+				options: &FieldOptions{
+					fieldType: FieldTypeSet,
+					keys:      true,
+				},
+			},
+			{
+				name:  "c",
+				index: &Index{name: "idxname", options: &IndexOptions{}},
+				options: &FieldOptions{
+					fieldType: FieldTypeTime,
+					keys:      true,
+				},
+			},
+		})
+		qt := QuantizedTime{}
+		qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
+		err := batch.Add(Row{
+			ID:     uint64(1),
+			Values: []interface{}{uint64(2), "bkey", "ckey"},
+			Time:   qt,
+		})
+		if err != nil {
+			t.Fatalf("adding row to batch: %v", err)
+		}
+
+		if batch.records[1]["a"] != uint64(2) {
+			t.Fatalf("unexpected batch.records: %+v", batch.records)
+		}
+		if batch.records[1]["b"] != "bkey" {
+			t.Fatalf("unexpected batch.records: %+v", batch.records)
+		}
+		if batch.records[1]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
+			t.Fatalf("unexpected batch.records: %+v", batch.records)
+		}
+		if batch.records[1]["c"].(map[string]interface{})["values"] != "ckey" {
+			t.Fatalf("unexpected batch.records: %+v", batch.records)
+		}
+
+	})
+
+	t.Run("keyed", func(t *testing.T) {
+		batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
+			{
+				name:  "a",
+				index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
+				options: &FieldOptions{
+					fieldType: FieldTypeSet,
+				},
+			},
+			{
+				name:  "b",
+				index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
+				options: &FieldOptions{
+					fieldType: FieldTypeSet,
+					keys:      true,
+				},
+			},
+			{
+				name:  "c",
+				index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
+				options: &FieldOptions{
+					fieldType: FieldTypeTime,
+					keys:      true,
+				},
+			},
+		})
+		qt := QuantizedTime{}
+		qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
+		err := batch.Add(Row{
+			ID:     "1",
+			Values: []interface{}{uint64(2), "bkey", "ckey"},
+			Time:   qt,
+		})
+
+		checkResult := func(batch *ingestAPIBatch, id string, err error) {
+			if err != nil {
+				t.Fatalf("adding row to batch: %v", err)
+			}
+
+			if batch.recordsK[id]["a"] != uint64(2) {
+				t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
+			}
+			if batch.recordsK[id]["b"] != "bkey" {
+				t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
+			}
+			if batch.recordsK[id]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
+				t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
+			}
+			if batch.recordsK[id]["c"].(map[string]interface{})["values"] != "ckey" {
+				t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
+			}
+		}
+		checkResult(batch, "1", err)
+
+		// test wrong type row ID
+		if err := batch.Add(Row{ID: 64.5}); !strings.Contains(err.Error(), "unsupported rowID") {
+			t.Fatalf("unexpected error w/ floating point rowID: %v", err)
+		}
+
+		// test that byte slice ID works same as string
+		err = batch.Add(Row{
+			ID:     []byte("2"),
+			Values: []interface{}{uint64(2), "bkey", "ckey"},
+			Time:   qt,
+		})
+		checkResult(batch, "2", err)
+
+	})
+}
+
+func TestIngestAPIBatch(t *testing.T) {
+	t.Skip("causing sporadic CI failures... on my list to debug, but this code doesn't affect anyone's production anyhow (jaffee)")
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	urls := make([]string, len(c.Nodes))
+	for i, n := range c.Nodes {
+		urls[i] = n.URL()
+	}
+
+	// Create a new client for the cluster
+	cli, err := newClientFromAddresses(urls, &ClientOptions{})
+	if err != nil {
+		t.Fatalf("getting new client: %v", err)
+	}
+	defer cli.Close()
+
+	cli.IngestSchema(map[string]interface{}{
+		"index-name":       "test-1",
+		"index-action":     "create",
+		"primary-key-type": "uint",
+		"field-action":     "create",
+		"fields": []map[string]interface{}{
+			{
+				"field-name":    "astr",
+				"field-type":    "string",
+				"field-options": map[string]interface{}{},
+			},
+			{
+				"field-name":    "bint",
+				"field-type":    "int",
+				"field-options": map[string]interface{}{},
+			},
+			{
+				"field-name":    "cid",
+				"field-type":    "id",
+				"field-options": map[string]interface{}{},
+			},
+			{
+				"field-name": "dtimestamp",
+				"field-type": "timestamp",
+				"field-options": map[string]interface{}{
+					"unit": "s",
+				},
+			},
+			{
+				"field-name": "etime",
+				"field-type": "string",
+				"field-options": map[string]interface{}{
+					"time-quantum": "YMD",
+				},
+			},
+			{
+				"field-name": "fdecimal",
+				"field-type": "decimal",
+				"field-options": map[string]interface{}{
+					"scale": 3,
+				},
+			},
+			{
+				"field-name":    "gbool",
+				"field-type":    "bool",
+				"field-options": map[string]interface{}{},
+			},
+		},
+	})
+
+	schema, err := cli.Schema()
+	if err != nil {
+		t.Fatalf("getting schema: %v", err)
+	}
+	index := schema.Index("test-1")
+	defer cli.DeleteIndex(index)
+
+	batch := NewIngestAPIBatch(cli, 10, logger.NopLogger, []*Field{
+		{
+			name:    "astr",
+			index:   &Index{name: "test-1", options: &IndexOptions{}},
+			options: &FieldOptions{fieldType: FieldTypeSet, keys: true},
+		},
+		{
+			name:    "bint",
+			options: &FieldOptions{fieldType: FieldTypeInt},
+		},
+		{
+			name:    "cid",
+			options: &FieldOptions{fieldType: FieldTypeSet, keys: false},
+		},
+		{
+			name:    "dtimestamp",
+			options: &FieldOptions{fieldType: FieldTypeTimestamp},
+		},
+		{
+			name:    "etime",
+			options: &FieldOptions{fieldType: FieldTypeTime, keys: true, timeQuantum: TimeQuantumYearMonthDay},
+		},
+		{
+			name:    "fdecimal",
+			options: &FieldOptions{fieldType: FieldTypeDecimal, scale: 3},
+		},
+		{
+			name:    "gbool",
+			options: &FieldOptions{fieldType: FieldTypeBool},
+		},
+	})
+
+	qt0 := &QuantizedTime{}
+	qt0.Set(time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC))
+	if err := batch.Add(Row{
+		ID:     uint64(7),
+		Values: []interface{}{"a", -2, 9, 1287367623, "e", 1.2345, true},
+		Time:   *qt0,
+	}); err != nil {
+		t.Fatalf("adding row: %v", err)
+	}
+
+	// test nil value case
+	if err := batch.Add(Row{
+		ID:     uint64(8),
+		Values: []interface{}{nil, nil, nil, nil, nil, nil, nil},
+		Time:   QuantizedTime{},
+	}); err != nil {
+		t.Fatalf("error adding all nil batch which should affect nothing: %v", err)
+	}
+
+	if err := batch.Import(); err != nil {
+		t.Fatalf("importing row: %v", err)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(astr=a)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(bint==-2) result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(cid=9) result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(dtimestamp=='2010-10-18T02:07:03Z') result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(etime=e, from='2010-01-01', to='2010-01-02') result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(fdecimal==1.234) result: %+v", resp.Result().Row().Columns)
+	}
+
+	if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
+		t.Fatalf("querying: %v", err)
+	} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
+		t.Fatalf("unexpected Row(gbool=true) result: %+v", resp.Result().Row().Columns)
+	}
+
+}

client/logimport.go

@@ -0,0 +1,32 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+import (
+	"encoding/gob"
+	"io"
+)
+
+type importLog struct {
+	Index     string
+	Path      string
+	Shard     uint64
+	IsRoaring bool
+	Timestamp int64 // Unix Nanoseconds
+	Data      []byte
+}
+
+type encoder interface {
+	Encode(thing interface{}) error
+}
+
+func newImportLogEncoder(w io.Writer) encoder {
+	return gob.NewEncoder(w)
+}
+
+type decoder interface {
+	Decode(thing interface{}) error
+}
+
+func newImportLogDecoder(r io.Reader) decoder {
+	return gob.NewDecoder(r)
+}

client/logimport_test.go

@@ -0,0 +1,142 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	tests := []importLog{
+		{
+			Index: "go-testindex",
+			Path:  "/index/go-testindex/field/importfield-batchsize/import?clear=false",
+			Shard: 0,
+			Data:  make([]byte, 3918),
+		},
+		{
+			Index: "go-testindex",
+			Path:  "/index/go-testindex/field/importfield-batchsize/import?clear=false",
+			Shard: 0,
+			Data:  make([]byte, 3918),
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 9,
+			Data:  []byte("something"),
+		},
+		{
+			Index: "",
+			Path:  "",
+			Shard: 0,
+			Data:  nil,
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 10,
+			Data:  []byte("blahaslkdjfeoiwujf"),
+		},
+		{
+			Index: "eheh",
+			Path:  "blah",
+			Shard: 10,
+			Data:  make([]byte, 10000),
+		},
+		{
+			Index: "zoop",
+			Path:  "blah",
+			Shard: 8923734,
+			Data:  []byte("blahaslkdjfeoiwujf"),
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			nl := importLog{
+				Index: test.Index,
+				Path:  test.Path,
+				Shard: test.Shard,
+				Data:  make([]byte, len(test.Data)),
+			}
+			copy(nl.Data, test.Data)
+			buf := &bytes.Buffer{}
+			enc := newImportLogEncoder(buf)
+			err := enc.Encode(nl)
+			if err != nil {
+				t.Fatalf("writing to buf: %v", err)
+			}
+			dec := newImportLogDecoder(buf)
+			l2 := &importLog{}
+			err = dec.Decode(l2)
+			if err != nil {
+				t.Fatalf("reading from buf: %v", err)
+			}
+			if l2.Index != test.Index {
+				t.Errorf("indexes not equal:\n%s\n%s", test.Index, l2.Index)
+			}
+			if l2.Path != test.Path {
+				t.Errorf("paths not equal:\n%s\n%s", test.Path, l2.Path)
+			}
+			if l2.Shard != test.Shard {
+				t.Errorf("shards not equal exp: %d got %d", test.Shard, l2.Shard)
+			}
+			if !reflect.DeepEqual(test.Data, l2.Data) {
+				t.Errorf("data not equal \n%v\n%v", test.Data, l2.Data)
+			}
+
+		})
+	}
+	buf, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("getting temp file: %v", err)
+	}
+	enc := newImportLogEncoder(buf)
+	for _, test := range tests {
+		a := &test
+		err := enc.Encode(a)
+		if err != nil {
+			t.Errorf("encoding to buf: %v", err)
+		}
+	}
+
+	name := buf.Name()
+	err = buf.Close()
+	if err != nil {
+		t.Fatalf("closing temp file: %v", err)
+	}
+
+	buf, err = os.Open(name)
+	if err != nil {
+		t.Fatalf("reopening: %v", err)
+	}
+
+	dec := newImportLogDecoder(buf)
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			l := &importLog{}
+			err := dec.Decode(l)
+			// err := l.ReadFrom(buf)
+			if err != nil {
+				t.Errorf("reading from buf: %v", err)
+			}
+			if l.Index != test.Index {
+				t.Errorf("indexes not equal:\n%s\n%s", test.Index, l.Index)
+			}
+			if l.Path != test.Path {
+				t.Errorf("paths not equal:\n%s\n%s", test.Path, l.Path)
+			}
+			if l.Shard != test.Shard {
+				t.Errorf("shards not equal exp: %d got %d", test.Shard, l.Shard)
+			}
+			if !reflect.DeepEqual(test.Data, l.Data) {
+				t.Errorf("data not equal \n%v\n%v", test.Data, l.Data)
+			}
+		})
+	}
+}

client/metrics.go

@@ -0,0 +1,16 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package client
+
+const (
+	// MetricBatchImportDurationSeconds records the full time of the
+	// RecordBatch.Import call. This includes starting and finishing a
+	// transaction, doing key translation, building fragments locally,
+	// importing all data, and resetting internal structures.
+	MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
+
+	// MetricBatchFlushDurationSeconds records the full time for
+	// RecordBatch.Flush (if splitBatchMode is in use). This includes
+	// starting and finishing a transaction, importing all data, and
+	// resetting internal structures.
+	MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
+)

client/record.go

@@ -0,0 +1,62 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+// Record is a Column or a FieldValue.
+type Record interface {
+	Shard(shardWidth uint64) uint64
+	Less(other Record) bool
+}
+
+// RecordIterator is an iterator for a record.
+type RecordIterator interface {
+	NextRecord() (Record, error)
+}
+
+// Column defines a single Pilosa column.
+type Column struct {
+	RowID     uint64
+	ColumnID  uint64
+	RowKey    string
+	ColumnKey string
+	Timestamp int64
+}
+
+// Shard returns the shard for this column.
+func (b Column) Shard(shardWidth uint64) uint64 {
+	return b.ColumnID / shardWidth
+}
+
+// Less returns true if this column sorts before the given Record.
+func (b Column) Less(other Record) bool {
+	if ob, ok := other.(Column); ok {
+		if b.RowID == ob.RowID {
+			return b.ColumnID < ob.ColumnID
+		}
+		return b.RowID < ob.RowID
+	}
+	return false
+}
+
+// FieldValue represents the value for a column within a
+// range-encoded field.
+type FieldValue struct {
+	ColumnID  uint64
+	ColumnKey string
+	Value     int64
+}
+
+// Shard returns the shard for this field value.
+func (v FieldValue) Shard(shardWidth uint64) uint64 {
+	return v.ColumnID / shardWidth
+}
+
+// Less returns true if this field value sorts before the given Record.
+func (v FieldValue) Less(other Record) bool {
+	if ov, ok := other.(FieldValue); ok {
+		return v.ColumnID < ov.ColumnID
+	}
+	return false
+}

client/record_test.go

@@ -0,0 +1,70 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client_test
+
+import (
+	"testing"
+
+	"github.com/molecula/featurebase/v3/client"
+)
+
+func TestColumnShard(t *testing.T) {
+	a := client.Column{RowID: 15, ColumnID: 55, Timestamp: 100101}
+	target := uint64(0)
+	if a.Shard(100) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(100))
+	}
+	target = 5
+	if a.Shard(10) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(10))
+	}
+}
+
+func TestColumnLess(t *testing.T) {
+	a := client.Column{RowID: 10, ColumnID: 200}
+	a2 := client.Column{RowID: 10, ColumnID: 1000}
+	b := client.Column{RowID: 200, ColumnID: 10}
+	c := client.FieldValue{ColumnID: 1}
+	if !a.Less(a2) {
+		t.Fatalf("%v should be less than %v", a, a2)
+	}
+	if !a.Less(b) {
+		t.Fatalf("%v should be less than %v", a, b)
+	}
+	if b.Less(a) {
+		t.Fatalf("%v should not be less than %v", b, a)
+	}
+	if c.Less(a) {
+		t.Fatalf("%v should not be less than %v", c, a)
+	}
+}
+
+func TestFieldValueShard(t *testing.T) {
+	a := client.FieldValue{ColumnID: 55, Value: 125}
+	target := uint64(0)
+	if a.Shard(100) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(100))
+	}
+	target = 5
+	if a.Shard(10) != target {
+		t.Fatalf("shard %d != %d", target, a.Shard(10))
+	}
+
+}
+
+func TestFieldValueLess(t *testing.T) {
+	a := client.FieldValue{ColumnID: 55, Value: 125}
+	b := client.FieldValue{ColumnID: 100, Value: 125}
+	c := client.Column{ColumnID: 1, RowID: 2}
+	if !a.Less(b) {
+		t.Fatalf("%v should be less than %v", a, b)
+	}
+	if b.Less(a) {
+		t.Fatalf("%v should not be less than %v", b, a)
+	}
+	if c.Less(a) {
+		t.Fatalf("%v should not be less than %v", c, a)
+	}
+}

client/response.go

@@ -0,0 +1,495 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/molecula/featurebase/v3/pb"
+)
+
+// QueryResponse types.
+const (
+	QueryResultTypeNil uint32 = iota
+	QueryResultTypeRow
+	QueryResultTypePairs
+	QueryResultTypePairsField
+	QueryResultTypeValCount
+	QueryResultTypeUint64
+	QueryResultTypeBool
+	QueryResultTypeRowIDs // this is not used by the client
+	QueryResultTypeGroupCounts
+	QueryResultTypeRowIdentifiers
+	QueryResultTypePair
+	QueryResultTypePairField
+	QueryResultTypeSignedRow
+)
+
+// QueryResponse represents the response from a Pilosa query.
+type QueryResponse struct {
+	ResultList   []QueryResult `json:"results,omitempty"`
+	ErrorMessage string        `json:"error-message,omitempty"`
+	Success      bool          `json:"success,omitempty"`
+}
+
+func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, error) {
+	if response.Err != "" {
+		return &QueryResponse{
+			ErrorMessage: response.Err,
+			Success:      false,
+		}, nil
+	}
+	results := make([]QueryResult, 0, len(response.Results))
+	for _, r := range response.Results {
+		result, err := newQueryResultFromInternal(r)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, result)
+	}
+
+	return &QueryResponse{
+		ResultList: results,
+		Success:    true,
+	}, nil
+}
+
+// Results returns all results in the response.
+func (qr *QueryResponse) Results() []QueryResult {
+	return qr.ResultList
+}
+
+// Result returns the first result or nil.
+func (qr *QueryResponse) Result() QueryResult {
+	if len(qr.ResultList) == 0 {
+		return nil
+	}
+	return qr.ResultList[0]
+}
+
+// QueryResult represents one of the results in the response.
+type QueryResult interface {
+	Type() uint32
+	Row() RowResult
+	CountItems() []CountResultItem
+	CountItem() CountResultItem
+	Count() int64
+	Value() int64
+	Changed() bool
+	GroupCounts() []GroupCount
+	RowIdentifiers() RowIdentifiersResult
+}
+
+func newQueryResultFromInternal(result *pb.QueryResult) (QueryResult, error) {
+	switch result.Type {
+	case QueryResultTypeNil:
+		return NilResult{}, nil
+	case QueryResultTypeRow:
+		return newRowResultFromInternal(result.Row)
+	case QueryResultTypePairs:
+		return countItemsFromInternal(result.Pairs), nil
+	case QueryResultTypePairsField:
+		return countItemsFromInternal(result.PairsField.Pairs), nil
+	case QueryResultTypeValCount:
+		return &ValCountResult{
+			Val: result.ValCount.Val,
+			Cnt: result.ValCount.Count,
+		}, nil
+	case QueryResultTypeUint64:
+		return IntResult(result.N), nil
+	case QueryResultTypeBool:
+		return BoolResult(result.Changed), nil
+	case QueryResultTypeRowIdentifiers:
+		return &RowIdentifiersResult{
+			IDs:  result.RowIdentifiers.Rows,
+			Keys: result.RowIdentifiers.Keys,
+		}, nil
+	case QueryResultTypeGroupCounts:
+		return groupCountsFromInternal(result.GroupCounts), nil
+	case QueryResultTypePair:
+		return CountItem{CountResultItem: countItemFromInternal(result.Pairs[0])}, nil
+	case QueryResultTypePairField:
+		return CountItem{CountResultItem: countItemFromInternal(result.PairField.Pair)}, nil
+	}
+
+	return nil, ErrUnknownType
+}
+
+// CountResultItem represents a result from TopN call.
+type CountResultItem struct {
+	ID    uint64 `json:"id"`
+	Key   string `json:"key,omitempty"`
+	Count uint64 `json:"count"`
+}
+
+func (c *CountResultItem) String() string {
+	if c.Key != "" {
+		return fmt.Sprintf("%s:%d", c.Key, c.Count)
+	}
+	return fmt.Sprintf("%d:%d", c.ID, c.Count)
+}
+
+type CountItem struct {
+	CountResultItem
+}
+
+// Type is the type of this result.
+func (CountItem) Type() uint32 { return QueryResultTypePairField }
+
+// Row returns a RowResult.
+func (CountItem) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (t CountItem) CountItems() []CountResultItem { return []CountResultItem{t.CountResultItem} }
+
+// CountItem returns a CountResultItem
+func (t CountItem) CountItem() CountResultItem { return t.CountResultItem }
+
+// Count returns the result of a Count call.
+func (CountItem) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (CountItem) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (CountItem) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (CountItem) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (CountItem) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+func countItemFromInternal(item *pb.Pair) CountResultItem {
+	return CountResultItem{ID: item.ID, Key: item.Key, Count: item.Count}
+}
+
+func countItemsFromInternal(items []*pb.Pair) TopNResult {
+	result := make([]CountResultItem, 0, len(items))
+	for _, v := range items {
+		result = append(result, countItemFromInternal(v))
+	}
+	return TopNResult(result)
+}
+
+// TopNResult is returned from TopN call.
+type TopNResult []CountResultItem
+
+// Type is the type of this result.
+func (TopNResult) Type() uint32 { return QueryResultTypePairsField }
+
+// Row returns a RowResult.
+func (TopNResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (t TopNResult) CountItems() []CountResultItem { return t }
+
+// CountItem returns a CountResultItem
+func (t TopNResult) CountItem() CountResultItem {
+	if len(t) >= 1 {
+		return t[0]
+	}
+	return CountResultItem{}
+}
+
+// Count returns the result of a Count call.
+func (TopNResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (TopNResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (TopNResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (TopNResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls.
+type RowResult struct {
+	Columns []uint64 `json:"columns"`
+	Keys    []string `json:"keys"`
+}
+
+func newRowResultFromInternal(row *pb.Row) (*RowResult, error) {
+	return &RowResult{
+		Columns: row.Columns,
+		Keys:    row.Keys,
+	}, nil
+}
+
+// Type is the type of this result.
+func (RowResult) Type() uint32 { return QueryResultTypeRow }
+
+// Row returns a RowResult.
+func (b RowResult) Row() RowResult { return b }
+
+// CountItems returns a CountResultItem slice.
+func (RowResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (RowResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (RowResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (RowResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (RowResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (RowResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (RowResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// MarshalJSON serializes this row result.
+func (b RowResult) MarshalJSON() ([]byte, error) {
+	columns := b.Columns
+	if columns == nil {
+		columns = []uint64{}
+	}
+	keys := b.Keys
+	if keys == nil {
+		keys = []string{}
+	}
+	return json.Marshal(struct {
+		Columns []uint64 `json:"columns"`
+		Keys    []string `json:"keys"`
+	}{
+		Columns: columns,
+		Keys:    keys,
+	})
+}
+
+// ValCountResult is returned from Min, Max and Sum calls.
+type ValCountResult struct {
+	Val int64 `json:"val"`
+	Cnt int64 `json:"count"`
+}
+
+// Type is the type of this result.
+func (ValCountResult) Type() uint32 { return QueryResultTypeValCount }
+
+// Row returns a RowResult.
+func (ValCountResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (ValCountResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (ValCountResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (c ValCountResult) Count() int64 { return c.Cnt }
+
+// Value returns the result of a Min, Max or Sum call.
+func (c ValCountResult) Value() int64 { return c.Val }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (ValCountResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (ValCountResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (ValCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// IntResult is returned from Count call.
+type IntResult int64
+
+// Type is the type of this result.
+func (IntResult) Type() uint32 { return QueryResultTypeUint64 }
+
+// Row returns a RowResult.
+func (IntResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (IntResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (IntResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (i IntResult) Count() int64 { return int64(i) }
+
+// Value returns the result of a Min, Max or Sum call.
+func (IntResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (IntResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (IntResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (IntResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// BoolResult is returned from Set and Clear calls.
+type BoolResult bool
+
+// Type is the type of this result.
+func (BoolResult) Type() uint32 { return QueryResultTypeBool }
+
+// Row returns a RowResult.
+func (BoolResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (BoolResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (BoolResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (BoolResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (BoolResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (b BoolResult) Changed() bool { return bool(b) }
+
+// GroupCounts returns the result of a GroupBy call.
+func (BoolResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// NilResult is returned from calls which don't return a value.
+type NilResult struct{}
+
+// Type is the type of this result.
+func (NilResult) Type() uint32 { return QueryResultTypeNil }
+
+// Row returns a RowResult.
+func (NilResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (NilResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (NilResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (NilResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (NilResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (NilResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (NilResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (NilResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// FieldRow represents a Group in a GroupBy call result.
+type FieldRow struct {
+	FieldName string `json:"field"`
+	RowID     uint64 `json:"rowID"`
+	RowKey    string `json:"rowKey"`
+	Value     *int64 `json:"value,omitempty"`
+}
+
+// GroupCount contains groups and their count in a GroupBy call result.
+type GroupCount struct {
+	Groups []FieldRow `json:"groups"`
+	Count  int64      `json:"count"`
+	Agg    int64      `json:"agg"`
+}
+
+// GroupCountResult is returned from GroupBy call.
+type GroupCountResult []GroupCount
+
+// Type is the type of this result.
+func (GroupCountResult) Type() uint32 { return QueryResultTypeGroupCounts }
+
+// Row returns a RowResult.
+func (GroupCountResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (GroupCountResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (GroupCountResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (GroupCountResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (GroupCountResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (GroupCountResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (r GroupCountResult) GroupCounts() []GroupCount { return r }
+
+// RowIdentifiers returns the result of a Rows call.
+func (GroupCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
+
+// RowIdentifiersResult is returned from a Rows call.
+type RowIdentifiersResult struct {
+	IDs  []uint64 `json:"ids"`
+	Keys []string `json:"keys,omitempty"`
+}
+
+// Type is the type of this result.
+func (RowIdentifiersResult) Type() uint32 { return QueryResultTypeRowIdentifiers }
+
+// Row returns a RowResult.
+func (RowIdentifiersResult) Row() RowResult { return RowResult{} }
+
+// CountItems returns a CountResultItem slice.
+func (RowIdentifiersResult) CountItems() []CountResultItem { return nil }
+
+// CountItem returns a CountResultItem
+func (RowIdentifiersResult) CountItem() CountResultItem { return CountResultItem{} }
+
+// Count returns the result of a Count call.
+func (RowIdentifiersResult) Count() int64 { return 0 }
+
+// Value returns the result of a Min, Max or Sum call.
+func (RowIdentifiersResult) Value() int64 { return 0 }
+
+// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
+func (RowIdentifiersResult) Changed() bool { return false }
+
+// GroupCounts returns the result of a GroupBy call.
+func (RowIdentifiersResult) GroupCounts() []GroupCount { return nil }
+
+// RowIdentifiers returns the result of a Rows call.
+func (r RowIdentifiersResult) RowIdentifiers() RowIdentifiersResult { return r }
+
+func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult {
+	result := make([]GroupCount, 0, len(items.Groups))
+	for _, g := range items.Groups {
+		groups := make([]FieldRow, 0, len(g.Group))
+		for _, f := range g.Group {
+			fr := FieldRow{
+				FieldName: f.Field,
+				RowID:     f.RowID,
+				RowKey:    f.RowKey,
+			}
+			if f.Value != nil {
+				fr.Value = &f.Value.Value
+			}
+			groups = append(groups, fr)
+		}
+		result = append(result, GroupCount{
+			Groups: groups,
+			Count:  int64(g.Count),
+			Agg:    int64(g.Agg),
+		})
+	}
+	return GroupCountResult(result)
+}

client/response_test.go

@@ -0,0 +1,272 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"reflect"
+	"testing"
+
+	"github.com/molecula/featurebase/v3/pb"
+)
+
+func TestNewRowResultFromInternal(t *testing.T) {
+	targetColumns := []uint64{5, 10}
+	row := &pb.Row{
+		Columns: []uint64{5, 10},
+	}
+	result, err := newRowResultFromInternal(row)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	if !reflect.DeepEqual(targetColumns, result.Columns) {
+		t.Fatal()
+	}
+}
+
+func TestNewQueryResponseFromInternal(t *testing.T) {
+	targetColumns := []uint64{5, 10}
+	targetCountItems := []CountResultItem{
+		{ID: 10, Count: 100},
+	}
+	row := &pb.Row{
+		Columns: []uint64{5, 10},
+	}
+	pairs := []*pb.Pair{
+		{ID: 10, Count: 100},
+	}
+	response := &pb.QueryResponse{
+		Results: []*pb.QueryResult{
+			{Type: QueryResultTypeRow, Row: row},
+			{Type: QueryResultTypePairs, Pairs: pairs},
+		},
+		Err: "",
+	}
+	qr, err := newQueryResponseFromInternal(response)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	if qr.ErrorMessage != "" {
+		t.Fatalf("ErrorMessage should be empty")
+	}
+	if !qr.Success {
+		t.Fatalf("IsSuccess should be true")
+	}
+
+	results := qr.Results()
+	if len(results) != 2 {
+		t.Fatalf("Number of results should be 2")
+	}
+	if results[0] != qr.Result() {
+		t.Fatalf("Result() should return the first result")
+	}
+	if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) {
+		t.Fatalf("The row result should contain the columns")
+	}
+	if !reflect.DeepEqual(targetCountItems, results[1].CountItems()) {
+		t.Fatalf("The response should include count items")
+	}
+}
+
+func TestNewQueryResponseWithErrorFromInternal(t *testing.T) {
+	response := &pb.QueryResponse{
+		Err: "some error",
+	}
+	qr, err := newQueryResponseFromInternal(response)
+	if err != nil {
+		t.Fatalf("Failed with error: %s", err)
+	}
+	if qr.ErrorMessage != "some error" {
+		t.Fatalf("The response should include the error message")
+	}
+	if qr.Success {
+		t.Fatalf("IsSuccess should be false")
+	}
+	if qr.Result() != nil {
+		t.Fatalf("If there are no results, Result should return nil")
+	}
+}
+
+func TestCountResultItemToString(t *testing.T) {
+	tests := []struct {
+		item     *CountResultItem
+		expected string
+	}{
+		{item: &CountResultItem{ID: 100, Count: 50}, expected: "100:50"},
+		{item: &CountResultItem{Key: "blah", Count: 50}, expected: "blah:50"},
+		{item: &CountResultItem{Key: "blah", ID: 22, Count: 50}, expected: "blah:50"},
+		{item: &CountResultItem{Key: "blah", ID: 22}, expected: "blah:0"},
+		{item: &CountResultItem{}, expected: "0:0"},
+	}
+
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d: ", i), func(t *testing.T) {
+			if tst.expected != tst.item.String() {
+				t.Fatalf("%s != %s", tst.expected, tst.item.String())
+			}
+		})
+	}
+}
+
+func TestMarshalResults(t *testing.T) {
+	row := &pb.Row{
+		Columns: []uint64{5, 10},
+	}
+	pairs := []*pb.Pair{
+		{ID: 10, Count: 100},
+	}
+	pbufResults := []*pb.QueryResult{
+		{Type: QueryResultTypeRow, Row: row},
+		{Type: QueryResultTypePairs, Pairs: pairs},
+	}
+	resultJSONStrings := make([]string, len(pbufResults))
+	for i, pr := range pbufResults {
+		r, err := newQueryResultFromInternal(pr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		b, err := json.Marshal(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resultJSONStrings[i] = string(b)
+	}
+	targetJSON := []string{
+		`{"columns":[5,10],"keys":[]}`,
+		`[{"id":10,"count":100}]`,
+	}
+	for i := range targetJSON {
+		if sortedString(targetJSON[i]) != sortedString(resultJSONStrings[i]) {
+			t.Fatalf("%v != %v ", targetJSON[i], resultJSONStrings[i])
+		}
+	}
+
+}
+
+func TestUnknownQueryResultType(t *testing.T) {
+	result := &pb.QueryResult{
+		Type: 999,
+	}
+	_, err := newQueryResultFromInternal(result)
+	if err != ErrUnknownType {
+		t.Fatalf("Should have failed with ErrUnknownType")
+	}
+}
+
+func TestTopNResult(t *testing.T) {
+	result := TopNResult{
+		CountResultItem{ID: 100, Count: 10},
+	}
+	expectResult(t, result, QueryResultTypePairsField, RowResult{}, []CountResultItem{{100, "", 10}}, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestRowResult(t *testing.T) {
+	result := RowResult{
+		Columns: []uint64{1, 2, 3},
+	}
+	targetBmp := RowResult{
+		Columns: []uint64{1, 2, 3},
+	}
+	expectResult(t, result, QueryResultTypeRow, targetBmp, nil, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestRowResultNilColumns(t *testing.T) {
+	result := RowResult{
+		Columns: nil,
+	}
+	_, err := result.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSumCountResult(t *testing.T) {
+	result := ValCountResult{
+		Val: 100,
+		Cnt: 50,
+	}
+	expectResult(t, result, QueryResultTypeValCount, RowResult{}, nil, 100, 50, false, nil, RowIdentifiersResult{})
+}
+
+func TestIntResult(t *testing.T) {
+	result := IntResult(11)
+	expectResult(t, result, QueryResultTypeUint64, RowResult{}, nil, 0, 11, false, nil, RowIdentifiersResult{})
+}
+
+func TestBoolResult(t *testing.T) {
+	result := BoolResult(true)
+	expectResult(t, result, QueryResultTypeBool, RowResult{}, nil, 0, 0, true, nil, RowIdentifiersResult{})
+}
+
+func TestNilResult(t *testing.T) {
+	result := NilResult{}
+	expectResult(t, result, QueryResultTypeNil, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{})
+}
+
+func TestGroupCountResult(t *testing.T) {
+	result := GroupCountResult{
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
+	}
+	expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
+		{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
+	}, RowIdentifiersResult{})
+}
+
+func TestGroupCountWithValueResult(t *testing.T) {
+	var a, b int64 = -1, 1
+
+	result := GroupCountResult{
+		{Groups: []FieldRow{{FieldName: "f1", Value: &a}}, Count: 1},
+		{Groups: []FieldRow{{FieldName: "f1", Value: &b}}, Count: 1},
+	}
+
+	var aa, bb int64 = -1, 1
+	expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
+		{Groups: []FieldRow{{FieldName: "f1", Value: &aa}}, Count: 1},
+		{Groups: []FieldRow{{FieldName: "f1", Value: &bb}}, Count: 1},
+	}, RowIdentifiersResult{})
+}
+
+func TestRowIdentifiersResult(t *testing.T) {
+	result := RowIdentifiersResult{
+		IDs: []uint64{1, 2, 3, 4},
+	}
+	expectResult(t, result, QueryResultTypeRowIdentifiers, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{
+		IDs: []uint64{1, 2, 3, 4},
+	})
+}
+
+func expectResult(t *testing.T, r QueryResult, resultType uint32, bmp RowResult,
+	countItems []CountResultItem, sum int64, count int64, changed bool,
+	groupCounts []GroupCount, rowIdentifiers RowIdentifiersResult) {
+	if resultType != r.Type() {
+		log.Fatalf("Result type: %d != %d", resultType, r.Type())
+	}
+	if !reflect.DeepEqual(bmp, r.Row()) {
+		log.Fatalf("Row: %v != %v", bmp, r.Row())
+	}
+	if !reflect.DeepEqual(countItems, r.CountItems()) {
+		log.Fatalf("Count items: %v != %v", countItems, r.CountItems())
+	}
+	if count != r.Count() {
+		log.Fatalf("Count: %d != %d", count, r.Count())
+	}
+	if sum != r.Value() {
+		log.Fatalf("Sum: %d != %d", sum, r.Value())
+	}
+	if changed != r.Changed() {
+		log.Fatalf("Changed: %v != %v", changed, r.Changed())
+	}
+	if !reflect.DeepEqual(groupCounts, r.GroupCounts()) {
+		log.Fatalf("Group counts: %v != %v", groupCounts, r.GroupCounts())
+	}
+	if !reflect.DeepEqual(rowIdentifiers, r.RowIdentifiers()) {
+		log.Fatalf("Row identifiers: %v != %v", rowIdentifiers, r.RowIdentifiers())
+	}
+}

client/shardnodes.go

@@ -0,0 +1,53 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"sync"
+
+	pnet "github.com/molecula/featurebase/v3/net"
+)
+
+type shardNodes struct {
+	data map[string]map[uint64][]*pnet.URI
+	mu   *sync.RWMutex
+}
+
+func newShardNodes() shardNodes {
+	return shardNodes{
+		data: make(map[string]map[uint64][]*pnet.URI),
+		mu:   &sync.RWMutex{},
+	}
+}
+
+func (s shardNodes) Get(index string, shard uint64) ([]*pnet.URI, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if idx, ok := s.data[index]; ok {
+		if uris, ok := idx[shard]; ok {
+			return uris, true
+		}
+	}
+	return nil, false
+}
+
+func (s shardNodes) Put(index string, shard uint64, uris []*pnet.URI) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	idx, ok := s.data[index]
+	if !ok {
+		idx = make(map[uint64][]*pnet.URI)
+	}
+	idx[shard] = uris
+	s.data[index] = idx
+}
+
+func (s shardNodes) Invalidate() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for k := range s.data {
+		delete(s.data, k)
+	}
+}

client/tracer.go

@@ -0,0 +1,76 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/log"
+)
+
+type NoopTracer struct{}
+
+type NoopSpan struct{}
+
+func (s NoopSpan) Finish() {
+	// pass
+}
+func (s NoopSpan) FinishWithOptions(opts opentracing.FinishOptions) {
+	// pass
+}
+
+func (s NoopSpan) Context() opentracing.SpanContext {
+	return nil
+}
+func (s NoopSpan) SetOperationName(operationName string) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) SetTag(key string, value interface{}) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) LogFields(fields ...log.Field) {
+	// pass
+}
+
+func (s NoopSpan) LogKV(alternatingKeyValues ...interface{}) {
+	// pass
+}
+
+func (s NoopSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span {
+	return s
+}
+
+func (s NoopSpan) BaggageItem(restrictedKey string) string {
+	return ""
+}
+
+func (s NoopSpan) Tracer() opentracing.Tracer {
+	return nil
+}
+
+func (s NoopSpan) LogEvent(event string) {
+	// pass
+}
+
+func (s NoopSpan) LogEventWithPayload(event string, payload interface{}) {
+	// pass
+}
+
+func (s NoopSpan) Log(data opentracing.LogData) {
+	// pass
+}
+
+func (t NoopTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
+	return NoopSpan{}
+}
+
+func (t NoopTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
+	return nil
+}
+
+func (t NoopTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
+	return nil, nil
+}

client/validate.go

@@ -0,0 +1,41 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import (
+	"regexp"
+)
+
+const (
+	maxLabel = 64
+	maxKey   = 64
+)
+
+var labelRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_-]*$")
+var keyRegex = regexp.MustCompile("^[A-Za-z0-9_{}+/=.~%:-]*$")
+
+// ValidLabel returns true if the given label is valid, otherwise false.
+func ValidLabel(label string) bool {
+	return len(label) <= maxLabel && labelRegex.Match([]byte(label))
+}
+
+// ValidKey returns true if the given key is valid, otherwise false.
+func ValidKey(key string) bool {
+	return len(key) <= maxKey && keyRegex.Match([]byte(key))
+}
+
+func validateLabel(label string) error {
+	if ValidLabel(label) {
+		return nil
+	}
+	return ErrInvalidLabel
+}
+
+func validateKey(key string) error {
+	if ValidKey(key) {
+		return nil
+	}
+	return ErrInvalidKey
+}

client/validate_test.go

@@ -0,0 +1,60 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+import "testing"
+
+func TestValidateLabel(t *testing.T) {
+	labels := []string{
+		"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, label := range labels {
+		if validateLabel(label) != nil {
+			t.Fatalf("Should be valid label: %s", label)
+		}
+	}
+}
+
+func TestValidateLabelInvalid(t *testing.T) {
+	labels := []string{
+		"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, label := range labels {
+		if validateLabel(label) == nil {
+			t.Fatalf("Should be invalid label: %s", label)
+		}
+	}
+}
+
+func TestValidateKey(t *testing.T) {
+	keys := []string{
+		"", "1", "ab", "ab1", "b-c", "d_e", "pilosa.com",
+		"bbf8d41c-7dba-40c4-94dc-94677b43bcf3",                 // UUID
+		"{bbf8d41c-7dba-40c4-94dc-94677b43bcf3}",               // Windows GUID
+		"https%3A//www.pilosa.com/about/%23contact",            // escaped URL
+		"aHR0cHM6Ly93d3cucGlsb3NhLmNvbS9hYm91dC8jY29udGFjdA==", // base64
+		"urn:isbn:1234567",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, key := range keys {
+		if validateKey(key) != nil {
+			t.Fatalf("Should be valid key: %s", key)
+		}
+	}
+}
+
+func TestValidateKeyInvalid(t *testing.T) {
+	keys := []string{
+		"\"", "'", "slice\\dice", "valid?no", "yüce", "*xyz", "with space", "<script>",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, key := range keys {
+		if validateKey(key) == nil {
+			t.Fatalf("Should be invalid key: %s", key)
+		}
+	}
+}

client/version.go

@@ -0,0 +1,8 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+// package ctl contains all pilosa subcommands other than 'server'. These are
+// generally administration, testing, and debugging tools.
+
+package client
+
+// Version is the client version.
+const Version = "v1.3.0"

cluster_internal_test.go

@@ -1,56 +1,37 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 
 import (
-	"bytes"
+	"context"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"os"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/gorilla/mux"
-	"github.com/pilosa/pilosa/v2/logger"
-	"github.com/pilosa/pilosa/v2/roaring"
-	"github.com/pkg/errors"
+	pnet "github.com/molecula/featurebase/v3/net"
+	"github.com/molecula/featurebase/v3/roaring"
+	"github.com/molecula/featurebase/v3/testhook"
+	"github.com/molecula/featurebase/v3/topology"
+	. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
 )
 
 // Ensure that fragCombos creates the correct fragment mapping.
 func TestFragCombos(t *testing.T) {
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
 
 	c := newCluster()
 	c.addNodeBasicSorted(node0)
@@ -90,13 +71,37 @@ func TestFragCombos(t *testing.T) {
 	}
 }
 
-// newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+// newHolderWithTempPath returns a new instance of Holder.
+func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(path, name)
+	cfg := mustHolderConfig()
+	cfg.StorageConfig.Backend = backend
+	h := NewHolder(path, cfg)
+	PanicOn(h.Open())
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
+	return h
+}
+
+// newIndexWithTempPath returns a new instance of Index.
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
+	if err != nil {
+		panic(err)
+	}
+	cfg := DefaultHolderConfig()
+	cfg.StorageConfig.FsyncEnabled = false
+	cfg.RBFConfig.FsyncEnabled = false
+	h := NewHolder(path, cfg)
+	PanicOn(h.Open())
+	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -105,28 +110,27 @@ func newIndexWithTempPath(name string) *Index {
 
 // Ensure that fragSources creates the correct fragment mapping.
 func TestFragSources(t *testing.T) {
-
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri3, err := NewURIFromAddress("host3")
+	uri3, err := pnet.NewURIFromAddress("host3")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
-	node3 := &Node{ID: "node3", URI: *uri3}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
+	node3 := &topology.Node{ID: "node3", URI: *uri3}
 
 	c1 := newCluster()
 	c1.ReplicaN = 1
@@ -157,27 +161,53 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
+	defer idx.Close()
+
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 101, nil)
+
+	// Obtain transaction.
+	var shard uint64
+	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, 101, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 1
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*2+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 2
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, ShardWidth*3+1, nil)
+	PanicOn(tx.Commit())
+
+	shard = 3
+	tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
+	defer tx.Rollback()
+
+	_, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	PanicOn(tx.Commit())
 
 	tests := []struct {
 		from     *cluster
@@ -194,8 +224,8 @@ func TestFragSources(t *testing.T) {
 				"node0": {},
 				"node1": {},
 				"node2": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -206,11 +236,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
+					{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(1)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -221,11 +251,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
-					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
+					{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
 				},
 				"node1": {
-					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
+					{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(3)},
 				},
 				"node2": {},
 			},
@@ -274,37 +304,37 @@ func TestFragSources(t *testing.T) {
 // Ensure that fragSources creates the correct fragment mapping.
 func TestResizeJob(t *testing.T) {
 
-	uri0, err := NewURIFromAddress("host0")
+	uri0, err := pnet.NewURIFromAddress("host0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri1, err := NewURIFromAddress("host1")
+	uri1, err := pnet.NewURIFromAddress("host1")
 	if err != nil {
 		t.Fatal(err)
 	}
-	uri2, err := NewURIFromAddress("host2")
+	uri2, err := pnet.NewURIFromAddress("host2")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node0 := &Node{ID: "node0", URI: *uri0}
-	node1 := &Node{ID: "node1", URI: *uri1}
-	node2 := &Node{ID: "node2", URI: *uri2}
+	node0 := &topology.Node{ID: "node0", URI: *uri0}
+	node1 := &topology.Node{ID: "node1", URI: *uri1}
+	node2 := &topology.Node{ID: "node2", URI: *uri2}
 
 	tests := []struct {
-		existingNodes []*Node
-		node          *Node
+		existingNodes []*topology.Node
+		node          *topology.Node
 		action        string
 		expectedIDs   map[string]bool
 	}{
 		{
-			existingNodes: []*Node{node0, node1},
+			existingNodes: []*topology.Node{node0, node1},
 			node:          node2,
 			action:        resizeJobActionAdd,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
 		},
 		{
-			existingNodes: []*Node{node0, node1, node2},
+			existingNodes: []*topology.Node{node0, node1, node2},
 			node:          node2,
 			action:        resizeJobActionRemove,
 			expectedIDs:   map[string]bool{node0.ID: false, node1.ID: false},
@@ -325,22 +355,27 @@ func TestResizeJob(t *testing.T) {
 // Ensure the cluster can fairly distribute partitions across the nodes.
 func TestCluster_Owners(t *testing.T) {
 	c := cluster{
-		nodes: []*Node{
+		noder: topology.NewLocalNoder([]*topology.Node{
 			{URI: NewTestURIFromHostPort("serverA", 1000)},
 			{URI: NewTestURIFromHostPort("serverB", 1000)},
 			{URI: NewTestURIFromHostPort("serverC", 1000)},
-		},
+		}),
 		Hasher:   NewTestModHasher(),
 		ReplicaN: 2,
 	}
 
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
 	// Verify nodes are distributed.
-	if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) {
+	if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*topology.Node{cNodes[0], cNodes[1]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 
 	// Verify nodes go around the ring.
-	if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) {
+	if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*topology.Node{cNodes[2], cNodes[0]}) {
 		t.Fatalf("unexpected owners: %s", spew.Sdump(a))
 	}
 }
@@ -351,7 +386,7 @@ func TestCluster_Partition(t *testing.T) {
 		c := newCluster()
 		c.partitionN = partitionN
 
-		partitionID := c.partition(index, shard)
+		partitionID := topology.ShardToShardPartition(index, shard, partitionN)
 		if partitionID < 0 || partitionID >= partitionN {
 			t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
 		}
@@ -381,7 +416,7 @@ func TestHasher(t *testing.T) {
 		{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
 	} {
 		for i, v := range tt.bucket {
-			hasher := &jmphasher{}
+			hasher := &topology.Jmphasher{}
 			if got := hasher.Hash(tt.key, i+1); got != v {
 				t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
 			}
@@ -391,9 +426,14 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
-	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
+	cNodes := c.noder.Nodes()
+
+	// Create a snapshot of the cluster to use for node/partition calculations.
+	snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
+
+	shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
 
 	if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
 		t.Fatalf("unexpected shars for node's index: %v", shards)
@@ -401,20 +441,22 @@ func TestCluster_ContainsShards(t *testing.T) {
 }
 
 func TestCluster_Nodes(t *testing.T) {
-	uri0 := NewTestURIFromHostPort("node0", 0)
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-	uri3 := NewTestURIFromHostPort("node3", 0)
+	const urisCount = 4
+	var uris []pnet.URI
+	arbitraryPorts := []int{17384, 17385, 17386, 17387}
+	for i := 0; i < urisCount; i++ {
+		uris = append(uris, NewTestURIFromHostPort(fmt.Sprintf("node%d", i), uint16(arbitraryPorts[i])))
+	}
 
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	node3 := &Node{ID: "node3", URI: uri3}
+	node0 := &topology.Node{ID: "node0", URI: uris[0]}
+	node1 := &topology.Node{ID: "node1", URI: uris[1]}
+	node2 := &topology.Node{ID: "node2", URI: uris[2]}
+	node3 := &topology.Node{ID: "node3", URI: uris[3]}
 
-	nodes := []*Node{node0, node1, node2}
+	nodes := []*topology.Node{node0, node1, node2}
 
 	t.Run("NodeIDs", func(t *testing.T) {
-		actual := Nodes(nodes).IDs()
+		actual := topology.Nodes(nodes).IDs()
 		expected := []string{node0.ID, node1.ID, node2.ID}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
@@ -422,24 +464,24 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Filter", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
-		expected := []URI{uri0, uri2}
+		actual := topology.Nodes(topology.Nodes(nodes).Filter(nodes[1])).URIs()
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("FilterURI", func(t *testing.T) {
-		actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
-		expected := []URI{uri0, uri2}
+		actual := topology.Nodes(topology.Nodes(nodes).FilterURI(uris[1])).URIs()
+		expected := []pnet.URI{uris[0], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
 	})
 
 	t.Run("Contains", func(t *testing.T) {
-		actualTrue := Nodes(nodes).Contains(node1)
-		actualFalse := Nodes(nodes).Contains(node3)
+		actualTrue := topology.Nodes(nodes).Contains(node1)
+		actualFalse := topology.Nodes(nodes).Contains(node3)
 		if !reflect.DeepEqual(actualTrue, true) {
 			t.Errorf("expected: %v, but got: %v", true, actualTrue)
 		}
@@ -449,9 +491,9 @@ func TestCluster_Nodes(t *testing.T) {
 	})
 
 	t.Run("Clone", func(t *testing.T) {
-		clone := Nodes(nodes).Clone()
-		actual := Nodes(clone).URIs()
-		expected := []URI{uri0, uri1, uri2}
+		clone := topology.Nodes(nodes).Clone()
+		actual := topology.Nodes(clone).URIs()
+		expected := []pnet.URI{uris[0], uris[1], uris[2]}
 		if !reflect.DeepEqual(actual, expected) {
 			t.Errorf("expected: %v, but got: %v", expected, actual)
 		}
@@ -459,9 +501,9 @@ func TestCluster_Nodes(t *testing.T) {
 }
 
 func TestCluster_PreviousNode(t *testing.T) {
-	node0 := &Node{ID: "node0"}
-	node1 := &Node{ID: "node1"}
-	node2 := &Node{ID: "node2"}
+	node0 := &topology.Node{ID: "node0"}
+	node1 := &topology.Node{ID: "node1"}
+	node2 := &topology.Node{ID: "node2"}
 
 	t.Run("OneNode", func(t *testing.T) {
 		c := newCluster()
@@ -512,332 +554,6 @@ func TestCluster_PreviousNode(t *testing.T) {
 	})
 }
 
-// NEXT: move this test to internal and unexport IsCoordinator
-func TestCluster_Coordinator(t *testing.T) {
-	uri1 := NewTestURIFromHostPort("node1", 0)
-	uri2 := NewTestURIFromHostPort("node2", 0)
-
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-
-	c1 := *newCluster()
-	c1.Node = node1
-	c1.Coordinator = node1.ID
-	c2 := *newCluster()
-	c2.Node = node2
-	c2.Coordinator = node1.ID
-
-	t.Run("IsCoordinator", func(t *testing.T) {
-		if !c1.isCoordinator() {
-			t.Errorf("!IsCoordinator error: %v", c1.Node)
-		} else if c2.isCoordinator() {
-			t.Errorf("IsCoordinator error: %v", c2.Node)
-		}
-	})
-}
-
-func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
-
-	uri0 := NewTestURIFromHostPort("host0", 0)
-	uri1 := NewTestURIFromHostPort("host1", 0)
-	uri2 := NewTestURIFromHostPort("host2", 0)
-	invalid := NewTestURIFromHostPort("invalid", 0)
-
-	node0 := &Node{ID: "node0", URI: uri0}
-	node1 := &Node{ID: "node1", URI: uri1}
-	node2 := &Node{ID: "node2", URI: uri2}
-	nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
-
-	t.Run("AddNode", func(t *testing.T) {
-		err := c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// add the same host.
-		err = c1.addNode(node1)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = c1.addNode(node2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		actual := c1.nodeIDs()
-		expected := []string{node0.ID, node1.ID, node2.ID}
-
-		if !reflect.DeepEqual(actual, expected) {
-			t.Errorf("expected: %v, but got: %v", expected, actual)
-		}
-	})
-
-	t.Run("ContainsID", func(t *testing.T) {
-		if !c1.Topology.ContainsID(node1.ID) {
-			t.Errorf("!ContainsHost error: %v", node1.ID)
-		} else if c1.Topology.ContainsID(nodeinvalid.ID) {
-			t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
-		}
-	})
-}
-
-// Ensure that general cluster functionality works as expected.
-func TestCluster_ResizeStates(t *testing.T) {
-
-	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		node := tc.Clusters[0]
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{node.Node.ID},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Ensure that node comes up in state NORMAL.
-		if node.State() != ClusterStateNormal {
-			t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"some-other-host"},
-		}
-		if err := tc.WriteTopology(node.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		expected := "coordinator node0 is not in topology: [some-other-host]"
-		err := tc.Open()
-		if err == nil || errors.Cause(err).Error() != expected {
-			t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-
-		node0 := tc.Clusters[0]
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// write topology to data file
-		top := &Topology{
-			nodeIDs: []string{"node0", "node2"},
-		}
-		if err := tc.WriteTopology(node0.Path, top); err != nil {
-			t.Fatalf("writing topology: %v", err)
-		}
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatalf("opening cluster: %v", err)
-		}
-
-		// Ensure that node is in state STARTING before the other node joins.
-		if node0.State() != ClusterStateStarting {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
-		}
-
-		// Expect an error by adding a node not in the topology.
-		expectedError := "host is not in topology: node1"
-		if err := tc.addNode(); err == nil || err.Error() != expectedError {
-			t.Errorf("did not receive expected error: %s", expectedError)
-		}
-
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node2 := tc.Clusters[2]
-
-		// Ensure that node comes up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node2.State() != ClusterStateNormal {
-			t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State())
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-
-	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node0 := tc.Clusters[0]
-
-		// Open TestCluster.
-		if err := tc.Open(); err != nil {
-			t.Fatal(err)
-		}
-
-		// Add Bit Data to node0.
-		if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
-			t.Fatalf("creating field: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-		if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil {
-			t.Fatalf("setting bit: %v", err)
-		}
-
-		// Before starting the resize, get the CheckSum to use for
-		// comparison later.
-		node0Field := node0.holder.Field("i", "f")
-		node0View := node0Field.view("standard")
-		node0Fragment := node0View.Fragment(1)
-		node0Checksum := node0Fragment.Checksum()
-
-		// addNode needs to block until the resize process has completed.
-		if err := tc.addNode(); err != nil {
-			t.Fatalf("adding node: %v", err)
-		}
-		node1 := tc.Clusters[1]
-
-		// Ensure that nodes come up in state NORMAL.
-		if node0.State() != ClusterStateNormal {
-			t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
-		} else if node1.State() != ClusterStateNormal {
-			t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
-		}
-
-		expectedTop := &Topology{
-			nodeIDs: []string{node0.Node.ID, node1.Node.ID},
-		}
-
-		// Verify topology file.
-		if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
-		} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
-			t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
-		}
-
-		// Bits
-		// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
-		node1Field := node1.holder.Field("i", "f")
-		node1View := node1Field.view("standard")
-		node1Fragment := node1View.Fragment(1)
-
-		// Ensure checksums are the same.
-		if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
-			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
-		}
-
-		// Close TestCluster.
-		if err := tc.Close(); err != nil {
-			t.Fatal(err)
-		}
-	})
-}
-
 func TestAE(t *testing.T) {
 	t.Run("AbortDoesn'tBlockUninitialized", func(t *testing.T) {
 		c := newCluster()
@@ -846,6 +562,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -857,11 +574,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@@ -893,97 +612,15 @@ func TestAE(t *testing.T) {
 			t.Fatalf("abort should not have blocked this long")
 		}
 	})
-
 }
 
-// Ensures that coordinator can be changed.
-func TestCluster_UpdateCoordinator(t *testing.T) {
-	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
-
-		oldNode := c.nodes[0]
-		newNode := c.nodes[1]
-
-		// Update coordinator to the same value.
-		if c.updateCoordinator(oldNode) {
-			t.Errorf("did not expect coordinator to change")
-		} else if c.Coordinator != oldNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
-		}
-
-		// Update coordinator to a new value.
-		if !c.updateCoordinator(newNode) {
-			t.Errorf("expected coordinator to change")
-		} else if c.Coordinator != newNode.ID {
-			t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
-		}
-	})
-}
-
-func TestCluster_confirmNodeDownUp(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-	if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be up")
-	}
-
-}
-func TestCluster_confirmNodeDownTimeout(t *testing.T) {
-	r := mux.NewRouter()
-	r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
-		fmt.Fprintln(w, "ignored")
-	}))
-	server := httptest.NewServer(r)
-	// Close the server when test finishes
-	defer server.Close()
-	u, err := url.Parse(server.URL)
-	if err != nil {
-		t.Error("bad test setup")
-	}
-	uri := URI{}
-	host, port, _ := net.SplitHostPort(u.Host)
-	uri.Scheme = u.Scheme
-	uri.Host = host
-	iport, err := strconv.ParseUint(port, 0, 16)
-	if err != nil {
-		t.Error(err)
-	}
-	uri.Port = uint16(iport)
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
-}
-
-func TestCluster_confirmNodeDownDown(t *testing.T) {
-	uri := URI{}
-	uri.Scheme = "http"
-	uri.Host = "DoesntMatter"
-	uri.Port = 6666
-
-	if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
-		t.Errorf("expected node to be down")
-	}
-
+func TestTranslateIndexKey(t *testing.T) {
+	c := newCluster()
+	node0 := &topology.Node{ID: "node0"}
+	c.addNodeBasicSorted(node0)
+	c.holder = newHolderWithTempPath(t, "rbf")
+	_, e := c.translateIndexKey(context.Background(), "i", "a", false)
+	if e == nil {
+		t.Fatal("expecting error")
+	}
 }

cmd.go

@@ -1,22 +1,10 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 
 import (
 	"io"
-	"log"
+
+	"github.com/molecula/featurebase/v3/logger"
 )
 
 // CmdIO holds standard unix inputs and outputs.
@@ -24,7 +12,7 @@ type CmdIO struct {
 	Stdin  io.Reader
 	Stdout io.Writer
 	Stderr io.Writer
-	logger *log.Logger
+	logger logger.Logger
 }
 
 // NewCmdIO returns a new instance of CmdIO with inputs and outputs set to the
@@ -34,10 +22,10 @@ func NewCmdIO(stdin io.Reader, stdout, stderr io.Writer) *CmdIO {
 		Stdin:  stdin,
 		Stdout: stdout,
 		Stderr: stderr,
-		logger: log.New(stderr, "", log.LstdFlags),
+		logger: logger.NewStandardLogger(stderr),
 	}
 }
 
-func (c *CmdIO) Logger() *log.Logger {
+func (c *CmdIO) Logger() logger.Logger {
 	return c.logger
 }

cmd/backup.go

@@ -0,0 +1,36 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/molecula/featurebase/v3/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newBackupCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
+	cmd := ctl.NewBackupCommand(stdin, stdout, stderr)
+	ccmd := &cobra.Command{
+		Use:   "backup",
+		Short: "Back up FeatureBase server",
+		Long: `
+Backs up a FeatureBase server to a local, tar-formatted snapshot file.
+`,
+		RunE: func(c *cobra.Command, args []string) error {
+			return cmd.Run(context.Background())
+		},
+	}
+
+	flags := ccmd.Flags()
+	flags.StringVarP(&cmd.OutputDir, "output", "o", "", "Output directory to write to.")
+	flags.BoolVar(&cmd.NoSync, "no-sync", false, "Disable file sync")
+	flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "Number of concurrent backup goroutines.")
+	flags.StringVar(&cmd.Host, "host", "localhost:10101", "The address (host:port) of FeatureBase (HTTP).")
+	flags.StringVar(&cmd.Index, "index", "", "Index to backup, default backs up all indexes. ")
+	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
+	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
+	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
+	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
+	return ccmd
+}

cmd/badloader/badloader.go

@@ -0,0 +1,137 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+
+	"fmt"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/encoding/proto"
+	pnet "github.com/molecula/featurebase/v3/net"
+	"github.com/molecula/featurebase/v3/vprint"
+
+	"os"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *pilosa.InternalClient) error {
+	t0 := time.Now()
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				vprint.PanicOn("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				vprint.PanicOn(err)
+			}
+			return nil
+		}
+		n++
+		if n%500 == 0 {
+			vprint.VV("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		//vv("parts = '%#v'", parts)
+		index := parts[1]
+		field := parts[2]
+		view := parts[4]
+		shard, err := strconv.ParseUint(parts[6], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				vprint.PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			vprint.PanicOn(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+// badloader reproduce a union in place issue for us. slurp is
+// the new "good" loader, and should always be preferred now
+// when not trying to repro that bug. pulled from 85fa67e8
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
+	vprint.PanicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	vprint.PanicOn(UploadTar(tarSrcPath, c))
+	vprint.VV("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pnet.URI
+
+func init() {
+	var err error
+	globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
+	vprint.PanicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
+	return globURI
+}

cmd/check.go

@@ -1,46 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"io"
-
-	"github.com/spf13/cobra"
-
-	"github.com/pilosa/pilosa/v2/ctl"
-)
-
-var checker *ctl.CheckCommand
-
-func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
-	checker = ctl.NewCheckCommand(stdin, stdout, stderr)
-	checkCmd := &cobra.Command{
-		Use:   "check <path> [path2]...",
-		Short: "Do a consistency check on a pilosa data file.",
-		Long: `
-Performs a consistency check on data files.
-`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 {
-				return fmt.Errorf("path required")
-			}
-			checker.Paths = args
-			return checker.Run(context.Background())
-		},
-	}
-	return checkCmd
-}

cmd/check_test.go

@@ -1,36 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd_test
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestCheckHelp(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "check", "--help")
-	if !strings.Contains(output, "Usage:") ||
-		!strings.Contains(output, "Flags:") ||
-		!strings.Contains(output, "pilosa check") || err != nil {
-		t.Fatalf("Command 'check --help' not working, err: '%v', output: '%s'", err, output)
-	}
-}
-
-func TestCheckNoPath(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "check")
-	if !strings.Contains(err.Error(), "path required") {
-		t.Fatalf("Command 'check' without args should error but: err: '%v', output: '%v'", err, output)
-	}
-}

cmd/chksum.go

@@ -0,0 +1,30 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/molecula/featurebase/v3/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newChkSumCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
+	cmd := ctl.NewChkSumCommand(stdin, stdout, stderr)
+	ccmd := &cobra.Command{
+		Use:   "chksum",
+		Short: "Digital signature of FeatureBase data",
+		Long: `
+			Generates a digital signature of all the data associated with a provided FeatureBase server
+			WARNING: could be slow if high cardinality fields exist
+`,
+		RunE: func(c *cobra.Command, args []string) error {
+			return cmd.Run(context.Background())
+		},
+	}
+
+	flags := ccmd.Flags()
+	flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
+	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
+	return ccmd
+}

cmd/config.go

@@ -1,17 +1,4 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd
 
 import (
@@ -20,8 +7,8 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/pilosa/pilosa/v2/ctl"
-	"github.com/pilosa/pilosa/v2/server"
+	"github.com/molecula/featurebase/v3/ctl"
+	"github.com/molecula/featurebase/v3/server"
 )
 
 var conf *ctl.ConfigCommand

cmd/doc.go

@@ -1,17 +1,4 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 /*
 Package cmd contains all the pilosa subcommand definitions (1 per file).
 

cmd/export.go

@@ -1,17 +1,4 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd
 
 import (
@@ -20,7 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/molecula/featurebase/v3/ctl"
 )
 
 var Exporter *ctl.ExportCommand
@@ -29,7 +16,7 @@ func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
 	Exporter = ctl.NewExportCommand(stdin, stdout, stderr)
 	exportCmd := &cobra.Command{
 		Use:   "export",
-		Short: "Export data from pilosa.",
+		Short: "Export data from FeatureBase.",
 		Long: `
 Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then
 the output is written to STDOUT.
@@ -46,11 +33,11 @@ The file does not contain any headers.
 	}
 	flags := exportCmd.Flags()
 
-	flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
-	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
+	flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of FeatureBase.")
+	flags.StringVarP(&Exporter.Index, "index", "i", "", "FeatureBase index to export")
 	flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
-	ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
 
 	return exportCmd
 }

cmd/export_test.go

@@ -1,31 +1,18 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd_test
 
 import (
 	"strings"
 	"testing"
 
-	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/molecula/featurebase/v3/cmd"
 )
 
 func TestExportHelp(t *testing.T) {
 	output, err := ExecNewRootCommand(t, "export", "--help")
 	if !strings.Contains(output, "Usage:") ||
 		!strings.Contains(output, "Flags:") ||
-		!strings.Contains(output, "pilosa export") || err != nil {
+		!strings.Contains(output, "featurebase export") || err != nil {
 		t.Fatalf("Command 'export --help' not working, err: '%v', output: '%s'", err, output)
 	}
 }

cmd/featurebase-parse-sql/main.go

@@ -0,0 +1,47 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/molecula/featurebase/v3/sql2"
+)
+
+func main() {
+	if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("featurebase-parse-sql", flag.ContinueOnError)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	q := fs.Arg(0)
+	if q == "" {
+		return fmt.Errorf("query required")
+	}
+
+	stmt, err := sql2.NewParser(strings.NewReader(q)).ParseStatement()
+	if err != nil {
+		return err
+	}
+
+	buf, err := json.MarshalIndent(stmt, "", "    ")
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(buf))
+
+	return nil
+}

cmd/featurebase/main.go

@@ -0,0 +1,20 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+/*
+This is the entrypoint for the Pilosa binary.
+*/
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/molecula/featurebase/v3/cmd"
+)
+
+func main() {
+	rootCmd := cmd.NewRootCommand(os.Stdin, os.Stdout, os.Stderr)
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

cmd/featurebase/main_test.go

@@ -0,0 +1,13 @@
+//go:build testrunmain
+// +build testrunmain
+
+package main
+
+import (
+	"testing"
+)
+
+// Wrapper test for main function used to get code coverage for end2end tests
+func TestRunMain(t *testing.T) {
+	main()
+}

cmd/generate_config.go

@@ -1,17 +1,4 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd
 
 import (
@@ -20,7 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/pilosa/pilosa/v2/ctl"
+	"github.com/molecula/featurebase/v3/ctl"
 )
 
 var generateConf *ctl.GenerateConfigCommand

cmd/import.go

@@ -1,36 +1,23 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd
 
 import (
 	"context"
 	"io"
 
-	"github.com/pilosa/pilosa/v2"
-	"github.com/pilosa/pilosa/v2/ctl"
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/ctl"
 	"github.com/spf13/cobra"
 )
 
 var Importer *ctl.ImportCommand
 
-// newImportCommand runs the Pilosa import subcommand for ingesting bulk data.
+// newImportCommand runs the FeatureBase import subcommand for ingesting bulk data.
 func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
 	Importer = ctl.NewImportCommand(stdin, stdout, stderr)
 	importCmd := &cobra.Command{
 		Use:   "import",
-		Short: "Bulk load data into pilosa.",
+		Short: "Bulk load data into FeatureBase.",
 		Long: `Bulk imports one or more CSV files to a host's index and field. The data
 of the CSV file are grouped by shard for the most efficient import.
 
@@ -48,14 +35,14 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	}
 
 	flags := importCmd.Flags()
-	flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
-	flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
+	flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of FeatureBase.")
+	flags.StringVarP(&Importer.Index, "index", "i", "", "FeatureBase index to import into.")
 	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
 	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
 	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
-	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
-	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
-	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
+	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
+	flags.Int64Var(&Importer.FieldOptions.Min.Value, "field-min", 0, "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
+	flags.Int64Var(&Importer.FieldOptions.Max.Value, "field-max", 0, "Specify the maximum for an int field on creation")
 	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
 	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
 	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
@@ -63,7 +50,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
-	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	flags.StringVar(&Importer.AuthToken, "auth-token", "", "Authentication token")
 
 	return importCmd
 }

cmd/import_test.go

@@ -1,33 +1,21 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2021 Molecula Corp. All rights reserved.
 package cmd_test
 
 import (
 	"strings"
 	"testing"
 
-	"github.com/pilosa/pilosa/v2"
+	"github.com/molecula/featurebase/v3"
 
-	"github.com/pilosa/pilosa/v2/cmd"
+	"github.com/molecula/featurebase/v3/cmd"
+	"github.com/molecula/featurebase/v3/pql"
 )
 
 func TestImportHelp(t *testing.T) {
 	output, err := ExecNewRootCommand(t, "import", "--help")
 	if !strings.Contains(output, "Usage:") ||
 		!strings.Contains(output, "Flags:") ||
-		!strings.Contains(output, "pilosa import") || err != nil {
+		!strings.Contains(output, "featurebase import") || err != nil {
 		t.Fatalf("Command 'import --help' not working, err: '%v', output: '%s'", err, output)
 	}
 }
@@ -58,8 +46,8 @@ field = "f1"
 				v.Check(cmd.Importer.Field, "f1")
 				v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
 					Keys:      true,
-					Max:       100,
-					Min:       -10,
+					Max:       pql.NewDecimal(100, 0),
+					Min:       pql.NewDecimal(-10, 0),
 					CacheType: pilosa.CacheTypeRanked,
 					CacheSize: 50000,
 				})

cmd/inspect.go

@@ -1,49 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"io"
-
-	"github.com/spf13/cobra"
-
-	"github.com/pilosa/pilosa/v2/ctl"
-)
-
-var inspector *ctl.InspectCommand
-
-func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
-	inspector = ctl.NewInspectCommand(stdin, stdout, stderr)
-
-	inspectCmd := &cobra.Command{
-		Use:   "inspect",
-		Short: "Get stats on a pilosa data file.",
-		Long: `
-Inspects a data file and provides stats.
-`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 {
-				return fmt.Errorf("path required")
-			} else if len(args) > 1 {
-				return fmt.Errorf("only one path allowed")
-			}
-			inspector.Path = args[0]
-			return inspector.Run(context.Background())
-		},
-	}
-	return inspectCmd
-}

cmd/inspect_test.go

@@ -1,42 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd_test
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestInspectHelp(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "inspect", "--help")
-	if !strings.Contains(output, "Usage:") ||
-		!strings.Contains(output, "pilosa inspect") || err != nil {
-		t.Fatalf("Command 'inspect --help' not working, err: '%v', output: '%s'", err, output)
-	}
-}
-
-func TestInspectNoPath(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "inspect")
-	if !strings.Contains(err.Error(), "path required") {
-		t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
-	}
-}
-
-func TestInspectMultiPath(t *testing.T) {
-	output, err := ExecNewRootCommand(t, "inspect", "one", "two")
-	if !strings.Contains(err.Error(), "only one path") {
-		t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
-	}
-}

cmd/keygen.go

@@ -0,0 +1,28 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/molecula/featurebase/v3/ctl"
+	"github.com/spf13/cobra"
+)
+
+func newKeygenCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
+	cmd := ctl.NewKeygenCommand(stdin, stdout, stderr)
+	ccmd := &cobra.Command{
+		Use:   "keygen",
+		Short: "Generate secret key for authentication.",
+		Long: `
+Generate secret key to configure FeatureBase for Authentication.
+`,
+		RunE: func(c *cobra.Command, args []string) error {
+			return cmd.Run(context.Background())
+		},
+	}
+
+	flags := ccmd.Flags()
+	flags.IntVarP(&cmd.KeyLength, "length", "l", 32, "length of the key to produce")
+	return ccmd
+}

cmd/pilosa-bench/main.go

@@ -0,0 +1,345 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"context"
+	"expvar"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/encoding/proto"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	requestCountVar          = expvar.NewInt("request_count")
+	requestCurrentLatencyVar = expvar.NewFloat("request_current_latency") // seconds
+	requestAvgLatencyVar     = expvar.NewFloat("request_avg_latency")     // seconds
+	requestTotalLatencyVar   = expvar.NewFloat("request_total_latency")   // seconds
+	requestPerSecVar         = expvar.NewFloat("request_per_sec")
+)
+
+func main() {
+	if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
+		os.Exit(1)
+	} else if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) (err error) {
+	fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
+	hostport := fs.String("hostport", "localhost:10101", "")
+	typ := fs.String("type", "row", "query type (row)")
+	n := fs.Int("n", 1000, "number of queries")
+	rate := fs.Int("rate", 1, "number of queries per second")
+	verbose := fs.Bool("v", false, "verbose logging")
+	from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
+	to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse from/to time.
+	var opt queryOptions
+	if *from != "" {
+		if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
+			return fmt.Errorf("cannot parse -from time")
+		}
+	}
+	if *to != "" {
+		if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
+			return fmt.Errorf("cannot parse -to time")
+		}
+	}
+
+	if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
+		return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
+	}
+
+	// Clear time prefix on log.
+	log.SetFlags(0)
+	if !*verbose {
+		log.SetOutput(ioutil.Discard)
+	}
+
+	// Setup PRNG to have consistent values for the same set of data.
+	rand.Seed(0)
+
+	// Setup connection to pilosa.
+	client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{}))
+	if err != nil {
+		return err
+	}
+
+	// Set up HTTP endpoint to provide /debug endpoints.
+	fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
+	go func() { _ = http.ListenAndServe(":7070", nil) }()
+
+	// Run separate goroutine to calculate the current req/sec & latency.
+	go monitor()
+
+	// Load all id/keys for each field.
+	log.Printf("loading field identifiers")
+	fieldIDMap, err := loadFields(ctx, client)
+	if err != nil {
+		return fmt.Errorf("cannot load field identifiers: %w", err)
+	} else if len(fieldIDMap) == 0 {
+		return fmt.Errorf("no field identifiers available, please verify data exists")
+	}
+
+	// Generate list of sorted keys.
+	fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
+	for k, f := range fieldIDMap {
+		switch *typ {
+		case "row-bsi":
+			if f.info.Options.Type != "int" {
+				continue
+			}
+		case "row-range", "topk":
+			if f.info.Options.Type != "time" {
+				continue
+			}
+		default:
+			if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
+				continue
+			}
+		}
+
+		fieldKeys = append(fieldKeys, k)
+	}
+	sort.Slice(fieldKeys, func(i, j int) bool {
+		return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
+	})
+
+	// Ensure we have appropriate fields for our query type.
+	if len(fieldKeys) == 0 {
+		return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
+	}
+
+	log.Printf("issuing %d queries at %d query/sec", *n, *rate)
+
+	// Repeatedly issue queries based on available row data.
+	ticker := time.NewTicker(time.Second / time.Duration(*rate))
+	var g errgroup.Group
+	for i := 0; i < *n; i++ {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		key := fieldKeys[rand.Intn(len(fieldKeys))]
+		q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
+		if err != nil {
+			return fmt.Errorf("cannot generate query: %w", err)
+		}
+
+		log.Printf("[query] %s", q)
+
+		g.Go(func() error {
+			t := time.Now()
+			_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
+			if err != nil {
+				return err
+			}
+			elapsed := time.Since(t).Seconds()
+			requestCountVar.Add(1)
+			requestTotalLatencyVar.Add(elapsed)
+			requestAvgLatencyVar.Set(requestTotalLatencyVar.Value() / float64(requestCountVar.Value()))
+			return nil
+		})
+	}
+
+	return g.Wait()
+}
+
+// monitor runs in a separate goroutine and updates metrics.
+func monitor() {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	var lastTime time.Time
+	var lastN int64
+	var lastLatency float64
+	for range ticker.C {
+		now, n := time.Now(), requestCountVar.Value()
+		latency := requestTotalLatencyVar.Value()
+
+		if !lastTime.IsZero() {
+			elapsed := lastTime.Sub(now).Seconds()
+			if n > 0 {
+				requestCurrentLatencyVar.Set((lastLatency - latency) / float64(n))
+			}
+			requestPerSecVar.Set(float64(lastN-n) / elapsed)
+		}
+		lastTime, lastN, lastLatency = now, n, latency
+	}
+}
+
+func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
+	switch typ {
+	case "row":
+		return generateRowQuery(index, field, identifiers), nil
+	case "row-bsi":
+		return generateRowBSIQuery(index, field), nil
+	case "row-range":
+		return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
+	case "count":
+		return generateCountQuery(index, field, identifiers), nil
+	case "intersect":
+		return generateIntersectQuery(index, field, identifiers), nil
+	case "union":
+		return generateUnionQuery(index, field, identifiers), nil
+	case "difference":
+		return generateDifferenceQuery(index, field, identifiers), nil
+	case "xor":
+		return generateXorQuery(index, field, identifiers), nil
+	case "groupby":
+		return generateGroupByQuery(index, field), nil
+	case "topk":
+		return generateTopKQuery(index, field, opt.from, opt.to), nil
+	default:
+		return "", fmt.Errorf("invalid query type: %q", typ)
+	}
+}
+
+func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
+	}
+	return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
+}
+
+func generateRowBSIQuery(index, field string) string {
+	return fmt.Sprintf("Row(%s > 0)", field)
+}
+
+func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
+	if len(identifiers.Rows) > 0 {
+		return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+	}
+	return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	a := make([]string, rand.Intn(9)+1)
+	for i := range a {
+		a[i] = generateRowQuery(index, field, identifiers)
+	}
+	return strings.Join(a, ", ")
+}
+
+func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
+}
+
+func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
+	return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
+}
+
+func generateGroupByQuery(index, field string) string {
+	return fmt.Sprintf("GroupBy(Rows(%s))", field)
+}
+
+func generateTopKQuery(index, field string, from, to time.Time) string {
+	return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
+}
+
+// loadFields returns a mapping of index/field names to field info & identifiers.
+func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) {
+	indexes, err := client.Schema(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[fieldKey]*fieldInfo)
+	for _, ii := range indexes {
+		for _, f := range ii.Fields {
+			log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
+
+			switch f.Options.Type {
+			case "set", "mutex", "time":
+				identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
+				if err != nil {
+					return nil, fmt.Errorf("fetch fields: %w", err)
+				} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
+					m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
+				}
+
+			case "int":
+				m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
+			}
+		}
+	}
+	return m, nil
+}
+
+// fetchFieldIDs returns a list of field IDs or keys.
+func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
+	resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
+	if err != nil {
+		return nil, err
+	}
+
+	switch result := resp.Results[0].(type) {
+	case *pilosa.RowIdentifiers:
+		return result, nil
+	case pilosa.RowIdentifiers:
+		return &result, nil
+	default:
+		return nil, fmt.Errorf("unexpected result type: %T", result)
+	}
+}
+
+func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
+	return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
+}
+
+func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
+	return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
+}
+
+type fieldKey struct {
+	index string
+	field string
+}
+
+type fieldInfo struct {
+	info        *pilosa.FieldInfo
+	identifiers *pilosa.RowIdentifiers
+}
+
+func compareFieldKeys(x, y fieldKey) int {
+	if cmp := strings.Compare(x.index, y.index); cmp != 0 {
+		return cmp
+	}
+	return strings.Compare(x.field, y.field)
+}
+
+type queryOptions struct {
+	from, to time.Time
+}

cmd/pilosa/main.go

@@ -1,33 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/*
-This is the entrypoint for the Pilosa binary.
-*/
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pilosa/pilosa/v2/cmd"
-)
-
-func main() {
-	rootCmd := cmd.NewRootCommand(os.Stdin, os.Stdout, os.Stderr)
-	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}

cmd/random-query/main.go

@@ -0,0 +1,596 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	nethttp "net/http"
+
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/client"
+	fb_proto "github.com/molecula/featurebase/v3/encoding/proto"
+	"github.com/molecula/featurebase/v3/pb"
+	"github.com/molecula/featurebase/v3/pql"
+	"github.com/molecula/featurebase/v3/vprint"
+	"github.com/pkg/errors"
+	vegeta "github.com/tsenart/vegeta/v12/lib"
+)
+
+// RandomQueryConfig
+type RandomQueryConfig struct {
+	HostPort     string
+	TreeDepth    int
+	QueryCount   int
+	Verbose      bool
+	GenerateOnly bool
+	NumRuns      int
+	TimeFromArg  string
+	TimeToArg    string
+	TimeFrom     time.Time
+	TimeTo       time.Time
+	TimeRange    int64
+	Index        string
+	QPM          int
+	Seed         int
+	SrcFile      string
+	Duration     time.Duration
+	Target       vegeta.Target
+
+	IndexMap map[string]*Features
+
+	API  *pilosa.API
+	Info []*pilosa.IndexInfo
+
+	BitmapFunc []string
+
+	Rnd *rand.Rand
+}
+
+type API interface {
+
+	// InternalClient
+	Schema(ctx context.Context) ([]*pilosa.IndexInfo, error)
+	Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error)
+
+	// API for contrast; just a little different:
+	//Schema(ctx context.Context) []*IndexInfo
+	//Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error)
+}
+
+// have to wrap because the ugly little differences between InternalClient and API
+type wrapper struct {
+	api *pilosa.API
+}
+
+func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
+	return w.api.Schema(ctx, false)
+}
+
+func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
+	r, err := w.api.Query(ctx, queryRequest)
+	return &r, err
+}
+
+func wrapApiToInternalClient(api *pilosa.API) *wrapper {
+	return &wrapper{api: api}
+}
+
+// These times are copied from the "kitchen sink" data generator to serve as defaults.
+var defaultEndTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
+var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour)
+
+// call DefineFlags before myflags.Parse()
+func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.")
+	fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.")
+	fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate")
+	fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ")
+	fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time")
+	fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s")
+	fs.StringVar(&cfg.Index, "index", "i", "index to run queries against")
+	fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10")
+	fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated")
+	fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package")
+	fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+	fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)")
+	fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating")
+}
+
+// call c.ValidateConfig() after myflags.Parse()
+func (c *RandomQueryConfig) ValidateConfig() error {
+	if c.TreeDepth < 1 {
+		return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth)
+	}
+	if c.QueryCount < 0 {
+		return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount)
+	}
+	var err error
+	c.TimeFrom, err = time.Parse(time.RFC3339, c.TimeFromArg)
+	if err != nil {
+		return fmt.Errorf("-time.from value couldn't be parsed: %w", err)
+	}
+	c.TimeTo, err = time.Parse(time.RFC3339, c.TimeToArg)
+	if err != nil {
+		return fmt.Errorf("-time.to value couldn't be parsed: %w", err)
+	}
+	c.TimeRange = int64(c.TimeTo.Sub(c.TimeFrom).Hours())
+	if c.TimeRange < 1 {
+		return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)",
+			c.TimeToArg, c.TimeFromArg)
+	}
+	if c.QPM <= 0 {
+		return fmt.Errorf("-qpm must be positive")
+	}
+	return nil
+}
+
+var ProgramName = "random-query"
+
+func main() {
+
+	myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
+	cfg := NewRandomQueryConfig()
+	cfg.DefineFlags(myflags)
+
+	err := myflags.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
+		os.Exit(1)
+	}
+	err = cfg.ValidateConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
+		os.Exit(1)
+	}
+	err = cfg.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *RandomQueryConfig) Run() (err error) {
+	remoteClient := nethttp.DefaultClient
+	cli, err := pilosa.NewInternalClient(cfg.HostPort, remoteClient, pilosa.WithSerializer(fb_proto.Serializer{}))
+	if err != nil {
+		return err
+	}
+	err = cfg.Setup(cli)
+	if err != nil {
+		return err
+	}
+	rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute}
+	duration := cfg.Duration
+	targeter := vegeta.NewStaticTargeter(cfg.Target)
+	attacker := vegeta.NewAttacker()
+
+	for i := 0; i < cfg.NumRuns; i++ {
+		vprint.VV("================")
+		var metrics vegeta.Metrics
+		for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") {
+			metrics.Add(res)
+		}
+		metrics.Close()
+		rpt := vegeta.NewTextReporter(&metrics)
+		rpt(os.Stdout)
+	}
+
+	return nil
+}
+
+type Features struct {
+	Slc           []IndexFieldRow
+	Ranges        []IndexFieldRange
+	Distinctables []IndexFieldRange
+	Stores        []IndexFieldRow
+	SlcWeight     int
+	RangeWeight   int
+}
+
+// Pick either a feature entry or a random query on a range, weighted
+// by number of features and approximate weight of ranges
+func (f *Features) RandomQuery(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Intn(f.SlcWeight + f.RangeWeight)
+	if r < f.SlcWeight {
+		return f.Slc[r].Query(cfg)
+	}
+	r = cfg.Rnd.Intn(len(f.Ranges))
+	return f.Ranges[r].Query(cfg)
+}
+
+func NewRandomQueryConfig() *RandomQueryConfig {
+	return &RandomQueryConfig{
+		IndexMap: make(map[string]*Features),
+	}
+}
+
+type IndexFieldRow struct {
+	Index    string
+	Field    string
+	RowID    uint64
+	RowKey   string
+	IsRowKey bool
+	HasTime  bool
+	IsInt    bool
+}
+
+func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
+	fromTo := ""
+	// 5% of queries on a time field will use the standard view
+	// anyway.
+	if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
+		startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
+		endHours := cfg.Rnd.Int63n(cfg.TimeRange-startHours) + 1 + startHours
+		startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
+		endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
+		fromTo = fmt.Sprintf(", from=%s, to=%s",
+			startTime.Format(pilosaTimeFmt),
+			endTime.Format(pilosaTimeFmt))
+	}
+	if fea.IsRowKey {
+		return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)}
+	}
+	return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)}
+}
+
+type IndexFieldRange struct {
+	Index           string
+	Field           string
+	Min, Max, Scale int64
+	ScaleDiv        float64
+	Range           uint64
+}
+
+// We want to pick one of (1) a single-operation filter, (2) a
+// between-filter of some kind.
+// So, that's one of <=, >=, ==, !=, >, <, or
+// one of [<, <], [<, <=], [<=, <=], [<=, <].
+var binaryOps = []string{
+	"<=", ">=", "==", "!=", "<", ">",
+}
+
+func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
+	r := cfg.Rnd.Int63n(10)
+	// this is unevenly weighted, but there's no Uint64N, and
+	// Int63n can't represent the whole range.
+	v1 := cfg.Rnd.Uint64() % i.Range
+	v2 := cfg.Rnd.Uint64() % i.Range
+	if v1 > v2 {
+		v1, v2 = v2, v1
+	}
+	v1 = v1 + uint64(i.Min)
+	v2 = v2 + uint64(i.Min)
+	var v1s, v2s string
+	if i.Scale != 0 {
+		v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1))/i.ScaleDiv)
+		v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2))/i.ScaleDiv)
+	} else {
+		v1s = strconv.FormatInt(int64(v1), 10)
+		v2s = strconv.FormatInt(int64(v2), 10)
+	}
+	if r < 4 {
+		lte := "<="
+		op1 := lte[:1+(r&1)]
+		op2 := lte[:1+((r>>1)&1)]
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s %s %s)",
+			v1s, op1, i.Field, op2, v2s)}
+	} else {
+		if cfg.Rnd.Int63n(2) == 1 {
+			v1s = v2s
+		}
+		return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r-4], v1s)}
+	}
+}
+
+// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects
+// and spits back a PQL query
+//
+func (cfg *RandomQueryConfig) Setup(api API) (err error) {
+	if cfg.SrcFile != "" {
+		return cfg.buildPayload()
+	}
+	ctx := context.Background()
+	cfg.Info, err = api.Schema(ctx)
+	if err != nil {
+		return err
+	}
+	foundIntField := false
+	any := false
+	for i, ii := range cfg.Info {
+		if ii.Name == cfg.Index {
+			any = true
+			_ = i
+			for k, fld := range ii.Fields {
+				_ = k
+				switch fld.Options.Type {
+				case "set", "mutex", "time":
+					pql := fmt.Sprintf("Rows(%v)", fld.Name)
+
+					res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
+					vprint.PanicOn(err)
+					switch x := res.Results[0].(type) {
+					case *pilosa.RowIdentifiers:
+						cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set")
+					case pilosa.RowIdentifiers:
+						cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set")
+					}
+				case "int":
+					foundIntField = true
+					fallthrough
+				case "decimal":
+					cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal")
+				default:
+					vprint.AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type)
+				}
+			}
+		}
+	}
+	if !any {
+		return errors.New(fmt.Sprintf("index %v not found", cfg.Index))
+	}
+	cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"}
+	if foundIntField {
+		cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct")
+	}
+	cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed)))
+	return cfg.buildPayload()
+}
+
+func (cfg *RandomQueryConfig) buildPayload() error {
+	var request strings.Builder
+	if cfg.SrcFile != "" {
+		b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name
+		if err != nil {
+			return err
+		}
+		request.WriteString(string(b))
+	} else {
+
+		for i := 0; i < cfg.QueryCount; i++ {
+			pql, err := cfg.GenQuery(cfg.Index)
+			if err != nil {
+				return err
+			}
+			request.WriteString(pql)
+		}
+	}
+	//TODO (twg) tls support
+	path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index)
+	header := nethttp.Header{
+		"Content-Type": []string{"application/x-protobuf"},
+		"Accept":       []string{"application/x-protobuf"},
+		"PQL-Version":  []string{client.PQLVersion},
+	}
+
+	req := &pb.QueryRequest{
+		Query: request.String(),
+	}
+	vprint.VV("%v", request.String())
+	if cfg.GenerateOnly {
+		// just output the PQL and exit
+		os.Exit(0)
+	}
+	payload, err := proto.Marshal(req)
+	if err != nil {
+		return errors.Wrap(err, "marshaling request to protobuf")
+	}
+	cfg.Target = vegeta.Target{
+		Method: "POST",
+		URL:    path,
+		Body:   payload,
+		Header: header,
+	}
+	return nil
+}
+func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) {
+	storeID := uint64(0)
+	for _, rowID := range x.Rows {
+		cfg.AddFeature(index, field, rowID, "", false, hasTime)
+		storeID = rowID
+	}
+	storeKey := ""
+	for _, rowKey := range x.Keys {
+		cfg.AddFeature(index, field, 0, rowKey, true, hasTime)
+		storeKey = rowKey
+	}
+	idx := cfg.IndexMap[index]
+	if isSet {
+		if storeID > 0 {
+			idx.Stores = append(idx.Stores, IndexFieldRow{
+				Index: index,
+				Field: field,
+				RowID: storeID + 1,
+			})
+		}
+		if storeKey != "" {
+			idx.Stores = append(idx.Stores, IndexFieldRow{
+				Index:    index,
+				Field:    field,
+				RowKey:   storeKey + "_1",
+				IsRowKey: true,
+			})
+
+		}
+	}
+}
+
+const maxEffectiveRange = 1000000
+
+func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Decimal, scale int64, decimal bool) {
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+
+	effectiveRange := uint64(max.Value) - uint64(min.Value) + 1
+	// if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which
+	// wraps to 0. Anything closer together will be fine. We accept the loss
+	// of accuracy in the range from not representing quite the full value
+	// in that edge case.
+	if effectiveRange == 0 {
+		effectiveRange--
+	}
+
+	// we assume that the Value of the field is already scaled, I guess?
+	newRange := IndexFieldRange{
+		Index:    index,
+		Field:    field,
+		Min:      min.Value,
+		Max:      max.Value,
+		Scale:    scale,
+		ScaleDiv: math.Pow(10, float64(scale)),
+		Range:    effectiveRange,
+	}
+	f.Ranges = append(f.Ranges, newRange)
+	if !decimal {
+		f.Distinctables = append(f.Distinctables, newRange)
+	}
+	// We want to add more values for larger int fields, but the
+	// default KitchenSink field has a range of 1<<64 which would make
+	// it completely dominate weights, so...
+	if effectiveRange > maxEffectiveRange {
+		effectiveRange = maxEffectiveRange
+	}
+	f.RangeWeight += int(effectiveRange)
+}
+
+func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
+	tree := cfg.GenTree(index, cfg.TreeDepth)
+	pql = tree.ToPQL()
+
+	dice := cfg.Rnd.Intn(9)
+	if dice == 3 { // 1 in 9 of getting a store
+		idx := cfg.IndexMap[index]
+		if len(idx.Stores) > 0 {
+			i := cfg.Rnd.Intn(len(idx.Stores))
+			fr := idx.Stores[i]
+			var key string
+			if fr.IsRowKey {
+				key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey)
+				c := strings.LastIndex(fr.RowKey, "_")
+				n, err := strconv.Atoi(fr.RowKey[c+1:])
+				vprint.PanicOn(err)
+				n += 1
+				fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n)
+			} else {
+				key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID)
+				fr.RowID = fr.RowID + 1
+			}
+			idx.Stores[i] = fr
+			pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key)
+			return
+		}
+	}
+	pql = fmt.Sprintf("Count(%v)", pql)
+	return
+}
+
+type Tree struct {
+	Chd  []*Tree
+	S    string
+	Args []string // Extra args to pass after children, such as a field for Distinct.
+}
+
+func (tr *Tree) StringIndent(ind int) (s string) {
+	spc := strings.Repeat("    ", ind)
+	spc1 := strings.Repeat("    ", ind+1)
+	var chds []string
+	leaf := true
+	if len(tr.Chd) == 0 {
+		// leaf
+	} else {
+		leaf = false
+		for _, chd := range tr.Chd {
+			chds = append(chds, chd.StringIndent(ind+1))
+		}
+	}
+	if leaf {
+		s += fmt.Sprintf("%v %v\n", spc1, tr.S)
+	} else {
+		for i, c := range chds {
+			if i == 0 {
+				s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c)
+			} else {
+				s += fmt.Sprintf("%v", c)
+			}
+		}
+	}
+	return
+}
+
+const pilosaTimeFmt = "2006-01-02T15:04"
+
+func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
+	features := cfg.IndexMap[index]
+	if depth == 0 {
+		return features.RandomQuery(cfg)
+	}
+
+	r := cfg.Rnd.Intn(len(cfg.BitmapFunc))
+	f := cfg.BitmapFunc[r]
+	tr = &Tree{S: f}
+	numChild := 2
+	switch f {
+	case "Union", "Intersect", "Xor":
+		numChild = cfg.Rnd.Intn(8) + 2
+	case "Not":
+		numChild = 1
+	case "Difference":
+		numChild = 2
+	case "Distinct":
+		numChild = 1
+		r = cfg.Rnd.Intn(len(features.Distinctables))
+		tr.Args = append(tr.Args, fmt.Sprintf("field=%s", features.Distinctables[r].Field))
+	}
+	for i := 0; i < numChild; i++ {
+		tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1))
+	}
+	return
+}
+
+func (tr *Tree) ToPQL() (s string) {
+	if len(tr.Chd) == 0 {
+		// leaf
+		return tr.S
+	}
+
+	var chds []string
+	for _, c := range tr.Chd {
+		chds = append(chds, c.ToPQL())
+	}
+	// If we had no extra args, this does nothing.
+	chds = append(chds, tr.Args...)
+	all := strings.Join(chds, ", ")
+	return fmt.Sprintf("%v(%v)", tr.S, all)
+}
+
+func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool, hasTime bool) {
+
+	f, ok := cfg.IndexMap[index]
+	if !ok {
+		f = &Features{}
+		cfg.IndexMap[index] = f
+	}
+	f.Slc = append(f.Slc, IndexFieldRow{
+		Index:    index,
+		Field:    field,
+		RowID:    rowID,
+		RowKey:   rowKey,
+		IsRowKey: isRowKey,
+		HasTime:  hasTime,
+	})
+	f.SlcWeight++
+}

cmd/random-query/main_test.go

@@ -0,0 +1,152 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"context"
+	"math/rand"
+	"strconv"
+	"testing"
+
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/boltdb"
+	"github.com/molecula/featurebase/v3/server"
+	"github.com/molecula/featurebase/v3/test"
+	. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
+)
+
+func Test_RandomQuery(t *testing.T) {
+
+	cfg := NewRandomQueryConfig()
+
+	nNodes := 1
+	nReplicas := 1
+
+	name := t.Name()
+	var nodeid []string
+	for i := 0; i < nNodes; i++ {
+		// work around a bug in the test.MustRunCluster that corrupts
+		// the .topology file if we only join name with one "_" underscore.
+		nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
+	}
+
+	c := test.MustRunCluster(t, nNodes,
+		[]server.CommandOption{
+			server.OptCommandServerOptions(
+				pilosa.OptServerNodeID(nodeid[0]),
+				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
+				pilosa.OptServerReplicaN(nReplicas),
+			)},
+	)
+	defer c.Close()
+
+	var nodes []*test.Command
+	var dirs []string
+	for i := 0; i < nNodes; i++ {
+		nd := c.GetNode(i)
+		nodes = append(nodes, nd)
+		dirs = append(dirs, nd.Server.Holder().Path())
+	}
+	_ = dirs
+
+	ctx := context.Background()
+
+	indexes := []string{"rick"}
+	fieldName := []string{"f"}
+	idx := make([]*pilosa.Index, len(indexes))
+	field := make([]*pilosa.Field, len(indexes))
+
+	var err error
+
+	for i := range indexes {
+
+		idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		if idx[i].CreatedAt() == 0 {
+			t.Fatal("index createdAt is empty")
+		}
+
+		field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+		if field[i].CreatedAt() == 0 {
+			t.Fatal("field createdAt is empty")
+		}
+	}
+
+	timestamp := int64(0)
+
+	for i := range indexes {
+
+		// Generate some keyed records.
+		rowIDs := []uint64{}
+		timestamps := []int64{}
+		N := 10
+		for j := 1; j <= N; j++ {
+			rowIDs = append(rowIDs, uint64(j))
+			timestamps = append(timestamps, timestamp)
+		}
+
+		var colKeys []string
+		switch i {
+		case 0:
+			// Keys are sharded so ordering is not guaranteed.
+			colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
+			colKeys = colKeys[:N]
+		case 1:
+			colKeys = []string{"col11", "col12"}
+			N = len(colKeys)
+			rowIDs = rowIDs[:N]
+			timestamps = timestamps[:N]
+		}
+
+		// Import data with keys to the primary and verify that it gets
+		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		req := &pilosa.ImportRequest{
+			Index:          indexes[i],
+			IndexCreatedAt: idx[i].CreatedAt(),
+			Field:          fieldName[i],
+			FieldCreatedAt: field[i].CreatedAt(),
+
+			// even though this says Shard: 0, that won't matter. The column keys
+			// get hashed and that decides the actual shard.
+			Shard:      0,
+			RowIDs:     rowIDs,
+			ColumnKeys: colKeys,
+			Timestamps: timestamps,
+		}
+		//vv("rowIDs = '%#v'", rowIDs)
+		//vv("colKeys = '%#v'", colKeys)
+
+		qcx := nodes[0].API.Txf().NewQcx()
+
+		if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
+			t.Fatal(err)
+		}
+		PanicOn(qcx.Finish())
+		//qcx.Reset()
+	}
+	// end of setup.
+	cfg.Index = indexes[0]
+	PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
+
+	for j := 0; j < 4; j++ {
+		index := indexes[rand.Intn(len(indexes))]
+
+		pql, err := cfg.GenQuery(index)
+		PanicOn(err)
+
+		//vv("pql = '%v'", pql)
+
+		// Query node0.
+		res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = res
+		//vv("success on pql = '%v'; res='%v'", pql, res.Results[0])
+	}
+}